from typing import List, Tuple

import polars as pl

from focus_converter.configs.base_config import (
    ConversionPlan,
    MissingColumnDType,
    SetColumnDTypesConversionArgs,
)
from focus_converter.conversion_functions.validations import ColumnValidator

class DeferredColumnFunctions:
    """
    # Set of functions that can only be executed once the lazyframe is loaded and the column names available
    # are known.
    """

    def __init__(self):
        # missing_column_plans = []
        self.__missing_column_plans__: List[Tuple[str, ConversionPlan]] = []

        # enforced column dtypes, need to be applied before any other conversion
        # if column is present then a cast operation can be applied, if not then a new column can be added
        # with null values and then cast operation can be applied
        self.__enforced_column_dtypes__: List[ConversionPlan] = []

    @staticmethod
    def convert_focus_data_type_polars_dtype(focus_data_type):
        if focus_data_type == "string":
            return pl.Utf8
        elif focus_data_type == "float":
            return pl.Float64
        elif focus_data_type == "int":
            return pl.Int64
        elif focus_data_type == "datetime":
            return pl.Datetime
        elif focus_data_type == "date":
            return pl.Date
        else:
            raise RuntimeError(f"data_type: {focus_data_type} not implemented")

    def map_missing_column_plan(
        self, plan: ConversionPlan, column_alias, column_validator: ColumnValidator
    ):
        self.__missing_column_plans__.append((column_alias, plan))
        column_validator.map_static_default_value_if_not_present(
            plan=plan, column_alias=column_alias
        )

    def map_dtype_plan(self, plan: ConversionPlan, column_validator: ColumnValidator):
        self.__enforced_column_dtypes__.append(plan)
        column_validator.map_dtype_enforced_node(plan=plan)

    def apply_missing_column_plan(self, lf: pl.LazyFrame):
        for column_alias, missing_column_plan in self.__missing_column_plans__:
            if missing_column_plan.column not in lf.columns:
                conversion_arg: MissingColumnDType = MissingColumnDType.model_validate(
                    missing_column_plan.conversion_args
                )

                if conversion_arg.data_type == "string":
                    dtype = pl.Utf8
                elif conversion_arg.data_type == "float":
                    dtype = pl.Float64
                elif conversion_arg.data_type == "int":
                    dtype = pl.Int64
                else:
                    raise RuntimeError(
                        f"data_type: {conversion_arg.data_types} not implemented"
                    )

                lf = lf.with_columns(
                    pl.lit(None).cast(dtype).alias(missing_column_plan.column)
                )
            else:
                lf = lf.with_columns(
                    pl.col(missing_column_plan.column).alias(missing_column_plan.column)
                )
        return lf

    def apply_dtype_plan(self, lf: pl.LazyFrame):
        for plan in self.__enforced_column_dtypes__:
            conversion_args = SetColumnDTypesConversionArgs.model_validate(
                plan.conversion_args
            )
            for column_obj in conversion_args.dtype_args:
                if column_obj.column_name not in lf.columns:
                    lf = lf.with_columns(
                        pl.lit(None)
                        .cast(
                            self.convert_focus_data_type_polars_dtype(column_obj.dtype),
                            strict=False,
                        )
                        .alias(column_obj.column_name)
                    )
                else:
                    # check if the column is i64 for the cast to work else fail, at the point the column
                    # should be used with datetime parser plan.
                    cast_type = self.convert_focus_data_type_polars_dtype(
                        column_obj.dtype
                    )
                    if cast_type in [pl.Datetime, pl.Date]:
                        if lf.schema[column_obj.column_name] in [pl.Datetime, pl.Date]:
                            # ignore if the column is already of type datetime/date
                            pass
                        elif lf.schema[column_obj.column_name] == pl.Utf8:
                            if cast_type == pl.Datetime:
                                lf = lf.with_columns(
                                    pl.col(column_obj.column_name).str.to_datetime()
                                )
                            else:
                                lf = lf.with_columns(
                                    pl.col(column_obj.column_name).str.to_date()
                                )
                        else:
                            # possibly a timestamp column
                            lf = lf.with_columns(
                                pl.col(column_obj.column_name).cast(
                                    self.convert_focus_data_type_polars_dtype(
                                        column_obj.dtype
                                    ),
                                    strict=True,
                                )
                            )
                    else:
                        lf = lf.with_columns(
                            pl.col(column_obj.column_name).cast(
                                self.convert_focus_data_type_polars_dtype(
                                    column_obj.dtype
                                ),
                                strict=False,
                            )
                        )
        return lf


from pyspark.sql import SparkSession
spark = SparkSession.builder.master("local[*]").getOrCreate()

import json
import numpy as np
import sys

sc = spark.sparkContext
text = sc.textFile(sys.argv[1])

df_fin =text.map(lambda x: json.loads(x))

def myFuncA(x):
    return ((x['reviewerID'], x['asin']),(x['overall'], x['unixReviewTime'], x['verified']))

OneRatingRDD = df_fin.map(lambda x: myFuncA(x)).groupByKey().map(lambda x: (x[0], list(x[1])[-1]))#.map(lambda x: func2(x))

#print(OneRatingRDD.take(2))

item25RDD = OneRatingRDD.map(lambda x: (x[0][1], (x[0][0],x[1][0]))).groupByKey().filter(lambda x: len(list(x[1]))>=25)

#print(item25RDD.take(2))

def interchange(x):
    result = []
    for i in x[1]:
        result.append((i[0], (x[0], i[1])))
    return result

User5itemsRDD = item25RDD.flatMap(lambda x: interchange(x)).groupByKey().filter(lambda x: len(list(x[1]))>=5)
#User5itemsRDD = OneRatingRDD.map(lambda x: (x[0][0], (x[0][1],x[1][0]))).groupByKey().filter(lambda x: len(list(x[1]))>=5)

#print(User5itemsRDD.take(1))

itemRDD = User5itemsRDD.flatMap(lambda x: interchange(x)).groupByKey()

#print(len((itemRDD.collect())))

def latFunc(x):
    #result = []
    count =0
    # userList = []
    # for i in fwBC.value:
    #     userList.append(i[0])
    for i in x[1]:
        if i[0] in fwBC.value:
            count+=1

    if count>1:
        return x

def meanCenter(x):
    result = []
    total = 0
    for i in x[1]:
        #i2 = list(i)
        total+= i[1]
    mean = float(total)/len(x[1])

    for i in x[1]:
        i2 = list(i)
        i2[1] = i2[1]-mean
        result.append((i2[0], i2[1], i2[1]+mean))

    return (x[0],result)

def cosine(x):
    l = meanList.value
    item1 = l[0][0]
    item2 = x[0]
    dot =0
    sumA = 0
    sumB = 0
    sim =0
    if item1 != item2:
        for a in l[0][1]:
            for b in x[1]:
                if b[0] == a[0]:
                    dot += a[1]*b[1]
        for a in l[0][1]:
            sumA += a[1]*a[1]
        for b in x[1]:
            sumB += b[1]*b[1]
            #sumB = 1

        sim = float(dot)/(np.sqrt(sumA) * np.sqrt(sumB))

    return ((x[0], sim), x[1])

def filterUsers(x):
    myList = []
    for itemId in itemList.value:
        myList.append(itemId[0][0])
    #result = []
    count = 0
    if(x[0] not in fwBC.value):
        for i in x[1]:
            if i[0] in myList:
                count+=1
        if(count>=2):
            return x
    #return result

def utility(x):
    num =0
    den =0
    for a, b in itemList.value:
        for record in b:
            if(record[0]==x[0]):
                num += a[1]*record[2]
                den += a[1]
    result = num/den

    return (x[0], result)

input  = sys.argv[2]
inputList = map(str, input.strip('[]').split(','))
for each in inputList:
    myItem = each.strip()
    myItem = myItem.replace("'", '')
    #print(myItem)
    broadv= itemRDD.filter(lambda x : x[0] == myItem).flatMap(lambda x: list(x[1])).map(lambda x: x[0]).collect()
    fwBC = sc.broadcast(broadv)
    neighborRDD = itemRDD.map(lambda x: (x[0], list(x[1]))).filter(lambda x: latFunc(x)).map(lambda x: meanCenter(x)) # a (ii)
    broadList = neighborRDD.filter(lambda x: x[0] == myItem).collect()
    meanList = sc.broadcast(broadList)
    cosineRDD = neighborRDD.map(lambda x: cosine(x)).filter(lambda x: x[0][1]>0)
    mynewRDD = sc.parallelize(cosineRDD.take(50))
    fItems = mynewRDD.collect()
    itemList = sc.broadcast(fItems)
    allUsers = User5itemsRDD.filter(lambda x: filterUsers(x))
    answerRDD = allUsers.map(lambda x: utility(x))#.sortByKey()
    #print(len(answerRDD.collect()))
    print('')
    print('Product id: '+myItem)
    for x in answerRDD.collect():
        print(x)
    print('\n \n')


'''
Created on 03/05/2015

@author: arsenioferreira
'''
import wx
import matplotlib
matplotlib.use('WXAgg')
from matplotlib.backends.backend_wxagg import FigureCanvasWxAgg as FigureCanvas
from matplotlib.backends.backend_wx import NavigationToolbar2Wx
from matplotlib.figure import Figure
from pandas.tools.plotting import scatter_matrix
from pandas import DataFrame
import numpy as np
import pylab
from py.una.pol.tava.model.mresult import ResultModel as rm
from py.una.pol.tava.model.miteration import InterationModel as im

class PanelScatterMatrix(wx.Panel):
    '''
    Clase que contiene la Figura que representa a la matriz dispersa
    '''
    def __init__(self, parent):
        wx.Panel.__init__(self, parent)
        self.figure = Figure()
        self.axes = self.figure.add_subplot(111, xlim=(0, 1), ylim=(0, 1),
                                 autoscale_on=False)
        self.canvas = FigureCanvas(self, -1, self.figure)
        self.sizer = wx.BoxSizer(wx.VERTICAL)
        self.sizer.Add(self.canvas, 1, wx.LEFT | wx.TOP | wx.GROW)
        self.add_toolbar()
        self.SetSizer(self.sizer)
        self.Fit()

    def add_toolbar(self):
        """copied verbatim from embedding_wx2.py"""
        self.toolbar = NavigationToolbar2Wx(self.canvas)
        self.toolbar.Realize()
        tw, th = self.toolbar.GetSizeTuple()
        fw, fh = self.canvas.GetSizeTuple()
        self.toolbar.SetSize(wx.Size(fw, th))
        self.sizer.Add(self.toolbar, 0, wx.LEFT | wx.EXPAND)
        self.toolbar.update()

    def setParameters(self, iterDir):
        self.iterId = iterDir

    def draw(self):
        from py.una.pol.tava.dao.dindividual import getIndividualsByIteracionId
        individuals = getIndividualsByIteracionId(self.iterId)
        data = ""
#         c = 0
        for individual in individuals:
            data += str(individual.objectives) + "\n"
#             if c == 3:
#                 break  #sacarrrrr
#             c = c + 1
        from StringIO import StringIO
        datos = np.genfromtxt(StringIO(data), delimiter=',')
        data = None
        data = datos
        pylab.clf()

        df = DataFrame(data)

        axs = scatter_matrix(df, alpha=0.2, ax=self.axes)
        n = len(df.columns)
        for x in range(n):
            for y in range(n):
                # to get the axis of subplots
                ax = axs[x, y]
                # to make x axis name vertical
                ax.xaxis.label.set_rotation(90)
                # to make y axis name horizontal
                ax.yaxis.label.set_rotation(0)
                # to make sure y axis names are outside the plot area
                ax.yaxis.labelpad = 50

class PanelScatterMatrixCongif(wx.Panel):
    def __init__(self, parent, testConfig):
        super(PanelScatterMatrixCongif, self).__init__(parent)

        self.initParameters(testConfig)

        sizer = wx.BoxSizer(wx.HORIZONTAL)

        # Parameters
        sbox = wx.StaticBox(self, label="Parameters")
        sboxsp = wx.StaticBoxSizer(sbox, wx.VERTICAL)

        # Attributes
        self.choice = wx.Choice(self, size=(200, 30), choices=self.itemsCombo)
        self.choice.SetSelection(0)
        csizer = wx.BoxSizer(wx.VERTICAL)
        csizer.Add(wx.StaticText(self, label="Select one: "))
        csizer.Add(self.choice)
        sboxsp.Add(csizer, 0, wx.ALL, 8)

        apply_button = wx.Button(self, 10, "Apply", (20, 20))
        apply_button.SetDefault()
        self.Bind(wx.EVT_BUTTON, self.OnButtonApply, apply_button)

        msizer = wx.BoxSizer(wx.VERTICAL)

        msizer.Add(sboxsp, 0, wx.ALL, 7)
        msizer.Add(apply_button, 0, wx.EXPAND | wx.ALL, 5)

        vsizer = wx.BoxSizer(wx.VERTICAL)
        self.scatter_matrix_panel = PanelScatterMatrix(self)

        # Tomamos el id de la primera iteracion
        iterId = testConfig.test_details[0].test_datas[0].iteration_id

        self.scatter_matrix_panel.setParameters(iterId)
        self.scatter_matrix_panel.draw()
        vsizer.Add(self.scatter_matrix_panel, 3, wx.EXPAND)

        sizer.Add(msizer, 0, wx.EXPAND)
        sizer.Add(vsizer, 1, wx.EXPAND)

        self.testConfig = testConfig
        self.SetSizer(sizer)
        self.SetSize((900, 710))
        self.Fit()

    def initParameters(self, testConfig):
        # Items para el combo de opciones
        self.itemsCombo = []
        # Diccionario empleado para almacenar los id de iteraciones asociadas a
        # cada opcion en el combo
        self.itemsDict = {}

        for tdet in testConfig.test_details:
            result = rm().getResultById(tdet.result_id)
            for tdat in tdet.test_datas:
                ite = im().getIterationById(tdat.iteration_id)
                item = str(result.name) + "\nIteration " +\
                                                        str(ite.identifier)
                self.itemsCombo.append(item)
                self.itemsDict[item] = tdat.iteration_id

    def OnButtonApply(self, event):
        iterId = self.itemsDict[self.choice.GetStringSelection()]
        self.scatter_matrix_panel.setParameters(iterId)
        self.scatter_matrix_panel.draw()
        self.scatter_matrix_panel.canvas.draw()

if __name__ == "__main__":
    from py.una.pol.tava.model.mtestconfig import TestConfigModel as tm
    testConfig = tm().getTestConfigById(1)
    app = wx.App()
    fr = wx.Frame(None, title='Configuration')
    panel = PanelScatterMatrixCongif(fr, testConfig)
    fr.Centre(wx.BOTH)
    fr.Fit()
    fr.Show()
    app.MainLoop()


import uuid

import flask
from flask import request
import json

from BusinessLayer.AnswerManager import AnswerManager
from BusinessLayer.IntentManager import IntentManager
from CustomConfigurations.AnswerRepositoryConfiguration import AnswerRepositoryConfiguration
from CustomConfigurations.IntentRepositoryConfiguration import IntentRepositoryConfiguration
from DataAccessLayer.AnswerRepository import AnswerRepository
from DataAccessLayer.IntentRepository import IntentRepository
from app import app

# export GOOGLE_APPLICATION_CREDENTIALS="/Users/serhatkumas/Desktop/BasicExternalDialogflowApi/authentication.json"

# Project Configuration
project_id = "chatbotapiproject-dsnf"
session_id = str(uuid.uuid4())
language_code = "en-US"

# Manager Object (Business Layer) Creation
answer_api_controller = AnswerManager(
    AnswerRepository(
        AnswerRepositoryConfiguration(
            project_id, session_id, language_code
        )
    )
)

# Manager Object (Business Layer) Creation
intent_api_controller = IntentManager(
    IntentRepository(
        IntentRepositoryConfiguration(project_id)
    )
)

# Configuration display end point
@app.route('/display-intent-repository-config')
def display_intent_repository_config_information():
    answer = intent_api_controller.display_repository_config_information()
    return answer

# All intents chatbot has, display web page
@app.route('/all-intents-page', methods=["GET"])
def get_all_intents_page():
    return flask.render_template("IntentListPage.html")

# All intents chatbot has, display end point
@app.route('/get-all-intents')
def get_all_intents():
    answer = intent_api_controller.get_all_intents()
    return json.dumps(answer)

@app.route('/all-intents-page-info')
def get_all_intent_page_info():
    info = []
    answer = intent_api_controller.get_all_intents()
    for intent in answer:
        info.append(intent)
        info.append(answer_api_controller.ask_question(intent))
    return json.dumps(info)

# Creation of intent by json data end point
@app.route('/create-intent-by-json', methods=['POST'])
def create_intent_by_json():
    number_of_created_intents = 0
    intents = request.json.get("intents")
    for intent in intents:
        display_name = intent.get('display_name')
        training_phase = intent.get('training_phase').split(',')
        message_text = intent.get('message_text').split(',')
        intent_api_controller.create_intent(display_name, training_phase, message_text)
        number_of_created_intents += 1
    return "{} new intent is created".format(number_of_created_intents)

# Display name update end point
@app.route('/update-display-name')
def update_display_name():
    content_type = request.headers.get('Content-Type')
    if content_type == 'application/json':
        json_data = request.get_json()
        display_name_from = json_data["display_name_from"]
        display_name_to = json_data["display_name_to"]
        answer = intent_api_controller.update_display_name_of_intent_by_display_name(display_name_from, display_name_to)
        return answer
    else:
        return 'Content-Type not supported!'

# Intent deletion end point
@app.route('/delete-intent')
def delete_intent_by_display_name():
    content_type = request.headers.get('Content-Type')
    if content_type == 'application/json':
        json_data = request.get_json()
        display_name = json_data["display_name"]
        answer = intent_api_controller.delete_intent_by_display_name(display_name)
        return answer
    else:
        return 'Content-Type not supported!'


from datetime import tzinfo, timedelta, datetime
from BaseLog import *

ZERO = timedelta(0)
HOUR = timedelta(hours=1)

# A UTC class.

class UTC(tzinfo):
    """UTC"""

    def utcoffset(self, dt):
        return ZERO

    def tzname(self, dt):
        return "UTC"

    def dst(self, dt):
        return ZERO

utc = UTC()

# A class building tzinfo objects for fixed-offset time zones.
# Note that FixedOffset(0, "UTC") is a different way to build a
# UTC tzinfo object.

class FixedOffset(tzinfo):
    """Fixed offset in minutes east from UTC."""

    def __init__(self, offset, name):
        self.__offset = timedelta(minutes = offset)
        self.__name = name

    def utcoffset(self, dt):
        return self.__offset

    def tzname(self, dt):
        return self.__name

    def dst(self, dt):
        return ZERO

# A class capturing the platform's idea of local time.

import time as _time

STDOFFSET = timedelta(seconds = -_time.timezone)
if _time.daylight:
    DSTOFFSET = timedelta(seconds = -_time.altzone)
else:
    DSTOFFSET = STDOFFSET

DSTDIFF = DSTOFFSET - STDOFFSET

class LocalTimezone(tzinfo):

    def utcoffset(self, dt):
        if self._isdst(dt):
            return DSTOFFSET
        else:
            return STDOFFSET

    def dst(self, dt):
        if self._isdst(dt):
            return DSTDIFF
        else:
            return ZERO

    def tzname(self, dt):
        return _time.tzname[self._isdst(dt)]

    def _isdst(self, dt):
        tt = (dt.year, dt.month, dt.day,
              dt.hour, dt.minute, dt.second,
              dt.weekday(), 0, -1)
        stamp = _time.mktime(tt)
        tt = _time.localtime(stamp)
        return tt.tm_isdst > 0

Local = LocalTimezone()

# A complete implementation of current DST rules for major US time zones.

def first_sunday_on_or_after(dt):
    days_to_go = 6 - dt.weekday()
    if days_to_go:
        dt += timedelta(days_to_go)
    return dt

# In the US, DST starts at 2am (standard time) on the first Sunday in April.
DSTSTART = datetime(1, 4, 1, 2)
# and ends at 2am (DST time; 1am standard time) on the last Sunday of Oct.
# which is the first Sunday on or after Oct 25.
DSTEND = datetime(1, 10, 25, 1)

class USTimeZone(tzinfo):

    def __init__(self, hours, reprname, stdname, dstname):
        self.stdoffset = timedelta(hours=hours)
        self.reprname = reprname
        self.stdname = stdname
        self.dstname = dstname

    def __repr__(self):
        return self.reprname

    def tzname(self, dt):
        if self.dst(dt):
            return self.dstname
        else:
            return self.stdname

    def utcoffset(self, dt):
        return self.stdoffset + self.dst(dt)

    def dst(self, dt):
        if dt is None or dt.tzinfo is None:
            # An exception may be sensible here, in one or both cases.
            # It depends on how you want to treat them.  The default
            # fromutc() implementation (called by the default astimezone()
            # implementation) passes a datetime with dt.tzinfo is self.
            return ZERO
        assert dt.tzinfo is self

        # If there is no DST name specified, then the timezone doesn't have
        # DST
        if(self.dstname is None):
            return ZERO
        else:
            # TODO - need to figure if we are looking at the old or new
            # way to calculate DST

            # New way
            # Find first Sunday in April & the last in October.
            start = first_sunday_on_or_after(DSTSTART.replace(year=dt.year))
            end = first_sunday_on_or_after(DSTEND.replace(year=dt.year))

            # Can't compare naive to aware objects, so strip the timezone from
            # dt first.
            if start <= dt.replace(tzinfo=None) < end:
                return HOUR
            else:
                return ZERO

Eastern  = USTimeZone(-5, "Eastern",  "EST", "EDT")
Central  = USTimeZone(-6, "Central",  "CST", "CDT")
Mountain = USTimeZone(-7, "Mountain", "MST", "MDT")
Pacific  = USTimeZone(-8, "Pacific",  "PST", "PDT")
Hawaii = USTimeZone(-10, "Hawaii", "HST", None)
NewZealand = USTimeZone(13, "NewZeland", "NZST", "NZDT")
UTC = USTimeZone(0, "Greenwich", "UTC", None)

tz_lookup = {
    'EST' : Eastern, 'EDT' : Eastern,
    'CST' : Central, 'CDT' : Central,
    'MST' : Mountain, 'MDT' : Mountain,
    'PST' : Pacific, 'PDT' : Pacific,
    'HST' : Hawaii,
    'NZST': NewZealand, 'NZDT' : NewZealand,
    'UTC' : UTC,
    'GMT' : UTC
    }

def convert_commline_to_utc(ts_string, time_zone):
    '''Given a entry from the comm.log of the form

    %a %b %d %H:%M:%S %Y

    and a specified time zone, as a string, this routine will
    convert that line to a time_struc in utc
    '''

    import time
    import datetime

    loc_t = time.strptime(ts_string, "%a %b %d %H:%M:%S %Y")
    try:
        t_zone = tz_lookup[time_zone.upper()]
    except:
        log_error("Unknown timezone %s - assuming UTC" % time_zone, max_count=5)
        t_zone = tz_lookup['UTC']

    ts = datetime.datetime(loc_t.tm_year, loc_t.tm_mon, loc_t.tm_mday, loc_t.tm_hour, loc_t.tm_min, loc_t.tm_sec, tzinfo=t_zone)
    return ts.utctimetuple()

if __name__ == "__main__":
    import os
    import time
    # Force time and date to be in UTC
    os.environ['TZ'] = 'UTC'
    time.tzset()

    base_opts = BaseOpts.BaseOptions("Testing for timezone handling")
    BaseLogger(base_opts) # initializes BaseLog

    ts_string = "Mon Jan 4 12:59:13 2007"
    time_zone = 'FOO'
    ts = convert_commline_to_utc(ts_string, time_zone)
    print(("%s %s" % (ts_string, time_zone)))
    print((time.strftime("%a %b %d %H:%M:%S %Y %Z", ts)))


from flask import Flask, request, render_template, redirect
import joblib
import requests

# Declare a Flask app
app = Flask(__name__)

@app.route('/', methods=['GET', 'POST'])
def main():
    if request.method == "POST":
        if request.form.get("predict")=="predict":
            return redirect("/predict")
        if request.form.get("dashboard")=="dashboard":
            return redirect("/dashboard")
    return render_template("website.html")

@app.route('/dashboard', methods=['GET', 'POST'])
def dashboard():
    return render_template("dashboard.html")

@app.route('/predict', methods=['GET', 'POST'])
def predict():
    if request.method == "POST":
        import pickle
        import numpy as np
        model = pickle.load(open(r"./heart_disease.sav", "rb"))
        scaled = pickle.load(open(r"./scaler.sav", "rb"))
        # Get values through input bars
        name = request.form.get("name")
        age =   request.form.get('age')
        sex =   request.form.get('sex')
        cp =   request.form.get('cp')
        trestbps =   request.form.get('trestbps')
        chol =   request.form.get('chol')
        fbs =   request.form.get('fbs')
        restecg =   request.form.get('restecg')
        thalach =   request.form.get('thalach')
        exang =   request.form.get('exang')
        oldpeak =   request.form.get('oldpeak')
        slope =   request.form.get('slope')
        ca =   request.form.get('ca')
        thal =   request.form.get('thal')

        user_input = [age,sex,cp,trestbps,chol,fbs,restecg,thalach,exang,oldpeak,slope,ca,thal]
        user_input = np.array(user_input)
        user_input = user_input.reshape(1,-1)
        user_input = scaled.fit_transform(user_input)
        prediction = model.predict(user_input)
         # token_response = requests.post('https://iam.cloud.ibm.com/identity/token', data={"apikey":
        # API_KEY, "grant_type": 'urn:ibm:params:oauth:grant-type:apikey'})
        # mltoken = token_response.json()["access_token"]
        # header = {'Content-Type': 'application/json', 'Authorization': 'Bearer ' + mltoken}
        # payload_scoring = {"input_data": [{"fields": ["Age","Sex","Chest pain type","BP","Cholesterol","FBS over 120","EKG results","Max HR","Exercise angina","ST depression","Slope of ST","Number of vessels fluro","Thallium"], "values": [[age,sex,cp,trestbps,chol,fbs,restecg,thalach,exang,oldpeak,slope,ca,thal]]}]}
        # response_scoring = requests.post('https://eu-gb.ml.cloud.ibm.com/ml/v4/deployments/2c94f5e4-04ea-43dd-9d5a-c0365c95b676/predictions?version=2022-11-10', json=payload_scoring,
        # headers={'Authorization': 'Bearer ' + mltoken})
        # response_scoring = response_scoring.json()
        # print(response_scoring)
        # # Get prediction
        # prediction = response_scoring["predictions"][0]["values"][0][0]
        if prediction=='Presence':
            output = "Sorry "+name+", you are at high risk of having a heart disease. Please consult a doctor as soon as possible"
            image = "../static/high-risk.jpg"
        elif prediction=='Absence':
            output = "Hi "+name+", you are at low risk of having a heart disease. If you are still not convinced please consult a doctor"
            image = "../static/low-risk.jpg"
        else:
            output = "Hey "+name+", there was some error processing your details. Please try again later."
            image = "../static/error.png"

        return render_template("result.html", output = output, result=image)

    return render_template("predictor.html")

# Running the app
if __name__ == '__main__':
    app.run(debug = True)


import os
import time
import shutil
import torch
import torch.nn as nn
import numpy as np
import pickle
from .kNN import kNNClassify
from .util import *
from sklearn.metrics import accuracy_score

def train(image_model, att_model, mod_model, mod_transformer, attn_img, attn_att, decoder_img, decoder_att,
          train_loader, test_seen_loader, test_unseen_loader, args):
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    torch.set_grad_enabled(True)
    # Initialize all of the statistics we want to keep track of
    batch_time = AverageMeter()
    data_time = AverageMeter()
    loss_meter = AverageMeter()
    progress = []
    best_epoch, best_acc = 0, -np.inf
    global_step, epoch = 0, 0
    start_time = time.time()
    exp_dir = args.exp_dir
    save_model_dir = os.path.join(exp_dir,'models')
    if not os.path.exists(save_model_dir):
        os.makedirs(save_model_dir)
    def _save_progress():
        progress.append([epoch, global_step, best_epoch, best_acc,
                time.time() - start_time])
        with open("%s/progress.pkl" % exp_dir, "wb") as f:
            pickle.dump(progress, f)

    epoch = 0

    if epoch != 0:
        image_model.load_state_dict(torch.load("%s/models/image_model.%d.pth" % (exp_dir, epoch)))
        att_model.load_state_dict(torch.load("%s/models/att_model.%d.pth" % (exp_dir, epoch)))
        print("loaded parameters from epoch %d" % epoch)

    # Memory
    memory = Variable(torch.normal(mean=0., std=0.5, size=(
        args.num_gst, args.out_DIM // args.num_heads)), requires_grad=True).to(device)

    image_model = image_model.to(device)
    att_model = att_model.to(device)
    mod_model = mod_model.to(device)
    mod_transformer = mod_transformer.to(device)
    attn_img = attn_img.to(device)
    attn_att = attn_att.to(device)
    decoder_img = decoder_img.to(device)
    decoder_att = decoder_att.to(device)
    # Set up the optimizer
    image_trainables = [p for p in image_model.parameters() if p.requires_grad] # if p.requires_grad
    att_trainables = [p for p in att_model.parameters() if p.requires_grad]
    mod_trainables = [p for p in mod_model.parameters() if p.requires_grad]
    mod_trans_trainables = [p for p in mod_transformer.parameters() if p.requires_grad]
    attn_img_trainables = [p for p in attn_img.parameters() if p.requires_grad]
    attn_att_trainables = [p for p in attn_att.parameters() if p.requires_grad]
    recon_img_trainables = [p for p in attn_att.parameters() if p.requires_grad]
    recon_att_trainables = [p for p in attn_att.parameters() if p.requires_grad]

    if args.optim == 'sgd':
        optimizer = torch.optim.SGD(att_trainables, args.lr_A,
                                    momentum=args.momentum,
                                    weight_decay=args.weight_decay)
        optimizer_img = torch.optim.SGD(image_trainables, args.lr_I,
                                    momentum=args.momentum,
                                    weight_decay=args.weight_decay)
        optimizer_mod = torch.optim.SGD(mod_trainables, args.lr_M,
                                    momentum=args.momentum,
                                    weight_decay=args.weight_decay)
        optimizer_mod_trans = torch.optim.SGD(mod_trans_trainables, args.lr_MT,
                                        momentum=args.momentum,
                                        weight_decay=args.weight_decay)
        optimizer_attn_img = torch.optim.SGD(attn_img_trainables, args.lr_AI,
                                        momentum=args.momentum,
                                        weight_decay=args.weight_decay)
        optimizer_attn_att = torch.optim.SGD(attn_att_trainables, args.lr_AA,
                                        momentum=args.momentum,
                                        weight_decay=args.weight_decay)
        optimizer_recon_img = torch.optim.SGD(recon_img_trainables, args.lr_AA,
                                             momentum=args.momentum,
                                             weight_decay=args.weight_decay)
        optimizer_recon_att = torch.optim.SGD(recon_att_trainables, args.lr_AA,
                                             momentum=args.momentum,
                                             weight_decay=args.weight_decay)

    elif args.optim == 'adam':
        optimizer = torch.optim.Adam(att_trainables, args.lr_A,
                                    weight_decay=args.weight_decay,
                                    betas=(0.95, 0.999))
        optimizer_img = torch.optim.Adam(image_trainables, args.lr_I,
                                    weight_decay=args.weight_decay,
                                    betas=(0.95, 0.999))
        optimizer_mod = torch.optim.Adam(mod_trainables, args.lr_M,
                                    weight_decay=args.weight_decay,
                                    betas=(0.95, 0.999))
        optimizer_mod_trans = torch.optim.Adam(mod_trans_trainables, args.lr_MT,
                                         weight_decay=args.weight_decay,
                                         betas=(0.95, 0.999))
        optimizer_attn_img = torch.optim.Adam(attn_img_trainables, args.lr_AI,
                                         weight_decay=args.weight_decay,
                                         betas=(0.95, 0.999))
        optimizer_attn_att = torch.optim.Adam(attn_att_trainables, args.lr_AA,
                                         weight_decay=args.weight_decay,
                                         betas=(0.95, 0.999))
        optimizer_recon_img = torch.optim.Adam(recon_img_trainables, args.lr_RI,
                                              weight_decay=args.weight_decay,
                                              betas=(0.95, 0.999))
        optimizer_recon_att = torch.optim.Adam(recon_att_trainables, args.lr_RA,
                                              weight_decay=args.weight_decay,
                                              betas=(0.95, 0.999))
    else:
        raise ValueError('Optimizer %s is not supported' % args.optim)

    """
    if epoch != 0:
        optimizer.load_state_dict(torch.load("%s/models/optim_state.%d.pth" % (exp_dir, epoch)))
        for state in optimizer.state.values():
            for k, v in state.items():
                if isinstance(v, torch.Tensor):
                    state[k] = v.to(device)
        print("loaded state dict from epoch %d" % epoch)
    """

    print("current #steps=%s, #epochs=%s" % (global_step, epoch))
    print("start training...")
    image_model.train()
    att_model.train()
    mod_model.train()
    mod_transformer.train()
    attn_img.train()
    attn_att.train()
    decoder_img.train()
    decoder_att.train()
    criterion_hinge = nn.TripletMarginLoss(margin=1.0, p=2)
    criterion_e = nn.MSELoss()
    criterion_s = nn.CosineSimilarity()
    criterion_c = nn.CrossEntropyLoss()
    criterion_k = nn.KLDivLoss()
    criterion_m = nn.MSELoss()
    criterion_b = nn.BCELoss()

    # 载入ZSL和GZSL评估所需要的id和attr文件
    print('载入id和attr文件')
    test_attr_file = os.path.join(args.data_path, args.test_class_attr)
    test_att = np.loadtxt(test_attr_file)

    all_attr_file = os.path.join(args.data_path, args.all_class_attr)
    all_att = np.loadtxt(all_attr_file)

    test_id_file = os.path.join(args.data_path, args.test_class_id)
    test_id = np.loadtxt(test_id_file, dtype=int)

    all_id_file = os.path.join(args.data_path, args.all_class_id)
    all_id = np.loadtxt(all_id_file, dtype=int)

    pre_acc = 0
    while epoch<=500:
        epoch += 1
        adjust_learning_rate(args.lr_A, args.lr_decay, optimizer, epoch)
        adjust_learning_rate(args.lr_I, args.lr_decay, optimizer_img, epoch)
        adjust_learning_rate(args.lr_M, args.lr_decay, optimizer_mod, epoch)
        adjust_learning_rate(args.lr_MT, args.lr_decay, optimizer_mod_trans, epoch)
        adjust_learning_rate(args.lr_AI, args.lr_decay, optimizer_attn_img, epoch)
        adjust_learning_rate(args.lr_AA, args.lr_decay, optimizer_attn_att, epoch)
        adjust_learning_rate(args.lr_RI, args.lr_decay, optimizer_recon_img, epoch)
        adjust_learning_rate(args.lr_RA, args.lr_decay, optimizer_recon_att, epoch)
        end_time = time.time()
        image_model.train()
        att_model.train()
        mod_model.train()
        mod_transformer.train()
        attn_img.train()
        attn_att.train()
        decoder_img.train()
        decoder_att.train()

        for i, (image_input, att_input, cls_id, key, label) in enumerate(train_loader):

            att_input = att_input.float().to(device)
            B = att_input.size(0)
            label = label.long().to(device)
            image_input = image_input.float().to(device)
            image_input = image_input.squeeze(1)

            optimizer.zero_grad()
            optimizer_img.zero_grad()
            optimizer_mod.zero_grad()
            optimizer_mod_trans.zero_grad()
            optimizer_attn_img.zero_grad()
            optimizer_attn_att.zero_grad()
            optimizer_recon_img.zero_grad()
            optimizer_recon_att.zero_grad()

            final_image_output = None
            final_att_output = None

            # 输出图像特征向量和属性向量
            image_output = image_model(image_input)
            att_output = att_model(att_input)
            # 通过 Modality Transformer 作转换
            mt_image_output = mod_transformer(image_output)
            mt_att_output = mod_transformer(att_output)
            # Modality Classifier 分类
            modal_image_output = mod_model(image_output)
            modal_att_output = mod_model(att_output)
            # modal_image_output = mod_model(mt_image_output)
            # modal_att_output = mod_model(mt_att_output)
            # Memory Fusion
            memory_image_output = attn_img(
                image_output.unsqueeze(1), memory.repeat(args.batch_size, 1, 1))
            memory_att_otuput = attn_att(
                att_output.unsqueeze(1), memory.repeat(args.batch_size, 1, 1))
            # memory_image_output = attn_img(
            #     mt_image_output.unsqueeze(1), memory.repeat(args.batch_size, 1, 1))
            # memory_att_otuput = attn_att(
            #     mt_att_output.unsqueeze(1), memory.repeat(args.batch_size, 1, 1))

            if args.modules_used == 'memory_fusion':
                final_image_output = memory_image_output
                final_att_output = memory_att_otuput
            elif args.modules_used == 'modal_classifier':
                final_image_output = mt_image_output
                final_att_output = mt_att_output
            elif args.modules_used == 'unused' or args.modules_used == 'classifier_only':
                final_image_output = image_output
                final_att_output = att_output
            else:
                raise ValueError('Error Method')

                # print('--------------------------------------------------------------------')
                # print('Best_zsl:', pre_acc)
                # print('According_gzsl: seen=%.4f, unseen=%.4f, h=%.4f' % (pre_seen, pre_useen, pre_H))

            #print(image_input.size())
            #print(att_input.size())
            #print(image_output.size())
            #print(att_output.size())
            #torch.Size([20, 3, 244, 244])
            #torch.Size([20, 312])
            #torch.Size([20, 2048])
            #torch.Size([20, 2048])

            # pooling_ratio = round(audio_input.size(-1) / audio_output.size(-1))
            # nframes.div_(pooling_ratio)

            # loss = sampled_margin_rank_loss(image_output, audio_output,
            #     nframes, margin=args.margin, simtype=args.simtype)

            # loss = torch.pow(image_output - audio_output, 2).sum()
            # loss = criterion(image_output , audio_output)

            # image_output = normalizeFeature(image_output)
            # audio_output = normalizeFeature(audio_output)
            # neg_samples = normalizeFeature(neg_samples)
            # loss_t = criterion(image_output,audio_output,neg_samples)
            loss = 0
            if args.Loss_cont:
                loss = criterion_e(final_image_output, final_att_output) * args.gamma_cont
            if args.Loss_batch:
                lossb1,lossb2 = batch_loss(final_image_output, final_att_output,cls_id,args)
                loss_batch = lossb1 + lossb2
                loss += loss_batch*args.gamma_batch

            if args.Loss_dist:
                loss_dist1, loss_dist2  = distribute_loss(final_image_output, final_att_output)
                loss_dist = loss_dist1 + loss_dist2
                loss += loss_dist*args.gamma_dist

            if args.Loss_hinge:
                neg_pair_audio,neg_pair_image = hardest_negative_mining_pair(image_output, audio_output, cls_id)
                # neg_single_audio = hardest_negative_mining_single(audio_output,cls_id)
                # neg_single_image = hardest_negative_mining_single(image_output,cls_id)
                hinge_IAA = criterion_hinge(image_output,att_output,neg_pair_att)
                hinge_AII = criterion_hinge(att_output,image_output,neg_pair_image)
                hinge_loss = hinge_AII + hinge_IAA
                loss += hinge_loss*args.gamma_hinge

            Adv_loss = 0
            if args.Loss_modal:
                # 第一种写法，跑过一次准确率反而降低了
                # y0 = np.zeros((args.batch_size,), dtype=np.int)
                # y0 = torch.from_numpy(y0).to(device)
                # y1 = np.ones((args.batch_size,), dtype=np.int)
                # y1 = torch.from_numpy(y1).to(device)

                # modal_image_loss = criterion_c(modal_image_output, y0)
                # modal_att_loss = criterion_c(modal_att_output, y1)
                # modal_loss = (modal_image_loss + modal_att_loss) * args.gamma_modal
                # # 这里类似G的角色，试图让D无法区分，尝试让modal_loss增大，所以是负号
                # loss -= modal_loss

                # 这里是C
                # modal_loss.backward(retain_graph=True)  # reusing computational graph
                # optimizer_mod.step()

                #  试试第二种写法
                # 以下为假定
                # prob_image0输出的是判断由图像特征向量转换而来的向量之前是来自图片的概率
                # prob_image1输出的是判断由属性向量转换而来的向量之前是来自图片的概率
                temp = 1e-5
                prob_image0 = modal_image_output[:,0]
                prob_image1 = modal_att_output[:,0]
                T_loss = -torch.mean(torch.log(1 - prob_image0 + temp) + torch.log(prob_image1 + temp))
                C_loss = -torch.mean(torch.log(prob_image0 + temp) + torch.log(1. - prob_image1 + temp))
                Adv_loss = T_loss + C_loss
                loss += Adv_loss * args.gamma_modal

            recon_loss = 0
            if args.Loss_recon:
                code_img = decoder_img(final_image_output)
                code_att = decoder_att(final_att_output)
                recon_loss = criterion_m(code_img, image_output)/45 + criterion_m(code_att, att_input)/17
                loss += recon_loss * args.gamma_recon

            loss.backward()
            optimizer.step()
            optimizer_img.step()
            optimizer_mod.step()
            optimizer_mod_trans.step()
            optimizer_attn_img.step()
            optimizer_attn_att.step()
            optimizer_recon_img.step()
            optimizer_recon_att.step()

            # record loss
            loss_meter.update(loss.item(), B)
            batch_time.update(time.time() - end_time)

            if i % 5 == 0:
                print('epoch: %d | iteration = %d | loss = %f | Adv_loss = %f | Recon_loss = %f'
                      %(epoch, i, loss, Adv_loss, recon_loss))

        if epoch % 2 == 0:
            # train: (8821,)
            # trainval: (7057,)
            # test_seen: (1764,)
            # test_unseen: (2967,)
            image_model.eval()
            att_model.eval()
            mod_model.eval()
            mod_transformer.eval()
            attn_img.eval()
            attn_att.eval()
            decoder_img.eval()
            decoder_att.eval()
            acc_zsl = compute_accuracy(image_model, att_model, mod_transformer, attn_img, attn_att, memory,
                                       test_unseen_loader, test_att, test_id, args)
            acc_seen_gzsl = compute_accuracy(image_model, att_model, mod_transformer, attn_img, attn_att, memory,
                                             test_seen_loader, all_att, all_id, args)
            acc_unseen_gzsl = compute_accuracy(image_model, att_model, mod_transformer, attn_img, attn_att, memory,
                                               test_unseen_loader, all_att, all_id, args)
            H = 2 * acc_seen_gzsl * acc_unseen_gzsl / (acc_seen_gzsl + acc_unseen_gzsl)
            if acc_zsl > pre_acc:
                pre_acc = acc_zsl
                pre_seen = acc_seen_gzsl
                pre_unseen = acc_unseen_gzsl
                pre_epoch = epoch
                pre_H = H

            print('epoch: %d | itr: %d | zsl: ACC=%.4f | gzsl: seen=%.4f, unseen=%.4f, h=%.4f'
                  % (epoch, i, acc_zsl, acc_seen_gzsl, acc_unseen_gzsl, H))
            print('max acc: epoch: %d | itr: %d | zsl: ACC=%.4f | gzsl: seen=%.4f, unseen=%.4f, h=%.4f'
                  % (pre_epoch, i, pre_acc, pre_seen, pre_unseen, pre_H))

def compute_accuracy(image_model, att_model, mod_transformer, attn_img, attn_att, memory, test_loader, test_att, test_id, args):
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    outpred = []
    test_label = []

    test_att = torch.from_numpy(test_att)
    test_att = test_att.float().to(device)
    test_att_output = att_model(test_att)
    test_size = test_att_output.size()[0]
    if args.modules_used == 'memory_fusion':
        # test_att_output = attn_att(mod_transformer(test_att_output).unsqueeze(1), memory.repeat(test_size, 1, 1))
        test_att_output = attn_att(test_att_output.unsqueeze(1), memory.repeat(test_size, 1, 1))
    elif args.modules_used == 'modal_classifier':
        test_att_output = mod_transformer(test_att_output)
    elif args.modules_used == 'unused' or args.modules_used == 'classifier_only':
        pass
    else:
        raise ValueError('Error Method')

    test_att_output = test_att_output / test_att_output.norm(dim=1,keepdim=True)

    for i, (image_input, att_input, cls_id, key, label) in enumerate(test_loader):
        att_input = att_input.float().to(device)
        label = list(label.numpy())

        image_input = image_input.float().to(device)
        image_input = image_input.squeeze(1)

        if args.modules_used == 'memory_fusion':
            batch_size = image_input.size()[0]
            # image_output = attn_img(mod_transformer(image_model(image_input)).unsqueeze(1), memory.repeat(batch_size, 1, 1))
            # att_output = attn_att(mod_transformer(att_model(att_input)).unsqueeze(1), memory.repeat(batch_size, 1, 1))
            image_output = attn_img(image_model(image_input).unsqueeze(1),
                                    memory.repeat(batch_size, 1, 1))
            att_output = attn_att(att_model(att_input).unsqueeze(1), memory.repeat(batch_size, 1, 1))
        elif args.modules_used == 'modal_classifier':
            image_output = mod_transformer(image_model(image_input))
            att_output = mod_transformer(att_model(att_input))
        elif args.modules_used == 'unused' or args.modules_used == 'classifier_only':
            image_output = image_model(image_input)
            att_output = att_model(att_input)
        else:
            raise ValueError('Error Method')

        image_output = image_output / image_output.norm(dim=1, keepdim=True)
        att_output = att_output / att_output.norm(dim=1,keepdim=True)

        for j in range(len(label)):
            outputLabel = kNNClassify(image_output.cpu().data.numpy()[j, :], test_att_output.cpu().data.numpy(), test_id, 1)
            outpred.append(outputLabel)
            test_label.append(label[j])

    outpred = np.array(outpred)
    test_label = np.array(test_label)
    unique_labels = np.unique(test_label)

    acc = 0
    for l in unique_labels:
        idx = np.nonzero(test_label == l)[0]
        acc += accuracy_score(test_label[idx], outpred[idx])

    acc = acc / unique_labels.shape[0]
    # acc = np.equal(outpred, test_label).mean()
    return acc


import json
import logging
import logging.handlers
import splunk.admin as admin
import splunk.entity as entity
import splunk.rest as rest
import splunk.util as util
import sys
import time

if sys.platform == "win32":
    import os, msvcrt
    msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)
    msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
    msvcrt.setmode(sys.stderr.fileno(), os.O_BINARY)

from splunk import ResourceNotFound
from splunk.clilib.bundle_paths import make_splunkhome_path
sys.path.append(make_splunkhome_path(['etc', 'apps', 'SA-Utils', 'lib']))
from SolnCommon.log     import setup_logger, SHORT_FORMAT
from SolnCommon.kvstore import KvStoreHandler
from SolnCommon.pooling import should_execute

logger = setup_logger('reviewstatuses_rest_handler', level=logging.INFO, format=SHORT_FORMAT)

def time_function_call(fx):
    """
    This decorator will provide a log message measuring how long a function call took.

    Arguments:
    fx -- The function to measure
    """

    def wrapper(*args, **kwargs):
        t = time.time()

        r = fx(*args, **kwargs)

        logger.debug('%s, duration=%.3f', fx.__name__, time.time() - t)

        return r
    return wrapper

"""
This class provides a mechanism for determining how long operations take. Results are submitted as debug
calls to the logger provided in the constructor or the instance in the global variable logger if no logger
is provided in the constructor.

Example:
with TimeLogger("doing_something", Logger()):
    time.sleep(2)
"""
class TimeLogger():

    def __init__(self, title, logger=None):
        self.title = title
        self.logger = logger

    def __enter__(self):

        # Define the start time
        self.start_time = int(time.time())

    def __exit__(self, type, value, traceback):

        # Determine how long the operation took
        time_spent = int(time.time()) - self.start_time

        # See if we can find a logger as a global variable
        if self.logger is None:
            try:
                self.logger = logger
            except NameError:
                raise Exception('Could not get a logger instance for the purposes of recording performance')

        # Log the time spent
        self.logger.debug('%s, duration=%d', self.title, time_spent)

class UnauthorizedUserException(Exception):
    pass

class InvalidConfigException(Exception):
    pass

class InvalidParameterValueException(InvalidConfigException):
    """
    Describes a config parameter that has an invalid value.
    """

    def __init__(self, field, value, value_must_be):
        message = "The value for the parameter '%s' is invalid: %s (was %s)" % (field, value_must_be, value)
        super(InvalidConfigException, self).__init__( message)

class UnsupportedParameterException(InvalidConfigException):
    """
    Describes a config parameter that is unsupported.
    """
    pass

class MissingTransitionException(InvalidConfigException):
    """
    Describes a capability that is missing.
    """
    def __init__(self, transitions):
        self.transitions = transitions
        super(InvalidConfigException, self).__init__( 'Missing transition detected' )

def _getFieldValue(args, name, default_value=None, max_length=None):
    """
    Get the field value from the argument list.
    """

    ## Get the value if defined or the default value if not defined
    value = args[name][0] or default_value if name in args else default_value

    ## Check the length
    if value and max_length and len(value) > max_length:
        raise admin.ArgValidationException(
            'App %s cannot be longer than %s character%s.' % (name, max_length, 's' if max_length > 1 else ''))
    ## return the value
    return value

def _addToDictIfNonNull(dict, name, value):
      """
      Add the given name and value to the dictionary if the value is not none.

      Arguments:
      dict -- the dictionary to add to
      name -- the name of the object to add
      value -- the value of the object to add (if not none)
      """
      if value is not None:
          dict[name] = value

class ReviewStatuses(admin.MConfigHandler):
    '''
    Set up supported arguments
    '''
    ## admin.py constants
    REQUESTED_ACTIONS       = { '1': 'ACTION_CREATE', '2': 'ACTION_LIST', '4': 'ACTION_EDIT', '8': 'ACTION_REMOVE', '16': 'ACTION_MEMBERS', '32': 'ACTION_RELOAD' }
     ## Permissions
    WRITE_CAPABILITY        = 'edit_reviewstatuses'
     ## Default Params
    PARAM_DISABLED          = 'disabled'
    PARAM_LABEL             = 'label'
    PARAM_DESCRIPTION       = 'description'
    PARAM_DEFAULT           = 'default'
    PARAM_SELECTED          = 'selected'
    PARAM_HIDDEN            = 'hidden'
    PARAM_END               = 'end'

    BOOLEAN_PARAMS          = [PARAM_DISABLED, PARAM_DEFAULT, PARAM_SELECTED, PARAM_HIDDEN, PARAM_END]

    VALID_PARAMS            = [PARAM_DISABLED,
                               PARAM_LABEL,
                               PARAM_DESCRIPTION,
                               PARAM_DEFAULT,
                               PARAM_SELECTED,
                               PARAM_HIDDEN,
                               PARAM_END]

    ## collections do not support '.' or '$' in params
    COLLECTION_PARAMS       = [PARAM_DISABLED,
                               PARAM_LABEL,
                               PARAM_DESCRIPTION,
                               PARAM_DEFAULT,
                               PARAM_SELECTED,
                               PARAM_HIDDEN,
                               PARAM_END]

    REQUIRED_PARAMS         = [PARAM_DISABLED, PARAM_LABEL, PARAM_DEFAULT, PARAM_HIDDEN]
     ## Default Vals
    DEFAULT_NAMESPACE       = 'SA-ThreatIntelligence'
    DEFAULT_OWNER           = 'nobody'
    DEFAULT_DISABLED        = 0
    DEFAULT_DEFAULT         = 0
    DEFAULT_SELECTED        = 0
    DEFAULT_HIDDEN          = 0
    DEFAULT_END             = 0

    PARAM_UNASSIGNED_STANZA = '0'
    PARAM_UNASSIGNED_DICT   = {PARAM_DISABLED:    DEFAULT_DISABLED,
                               PARAM_LABEL:       'Unassigned',
                               PARAM_DESCRIPTION: 'An error is preventing the issue from having a valid status assignment',
                               PARAM_DEFAULT:     DEFAULT_DEFAULT,
                               PARAM_SELECTED:    DEFAULT_SELECTED,
                               PARAM_HIDDEN:      DEFAULT_HIDDEN,
                               PARAM_END:         DEFAULT_END}

    def setup(self):
        logger.info('Setting up reviewstatuses_rest_handler')

        ## set write capability
        self.setWriteCapability(ReviewStatuses.WRITE_CAPABILITY)

        if self.requestedAction == admin.ACTION_EDIT or self.requestedAction == admin.ACTION_CREATE:
            ## Fill required params
            for arg in ReviewStatuses.REQUIRED_PARAMS:
                self.supportedArgs.addReqArg(arg)

            ## Fill valid params
            for arg in ReviewStatuses.VALID_PARAMS:
                if arg not in ReviewStatuses.REQUIRED_PARAMS:
                    self.supportedArgs.addOptArg(arg)

    @time_function_call
    def handleCreate(self, confInfo):
        """
        Handles creation of a review status
        """
        sessionKey = self.getSessionKey()
        ## Get requested action
        actionStr = str(self.requestedAction)
        if actionStr in ReviewStatuses.REQUESTED_ACTIONS:
            actionStr = ReviewStatuses.REQUESTED_ACTIONS[actionStr]

        logger.info('Entering %s', actionStr)

        ## Refresh
        self.handleReload(makeKVS=False)

        authorizeDict         = self.readConf('authorize')
        reviewstatusesDict    = self.readConfCtx('reviewstatuses')

        name = self.callerArgs.id
        args = self.callerArgs.data

        ## Check name
        if not name:
            raise admin.ArgValidationException('The name (stanza) of the status must not be empty')
        ## Make sure the item is not '0'
        ## '0' is reserved for the Unassigned status
        elif name==ReviewStatuses.PARAM_UNASSIGNED_STANZA:
            raise admin.ArgValidationException('The name (stanza) of the status must not be 0 (this is reserved for the Unassigned status)')
        ## Make sure the item does not already exist
        ## This should not be possible based on getUID, but no hurt in double checking
        elif name in reviewstatusesDict:
            raise admin.AlreadyExistsException('A reviewstatuses.conf entry already exists for status ID %s' % name)

        ## Get the field values
        disabled    = _getFieldValue(args, ReviewStatuses.PARAM_DISABLED)
        label       = _getFieldValue(args, ReviewStatuses.PARAM_LABEL)
        description = _getFieldValue(args, ReviewStatuses.PARAM_DESCRIPTION)
        default     = _getFieldValue(args, ReviewStatuses.PARAM_DEFAULT)
        selected    = _getFieldValue(args, ReviewStatuses.PARAM_SELECTED)
        hidden      = _getFieldValue(args, ReviewStatuses.PARAM_HIDDEN)
        end         = _getFieldValue(args, ReviewStatuses.PARAM_END)

        ## Add the field values to a configuration dictionary (that will be verified)
        conf           = entity.getEntity('configs/conf-reviewstatuses', '_new', sessionKey=sessionKey)
        conf.namespace = self.appName # always save things to SOME app context.
        conf.owner     = ((self.context == admin.CONTEXT_APP_AND_USER) and self.userName) or "-"
        conf['name']   = name

        _addToDictIfNonNull(conf, ReviewStatuses.PARAM_DISABLED, disabled)
        _addToDictIfNonNull(conf, ReviewStatuses.PARAM_LABEL, label)
        _addToDictIfNonNull(conf, ReviewStatuses.PARAM_DESCRIPTION, description)
        _addToDictIfNonNull(conf, ReviewStatuses.PARAM_DEFAULT, default)
        _addToDictIfNonNull(conf, ReviewStatuses.PARAM_SELECTED, selected)
        _addToDictIfNonNull(conf, ReviewStatuses.PARAM_HIDDEN, hidden)
        _addToDictIfNonNull(conf, ReviewStatuses.PARAM_END, end)

        ## Check the configuration
        ## on create we don't care about other default statuses because we are going to unset them
        ## we do however need to ensure that we are not creating a disabled default
        try:
            if util.normalizeBoolean(default):
                ReviewStatuses.checkConf(conf, name, defaultStatuses=[name], checkDefault=True)
            else:
                ReviewStatuses.checkConf(conf, name, checkDefault=False)

        except InvalidConfigException as e:
            e = "The configuration for the new review status is invalid and could not be created: %s" % e
            logger.error(e)
            raise admin.ArgValidationException(e)

        ## switch default
        if util.normalizeBoolean(default):
            self.switchDefault(reviewstatusesDict)

        ## Write out an update to the reviewstatuses config file
        entity.setEntity(conf, sessionKey=sessionKey)
        logger.info('Successfully added review status: %s', label)

        ## Create new transitions
        transitions = []
        transitions.extend(ReviewStatuses.makeTransitions('0', name, toOnly=True))

        for stanza in reviewstatusesDict:
            if stanza != 'default' and stanza != ReviewStatuses.PARAM_UNASSIGNED_STANZA:
                transitions.extend(ReviewStatuses.makeTransitions(name, stanza))

        transitions.sort()

        ## Write out an update to the authorize config file
        conf             = entity.getEntity('configs/conf-authorize', '_new', sessionKey=sessionKey)
        conf.namespace   = self.appName # always save things to SOME app context.
        conf.owner       = ((self.context == admin.CONTEXT_APP_AND_USER) and self.userName) or "-"
        conf[ReviewStatuses.PARAM_DISABLED] = '0'
        with TimeLogger('setting_transitions', logger):
            for transition in transitions:
                if transition in authorizeDict:
                    logger.warn('An authorize.conf entry already exists for %s', transition)

                else:
                    conf['name'] = transition
                    entity.setEntity(conf, sessionKey=sessionKey)
                    logger.info('Successfully added transition %s', transition)

        ## Reload reviewstatuses (makeKVS)
        self.handleReload()

        logger.info('%s completed successfully', actionStr)

    @time_function_call
    def switchDefault(self, reviewstatusesDict):
        """
        Handles the unsetting of the default flag if this is the new default

        raises Exceptions for any issues with unsetting perceived defaults
        """
        for stanza in reviewstatusesDict:
            reviewstatus = reviewstatusesDict[stanza]
            if stanza != 'default' and util.normalizeBoolean(reviewstatus[ReviewStatuses.PARAM_DEFAULT]):
                logger.info('Unsetting default param for reviewstatus %s', stanza)
                response, content = rest.simpleRequest(
                    '/servicesNS/nobody/%s/configs/conf-reviewstatuses/%s' % (reviewstatus[admin.EAI_ENTRY_ACL]['app'], stanza),
                    sessionKey=self.getSessionKey(), postargs={'default': 0}, raiseAllErrors=True
                )

    @time_function_call
    def handleList(self, confInfo):
        """
        Handles listing of a review statuses
        """
        ## Get requested action
        actionStr = str(self.requestedAction)
        if actionStr in ReviewStatuses.REQUESTED_ACTIONS:
            actionStr = ReviewStatuses.REQUESTED_ACTIONS[actionStr]

        logger.info('Entering %s', actionStr)

        ## Refresh
        self.handleReload(makeKVS=False)

        ## Get the configurations from reviewstatuses.conf
        reviewstatusesDict = self.readConfCtx('reviewstatuses')
        authorizeDict      = self.readConf('authorize')

        ## Get all correlations searches and provide the relevant options
        if reviewstatusesDict:
            ## Check default count
            defaultCount = len(ReviewStatuses.getSpecialStatuses(reviewstatusesDict))
            if defaultCount!=1:
                logger.error('The reviewstatuses.conf configurations are invalid because %s default statuses are set; should be one', defaultCount)

            ## Check end count
            endCount = len(ReviewStatuses.getSpecialStatuses(reviewstatusesDict, ReviewStatuses.PARAM_END))
            if endCount!=1:
                logger.error('The reviewstatuses.conf configurations are invalid because %s end statuses are set; should be one', endCount)

            ## Check each conf
            for stanza, settings in reviewstatusesDict.items():
                ## Make sure the item is not '0'
                ## '0' is reserved for the Unassigned status
                if stanza != 'default' and stanza != ReviewStatuses.PARAM_UNASSIGNED_STANZA:
                    try:
                        ## Check config
                        ReviewStatuses.checkConf(settings, stanza, confInfo)

                        ## Check transitions
                        ReviewStatuses.checkTransitions(stanza, reviewstatusesDict, authorizeDict)

                    except MissingTransitionException as e:
                        for exc in e.transitions:
                            logger.error("The configuration for status ID '%s' is invalid: %s", stanza, exc)

                    except InvalidConfigException as e:
                        logger.error("The configuration for status ID '%s' is invalid: %s", stanza, e)

            ## Add static "Unassigned"
            for key, val in ReviewStatuses.PARAM_UNASSIGNED_DICT.items():
                confInfo[ReviewStatuses.PARAM_UNASSIGNED_STANZA].append(key, val)

        logger.info('%s completed successfully', actionStr)

    @time_function_call
    def handleReload(self, confInfo=None, makeKVS=True):
        """
        Handles refresh/reload of the configuration options
        """
        ## Get requested action
        actionStr = str(self.requestedAction)
        if actionStr in ReviewStatuses.REQUESTED_ACTIONS:
            actionStr = ReviewStatuses.REQUESTED_ACTIONS[actionStr]

        logger.info('Entering %s', actionStr)
        logger.info('Refreshing reviewstatuses configurations via properties endpoint')
        try:
            refreshInfo = entity.refreshEntities('properties/reviewstatuses', sessionKey=self.getSessionKey())
        except Exception as e:
            logger.warn('Could not refresh reviewstatuses configurations via properties endpoint: %s', e)

        logger.info('Refreshing authorize configurations via properties endpoint')
        try:
            refreshInfo = entity.refreshEntities('properties/authorize', sessionKey=self.getSessionKey())
        except Exception as e:
            logger.warn('Could not refresh authorize configurations via properties endpoint: %s', e)

        if makeKVS:
            # Only execute on standalone or on cluster master.
            exec_status, exec_status_msg = should_execute(session_key=self.getSessionKey())
            logger.info(exec_status_msg)
            if exec_status:
                self.makeKVS()

        logger.info('%s completed successfully', actionStr)

    @time_function_call
    def makeKVS(self, collection='reviewstatuses', reviewstatusesDict=None):
        """
        Handles maintenance of reviewstatuses kvstore collection
        """
        logger.info('Creating %s collection', collection)
        ## options
        options = {'app': ReviewStatuses.DEFAULT_NAMESPACE, 'owner': ReviewStatuses.DEFAULT_OWNER, 'collection': collection}
        ## get reviewstatuses conf
        reviewstatusesDict = reviewstatusesDict or self.readConf('reviewstatuses')

        if not reviewstatusesDict:
            logger.error('Reviewstatuses dictionary is None; cannot makeKVS')
        else:
            ## get reviewstatuses collection
            gr, gc = KvStoreHandler.get(None, self.getSessionKey(), options)

            if gr.status==200:
                existing_keys  = [x['_key'] for x in json.loads(gc)]
                reviewstatuses = []

                ## Iterate reviewstatuses
                for stanza, settings in sorted(reviewstatusesDict.items()):
                    if stanza!='default':
                        reviewstatus = { '_key': stanza, 'status': stanza }
                        is_default   = False
                        for key, val in settings.items():
                            if not is_default and key == ReviewStatuses.PARAM_DEFAULT:
                                if util.normalizeBoolean(val):
                                    is_default = True
                            if key in ReviewStatuses.COLLECTION_PARAMS:
                                ## kvstore prefers None over empty string
                                reviewstatus[key] = val or None
                        reviewstatuses.append(reviewstatus)

                        if is_default:
                            default_reviewstatus = reviewstatus.copy()
                            default_reviewstatus['_key'] = '-1'
                            reviewstatuses.append(default_reviewstatus)

                pr, pc = KvStoreHandler.batch_create(reviewstatuses, self.getSessionKey(), options)
                if pr.status!=200:
                    logger.error('Error in upserting records to %s collection: %s', collection, pc)

                removed_keys = list(set(existing_keys).difference([x['_key'] for x in reviewstatuses]))
                if len(removed_keys)>0:
                    duri   = '/servicesNS/{owner}/{app}/storage/collections/data/{collection}'.format(**options)
                    dquery = {'$or': []}

                    for removed_key in removed_keys:
                        dquery['$or'].append({'_key': removed_key})

                    dr, dc = rest.simpleRequest(duri, sessionKey=self.getSessionKey(), method='DELETE', getargs={'query': json.dumps(dquery)})

                    if dr.status!=200:
                        logger.error('Error in deleting records from %s collection: %s', collection, dc)
            else:
                logger.error('Error retrieving records from %s collection: %s', collection, gc)

    @time_function_call
    def correctMissingTransition(self, transition, save=True):
        """
        Create the messing transition and return the created transition.
        """
        logger.info('Transition %s does not exist but should, it will be created', transition)
        # Get the _new entity in order to start editing
        conf = entity.getEntity('configs/conf-authorize', '_new', sessionKey=self.getSessionKey())
        # Set the namespace and owner
        conf.namespace   = self.appName
        conf.owner       = ReviewStatuses.DEFAULT_OWNER
        # Set the transition
        conf['name']     = transition
        conf[ReviewStatuses.PARAM_DISABLED] = '1'

        # Save the transition
        if save:
            entity.setEntity(conf, sessionKey=self.getSessionKey())
            logger.info('Successfully added transition %s', transition)
        else:
            logger.info('Preparing the creation of the missing transition %s; save is deferred until other edits are complete', transition)

        return conf

    @time_function_call
    def handleEdit(self, confInfo):
        """
        Handles edits to the configuration options
        """
        sessionKey = self.getSessionKey()
        ## Get requested action
        actionStr = str(self.requestedAction)
        if actionStr in ReviewStatuses.REQUESTED_ACTIONS:
            actionStr = ReviewStatuses.REQUESTED_ACTIONS[actionStr]

        logger.info('Entering %s', actionStr)

        ## Refresh
        self.handleReload(makeKVS=False)

        reviewstatusesDict = self.readConfCtx('reviewstatuses')

        name = self.callerArgs.id
        args = self.callerArgs

        # Stop if no name was provided
        if not name:
            raise admin.ArgValidationException('No name provided')
        elif name==ReviewStatuses.PARAM_UNASSIGNED_STANZA:
            raise admin.ArgValidationException('The name (stanza) of the status must not be 0 (this is reserved for the Unassigned status)')
        try:
            conf = entity.getEntity('configs/conf-reviewstatuses', name, sessionKey=sessionKey)
        except ResourceNotFound:
            raise admin.NotFoundException("A status with the given name '%s' could not be found" % name)

        # Create the resulting configuration that would be persisted if the settings provided are applied
        for key, val in conf.items():
            if key in args.data:
                # Get the new value
                new_value = args[key][0]
                # Set the value to a single space if empty or none, otherwise, Splunk won't save it (SOLNPCI-532)
                if new_value in [None, '']:
                    new_value = ' '
                # Assign the value
                conf[key] = new_value

        ## namespace/owner
        if admin.EAI_ENTRY_ACL in conf:
            conf.namespace = conf[admin.EAI_ENTRY_ACL].get('app', None) or ReviewStatuses.DEFAULT_NAMESPACE
            conf.owner     = conf[admin.EAI_ENTRY_ACL].get('owner', None) or ReviewStatuses.DEFAULT_OWNER
        else:
            conf.namespace = ReviewStatuses.DEFAULT_NAMESPACE
            conf.owner     = ReviewStatuses.DEFAULT_OWNER

        ## Check the resulting configuration
        ## here we need to pass the existing defaults
        defaultStatuses = ReviewStatuses.getSpecialStatuses(reviewstatusesDict)
        try:
            ReviewStatuses.checkConf(conf, name, defaultStatuses=defaultStatuses, checkDefault=True)
        except InvalidConfigException as e:
            e = "The edit attempt for review status '%s' produced an invalid configuration: %s" % (name, e)
            logger.error(e)
            raise admin.ArgValidationException(e)

        ## switch default
        if util.normalizeBoolean(conf['default']):
            self.switchDefault(reviewstatusesDict)

         ## Retain is_disabled
        status_disabled = util.normalizeBoolean(conf[ReviewStatuses.PARAM_DISABLED])

        ## Enable/Disable transitions
        transitions = []
        transitions.extend(ReviewStatuses.makeTransitions('0', name, toOnly=True))
        logger.info('transitions: %s', transitions)

        for stanza in reviewstatusesDict:
            if stanza != 'default' and stanza != ReviewStatuses.PARAM_UNASSIGNED_STANZA:
                transitions.extend(ReviewStatuses.makeTransitions(name, stanza))

        transitions.sort()
        logger.info('transitions: %s', transitions)

        # Get the existing set of transition capabilities; we are going to store these so that we don't have to call
        # getEntity on each capability since this takes a long time (about 20 seconds each)
        existing_transitions = entity.getEntities('configs/conf-authorize', sessionKey=sessionKey, count=-1)

        ## Write out an update to the authorize config file
        for transition in transitions:
            changed = False

            # Get the entry that we are editing
            if transition in existing_transitions:
                transition_conf = existing_transitions[transition]
            else:
                # Uh oh, the transition wasn't found. Go ahead and create it.
                transition_conf = self.correctMissingTransition(transition, save=False)

                # If the transition was not returned, then skip this entry.
                if transition_conf is None:
                    continue

                changed = True

            transition_conf.namespace = self.appName
            transition_conf.owner = ReviewStatuses.DEFAULT_OWNER

            ## Disable only transitions to the disabled status
            if status_disabled and transition.find('to_%s' % name) != -1:
                action = 'disabled'

                # Note that we are changing the value
                if str(transition_conf['disabled']) != '1':
                    changed = True
                    transition_conf['disabled'] = '1'

            else:
                action = 'enabled'

                # Note that we are changing the value
                if str(transition_conf['disabled']) != '0':
                    changed = True
                    transition_conf['disabled'] = '0'

            # Set the settings if they have changed; try to avoid performing the changes unless we need to since this is a slow operation
            if changed:
                with TimeLogger( ('Set %s transition in handleEdit: %s' % (action, transition)), logger):
                    entity.setEntity(transition_conf, sessionKey=sessionKey)
                logger.info('Successfully %s transition: %s', action, transition)

        ## Write out an update to the reviewstatuses config file
        ## Note that we are doing this after setting the transitions
        with TimeLogger('setting entity in handleEdit', logger):
            entity.setEntity(conf, sessionKey=sessionKey)

        logger.info('Successfully updated review status id %s', name)

        ## Reload reviewstatuses (makeKVS)
        self.handleReload()

        logger.info('%s completed successfully', actionStr)

    def handleRemove(self, confInfo):
        pass

    @staticmethod
    def getUID(reviewstatuses=[]):
        """
        Returns a unique identifier to be used as a stanza name
        """
        statusInts = []
        for reviewstatus in reviewstatuses:
            try:
                statusInt = int(reviewstatus)
                if statusInt > 0:
                    statusInts.append(statusInt)
            except:
                pass
        if len(statusInts)==0:
            uid = 1
        else:
            uid = sorted(statusInts)[-1] + 1

        return uid

    @staticmethod
    def getSpecialStatuses(confDict, typ=PARAM_DEFAULT):
        logger.info('Retrieving statuses with %s set', typ)
        statuses = [x for x in confDict if util.normalizeBoolean(confDict[x][typ])]
        logger.info('Successfully retrieved status(es) with %s set: %s', typ, statuses)
        return statuses

    @staticmethod
    def checkConf(settings, stanza=None, confInfo=None, defaultStatuses=[], checkDefault=False, throwExceptionOnError=False):
        """
        Checks the settings and raises an exception if the configuration is invalid.
        """
        ## Below is a list of the required fields. The entries in this list will be removed as they
        ## are observed. An empty list at the end of the config check indicates that all necessary
        ## fields where provided.
        required_fields = ReviewStatuses.REQUIRED_PARAMS[:]

        if stanza is not None and confInfo is not None:
            # Add each of the settings
            for key, val in settings.items():
                ## Set val to empty if None
                if val is None:
                    val = ''

                if key in ReviewStatuses.VALID_PARAMS:
                    confInfo[stanza].append(key, val)

                ## Key is eai;acl Set meta
                elif key.startswith(admin.EAI_ENTRY_ACL):
                    confInfo[stanza].setMetadata(key, val)

                ## Key is eai; userName/appName
                elif key.startswith(admin.EAI_META_PREFIX):
                    confInfo[stanza].append(key, val)

                ## Key is not proper
                else:
                    pass

        ## Check each of the settings individually
        logger.info("Checking general settings for the '%s' review status", stanza)
        for key, val in settings.items():
            ## Set val to empty if None
            if val is None:
                val = ''

            ## Check the disabled/selected value
            if key in ReviewStatuses.BOOLEAN_PARAMS:
                try:
                    util.normalizeBoolean(val, enableStrictMode=True)

                    ## Remove the field from the list of required fields
                    try:
                        required_fields.remove(key)

                    except ValueError:
                        pass # Field not available, probably because it is not required

                except ValueError:
                    raise InvalidParameterValueException(key, val, 'must be a valid boolean')

            elif key in ReviewStatuses.REQUIRED_PARAMS:
                ## Remove the field from the list of required fields
                try:
                    required_fields.remove(key)

                except ValueError:
                    pass # Field not available, probably because it is not required

            elif key in ReviewStatuses.VALID_PARAMS:
                pass

            ## Key is eai
            elif key.startswith(admin.EAI_META_PREFIX):
                pass

            ## Key is not proper
            else:
                if throwExceptionOnError:
                    raise UnsupportedParameterException()

                else:
                    logger.warn("The configuration for the '%s' review status contains an unsupported parameter: %s", stanza, key)
         ## Error if some of the required fields were not provided
        if len(required_fields) > 0:
            raise InvalidConfigException('The following fields must be defined in the configuration but were not: ' + ', '.join(required_fields).strip())

        ## Error if checkDefault and...
        if checkDefault:
            ## disabled
            disabled = util.normalizeBoolean(ReviewStatuses.DEFAULT_DISABLED)
            if ReviewStatuses.PARAM_DISABLED in settings:
                disabled = util.normalizeBoolean(settings[ReviewStatuses.PARAM_DISABLED])
            ## default
            default = util.normalizeBoolean(ReviewStatuses.DEFAULT_DEFAULT)
            if ReviewStatuses.PARAM_DEFAULT in settings:
                default = util.normalizeBoolean(settings[ReviewStatuses.PARAM_DEFAULT])
            if default:
                defaultStatuses.append(stanza)
            else:
                try:
                    defaultStatuses.remove(stanza)
                except:
                    pass

            ## end
            end = util.normalizeBoolean(ReviewStatuses.DEFAULT_END)
            if ReviewStatuses.PARAM_END in settings:
                end = util.normalizeBoolean(settings[ReviewStatuses.PARAM_END])

            ## 1. No disabled defaults
            if default and disabled:
                raise InvalidConfigException('Default review statuses cannot be disabled.  If you want to disable this status, then unset default.')

            ## 2. One default always
            defaultStatuses = set(defaultStatuses)
            if len(defaultStatuses)==0:
                raise InvalidConfigException('There must be one default review status at all times')
            elif len(defaultStatuses)>1:
                logger.warn('More than one default status detected: %s', defaultStatuses)

            ## 3. No end defaults
            if default and end:
                raise InvalidConfigException('Default review status cannot also be end status.  If you want this status to be default, then unset end.')

    @staticmethod
    def makeTransitions(statusA, statusB, toOnly=False):
        transitions = []

        if statusA != statusB:
            transitions.append('capability::transition_reviewstatus-%s_to_%s' % (statusA, statusB))
            if not toOnly:
                transitions.append('capability::transition_reviewstatus-%s_to_%s' % (statusB, statusA))

        return transitions

    @staticmethod
    def checkTransitions(status, reviewstatusesDict, authorizeDict):
        transitions = []
        missingTransitions = []
        transitions.extend(ReviewStatuses.makeTransitions('0', status, toOnly=True))

        for stanza in reviewstatusesDict:
            if stanza != 'default' and stanza != '0':
                transitions.extend(ReviewStatuses.makeTransitions(status, stanza))

        for transition in transitions:
            if transition not in authorizeDict:
                missingTransitions.append('Missing capability ' + transition)

        if len(missingTransitions) > 0:
            raise MissingTransitionException(missingTransitions)

# initialize the handler
admin.init(ReviewStatuses, admin.CONTEXT_APP_AND_USER)


import sys, json, csv
import concurrent.futures as futures
from datetime import datetime
import utility.airtable_manager as airtable
import sheets.LO_OverChallengesWeeks as s1
import sheets.LO_OverDaysNotPersistent as s2
import sheets.LO_OverDaysPersistent as s3
import sheets.StudentEvaluations_OverDaysNotPersistent as s4
import sheets.UserTypes as s5
import sheets.LO_OverDaysNotPersistentForSubSamples as s6
import sheets.ExportLog as s7
import shutil

if __name__ == "__main__":

    sys.stdout.write("Getting data...\n")
    # Getting Data from Airtable, Function returns
    # Students_data = Every LO Assemssment one by one from each unique student.
    # Filtered Students = Unique students whole data

    students_data, uniqueStudents = airtable.getStudentsContent()
    users, checkPointTickersLearners, autonomousUsersLearners, enthusiastUsersLearners, excludedUsers, outdatedExcludedUsers, nonUsersExcludedUsers,averageNumberOfEvaluation,averageSessionByStudents = airtable.createAllSubsamples(uniqueStudents)

    #TODO: generate data for subsets. students_data is basically useless.

    OUTPUT_PATH = 'output/'

   #-----------------------OVER DAYS NOT PERSISTENT---------------
    SHEET1_OUTPUT_FILE = 'LO_OverDaysNotPersistent/LO_OverDaysNotPersistent_AllUsers_' + datetime.now().strftime("%Y|%m|%d_") + f"{len(uniqueStudents)}" + '.tsv'
    SHEET2_OUTPUT_FILE = 'LO_OverDaysNotPersistent/LO_OverDaysNotPersistent_Users_' + datetime.now().strftime("%Y|%m|%d_") + f"{len(users)}" + '.tsv'
    SHEET3_OUTPUT_FILE = 'LO_OverDaysNotPersistent/LO_OverDaysNotPersistent_Users_CheckPointTicker_' + datetime.now().strftime("%Y|%m|%d_") + f"{len(checkPointTickersLearners)}" + '.tsv'
    SHEET4_OUTPUT_FILE = 'LO_OverDaysNotPersistent/LO_OverDaysNotPersistent_Users_Enthusiast_' + datetime.now().strftime("%Y|%m|%d_") + f"{len(enthusiastUsersLearners)}" + '.tsv'
    SHEET5_OUTPUT_FILE = 'LO_OverDaysNotPersistent/LO_OverDaysNotPersistent_Users_Autonomous_' + datetime.now().strftime("%Y|%m|%d_") + f"{len(autonomousUsersLearners)}" + '.tsv'
    SHEET6_OUTPUT_FILE = 'LO_OverDaysNotPersistent/LO_OverDaysNotPersistent_ExcludedUsers_' + datetime.now().strftime("%Y|%m|%d_") + f"{len(excludedUsers)}" + '.tsv'
    SHEET7_OUTPUT_FILE = 'LO_OverDaysNotPersistent/LO_OverDaysNotPersistent_ExcludedUsers_NonUsers_' + datetime.now().strftime("%Y|%m|%d_") + f"{len(nonUsersExcludedUsers)}" + '.tsv'
    SHEET8_OUTPUT_FILE = 'LO_OverDaysNotPersistent/LO_OverDaysNotPersistent_ExcludedUsers_OutDated_' + datetime.now().strftime("%Y|%m|%d_") + f"{len(outdatedExcludedUsers)}" + '.tsv'

    #----------------------OVER DAYS PERSISTENT------------------------
    SHEET9_OUTPUT_FILE = 'LO_OverDaysPersistent/LO_OverDaysPersistent_AllUsers_' + datetime.now().strftime("%Y|%m|%d_") + f"{len(uniqueStudents)}" + '.tsv'
    SHEET10_OUTPUT_FILE = 'LO_OverDaysPersistent/LO_OverDaysPersistent_Users_' + datetime.now().strftime("%Y|%m|%d_") + f"{len(users)}" + '.tsv'
    SHEET11_OUTPUT_FILE = 'LO_OverDaysPersistent/LO_OverDaysPersistent_Users_CheckPointTicker_' + datetime.now().strftime("%Y|%m|%d_") + f"{len(checkPointTickersLearners)}" + '.tsv'
    SHEET12_OUTPUT_FILE = 'LO_OverDaysPersistent/LO_OverDaysPersistent_Users_Enthusiast_' + datetime.now().strftime("%Y|%m|%d_") + f"{len(enthusiastUsersLearners)}" + '.tsv'
    SHEET13_OUTPUT_FILE = 'LO_OverDaysPersistent/LO_OverDaysPersistent_Users_Autonomous_' + datetime.now().strftime("%Y|%m|%d_") + f"{len(autonomousUsersLearners)}" + '.tsv'
    SHEET14_OUTPUT_FILE = 'LO_OverDaysPersistent/LO_OverDaysPersistent_ExcludedUsers_' + datetime.now().strftime("%Y|%m|%d_") + f"{len(excludedUsers)}" + '.tsv'
    SHEET15_OUTPUT_FILE = 'LO_OverDaysPersistent/LO_OverDaysPersistent_ExcludedUsers_NonUsers_' + datetime.now().strftime("%Y|%m|%d_") + f"{len(nonUsersExcludedUsers)}" + '.tsv'
    SHEET16_OUTPUT_FILE = 'LO_OverDaysPersistent/LO_OverDaysPersistent_ExcludedUsers_Outdated_' + datetime.now().strftime("%Y|%m|%d_") + f"{len(outdatedExcludedUsers)}" + '.tsv'

    #----------------------STUDENT EVALUATION NOT PERSISTENT--------------------
    SHEET17_OUTPUT_FILE = 'StudentEvaluations_OverDaysNotPersistent_' + datetime.now().strftime("%Y|%m|%d_") + f"{len(uniqueStudents)}" + '.tsv'
    SHEET18_OUTPUT_FILE = 'UserTypes_' + datetime.now().strftime("%Y|%m|%d_") + f"{len(uniqueStudents)}" + '.tsv'

    SHEET19_OUTPUT_FILE = 'ExportLog_'+ datetime.now().strftime("%Y|%m|%d_") + '.txt'

    overDaysNotPersistentDictionary = {
        SHEET2_OUTPUT_FILE:users,
        SHEET3_OUTPUT_FILE:checkPointTickersLearners,
        SHEET4_OUTPUT_FILE:enthusiastUsersLearners,
        SHEET5_OUTPUT_FILE:autonomousUsersLearners,
        SHEET6_OUTPUT_FILE:excludedUsers,
        SHEET7_OUTPUT_FILE:nonUsersExcludedUsers,
        SHEET8_OUTPUT_FILE:outdatedExcludedUsers
         }
    overDaysPersistentDictionary ={
        SHEET9_OUTPUT_FILE:uniqueStudents,
        SHEET10_OUTPUT_FILE:users,
        SHEET11_OUTPUT_FILE:checkPointTickersLearners,
        SHEET12_OUTPUT_FILE:enthusiastUsersLearners,
        SHEET13_OUTPUT_FILE:autonomousUsersLearners,
        SHEET14_OUTPUT_FILE:excludedUsers,
        SHEET15_OUTPUT_FILE:nonUsersExcludedUsers,
        SHEET16_OUTPUT_FILE:outdatedExcludedUsers
    }

    sys.stdout.write("\nGenerating LO_OverDaysNotPersistent sheet for all Learners ...")
    second_sheet = s2.LO_OverDaysNotPersistent(OUTPUT_PATH + SHEET1_OUTPUT_FILE, students_data.copy())

with futures.ThreadPoolExecutor(max_workers=22) as executor:
    for key in overDaysPersistentDictionary:
        sys.stdout.write(f"\nGenerating LO_OverDaysPersistent sheet  {key} ...")
        executor.submit(s3.LO_OverDaysPersistent,(OUTPUT_PATH + key),overDaysPersistentDictionary[key].copy())
    for key in overDaysNotPersistentDictionary:
        sys.stdout.write(f"\nGenerating LOs_OverDaysNotPersistent sheet {key} ...")
        executor.submit(s6.LO_OverDaysNotPersistentForSubSamples,(OUTPUT_PATH + key),overDaysNotPersistentDictionary[key].copy())

    sys.stdout.write("\nGenerating StudentEvaluations_OverDaysNotPersistent sheet for all Learners ...")
    executor.submit(s4.StudentEvaluations_OverDaysNotPersistent,(OUTPUT_PATH + SHEET17_OUTPUT_FILE),uniqueStudents.copy())

    sys.stdout.write("\nGenerating UserTypes sheet..")
    executor.submit(s5.UserTypes,(OUTPUT_PATH + SHEET18_OUTPUT_FILE),uniqueStudents.copy())

    sys.stdout.write("\nGeneratin Scope and Sequence ..")
    executor.submit(shutil.copy(r"input/Challenges.csv",OUTPUT_PATH+"ScopeAndSequence.csv"))

    sys.stdout.write("\nGeneratin Documentation ..")
    executor.submit(shutil.copy(r"input/Documentation.pdf",OUTPUT_PATH+"Documentation.pdf"))

sys.stdout.write("\nGenerating ExportLog sheet..")
s7.ExportLog(("output/" + SHEET19_OUTPUT_FILE),("output/"+ SHEET18_OUTPUT_FILE),len(uniqueStudents),averageNumberOfEvaluation,averageSessionByStudents)


import numpy as np

multi_inv = {
    1: 1,
    3: 9,
    5: 21,
    7: 15,
    9: 3,
    11: 19,
    15: 7,
    17: 23,
    19: 11,
    21: 5,
    23: 17,
    25: 25
}

def encrypt(secret, key):
    msg = np.array([ord(x) - 65 for x in secret])

    encrypted = ""

    if len(msg) % len(key) != 0:
        msg = np.append(msg, [23] * (len(key) - (len(msg) % len(key))))

    msg = msg.reshape((-1, len(key)))
    key = np.array(key)

    for triplet in msg:
        multed = np.matmul(triplet, key) % 26
        encrypted += "".join([chr(x + 65) for x in multed])

    return encrypted

def decrypt(secret, key):
    msg = np.array([ord(x) - 65 for x in secret])

    decrypted = ""

    msg = msg.reshape((-1, len(key)))
    det = int(np.linalg.det(key))

    inv = np.round_(det * np.linalg.inv(key) * multi_inv[det % 26]).astype(int) % 26

    for triplet in msg:
        multed = np.matmul(triplet, inv) % 26
        decrypted += "".join([chr(x + 65) for x in multed])

    return decrypted

secret = str(input("Enter the secret to encrypt: ")).replace(" ", "").upper()

dims = int(input("Enter the dimensions of the key: "))

key = np.array(list(map(int, input("Enter the key: ").strip().split(" ")))).reshape((dims, dims))

encrypted = encrypt(secret, key)

victim = str(input("Enter the secret to decrypt: ")).replace(" ", "").upper()

decrypted = decrypt(victim, key)

print("Encrypted: ", encrypted)
print("Decrypted: ", decrypted)


import numpy as np

np.seterr(invalid='ignore')

from configs.config import BaseEnvConfig
from envs.model.agent import *
from envs.model.mdp import JointState
from shapely.geometry import *

tmp_config = BaseEnvConfig()

def tensor_to_joint_state(state):  # 恢复原先尺度
    robot_states, human_states = state

    robot_states = robot_states.cpu().squeeze(0).data.numpy()
    robot_states = [RobotState(robot_state[0] * tmp_config.env.nlon,
                               robot_state[1] * tmp_config.env.nlat,
                               robot_state[2] * tmp_config.env.rotation_limit,
                               robot_state[3] * tmp_config.env.max_uav_energy) for robot_state in robot_states]

    human_states = human_states.cpu().squeeze(0).data.numpy()
    human_states = [HumanState(human_state[0] * tmp_config.env.nlon,
                               human_state[1] * tmp_config.env.nlat,
                               human_state[2] * tmp_config.env.rotation_limit,
                               human_state[3] * tmp_config.env.num_timestep) for human_state in human_states]

    return JointState(robot_states, human_states)

def tensor_to_robot_states(robot_state_tensor):
    robot_states = robot_state_tensor.cpu().squeeze(0).data.numpy()
    robot_states = [RobotState(robot_state[0] * tmp_config.env.nlon,
                               robot_state[1] * tmp_config.env.nlat,
                               robot_state[2] * tmp_config.env.rotation_limit,
                               robot_state[3] * tmp_config.env.max_uav_energy) for robot_state in robot_states]
    return robot_states

def get_human_position_list(selected_timestep, human_df):
    selected_timestamp = tmp_config.env.start_timestamp + selected_timestep * tmp_config.env.step_time
    selected_data = human_df[human_df.timestamp == selected_timestamp]
    selected_data = selected_data.set_index("id")

    if selected_timestep < tmp_config.env.num_timestep:
        selected_next_data = human_df[human_df.timestamp == (selected_timestamp + tmp_config.env.step_time)]
        selected_next_data = selected_next_data.set_index("id")
    else:
        selected_next_data = None

    return selected_data, selected_next_data

def get_human_position_from_list(selected_timestep, human_id, selected_data, selected_next_data):
    px, py = selected_data.loc[human_id, ["x", "y"]]

    if selected_timestep < tmp_config.env.num_timestep:
        npx, npy = selected_next_data.loc[human_id, ["x", "y"]]
        theta = get_theta(0, 0, npx - px, npy - py)
        # print(px, py, npx, npy, theta)
    else:
        theta = 0

    return px, py, theta

def judge_aoi_update(human_position, robot_position):
    should_reset = False
    for robot_id in range(tmp_config.env.robot_num):
        unit_distance = np.sqrt(np.power(robot_position[robot_id][0] - human_position[0], 2)
                                + np.power(robot_position[robot_id][1] - human_position[1], 2))
        if unit_distance <= tmp_config.env.sensing_range:
            should_reset = True
            break

    return should_reset

def inPoly(polygon, x, y):
    pt = (x, y)
    line = LineString(polygon)
    point = Point(pt)
    polygon = Polygon(line)
    return polygon.contains(point)

def iscrosses(line1, line2):
    if LineString(line1).crosses(LineString(line2)):
        return True
    return False

def crossPoly(square, x1, y1, x2, y2):
    our_line = LineString([[x1, y1], [x2, y2]])
    line1 = LineString([square[0], square[2]])
    line2 = LineString([square[1], square[3]])
    if our_line.crosses(line1) or our_line.crosses(line2):
        return True
    else:
        return False

def judge_collision(new_robot_px, new_robot_py, old_robot_px, old_robot_py):
    if tmp_config.env.no_fly_zone is None:
        return False

    for square in tmp_config.env.no_fly_zone:
        if inPoly(square, new_robot_px, new_robot_py):
            return True
        if crossPoly(square, new_robot_px, new_robot_py, old_robot_px, old_robot_py):
            return True
    return False

def get_theta(x1, y1, x2, y2):
    ang1 = np.arctan2(y1, x1)
    ang2 = np.arctan2(y2, x2)
    theta = np.rad2deg((ang1 - ang2) % (2 * np.pi))
    return theta

def consume_uav_energy(fly_time, hover_time):
    tmp_config = BaseEnvConfig()

    # configs
    Pu = 0.5  # the average transmitted power of each user, W,  e.g. mobile phone
    P0 = 79.8563  # blade profile power, W
    P1 = 88.6279  # derived power, W
    U_tips = 120  # tip speed of the rotor blade of the UAV,m/s
    v0 = 4.03  # the mean rotor induced velocity in the hovering state,m/s
    d0 = 0.6  # fuselage drag ratio
    rho = 1.225  # density of air,kg/m^3
    s0 = 0.05  # the rotor solidity
    A = 0.503  # the area of the rotor disk, m^2
    Vt = tmp_config.env.velocity  # velocity of the UAV,m/s

    Power_flying = P0 * (1 + 3 * Vt ** 2 / U_tips ** 2) + \
                   P1 * np.sqrt((np.sqrt(1 + Vt ** 4 / (4 * v0 ** 4)) - Vt ** 2 / (2 * v0 ** 2))) + \
                   0.5 * d0 * rho * s0 * A * Vt ** 3

    Power_hovering = P0 + P1

    return fly_time * Power_flying + hover_time * Power_hovering

def get_border(ur, lf):
    upper_left = [lf[0], ur[1]]
    upper_right = [ur[0], ur[1]]
    lower_right = [ur[0], lf[1]]
    lower_left = [lf[0], lf[1]]

    coordinates = [
        upper_left,
        upper_right,
        lower_right,
        lower_left,
        upper_left
    ]

    geo_json = {"type": "FeatureCollection",
                "properties": {
                    "lower_left": lower_left,
                    "upper_right": upper_right
                },
                "features": []}

    grid_feature = {
        "type": "Feature",
        "geometry": {
            "type": "Polygon",
            "coordinates": [coordinates],
        }
    }

    geo_json["features"].append(grid_feature)

    return geo_json

def traj_to_timestamped_geojson(index, trajectory, robot_num, color):
    point_gdf = trajectory.df.copy()
    point_gdf["previous_geometry"] = point_gdf["geometry"].shift()
    point_gdf["time"] = point_gdf.index
    point_gdf["previous_time"] = point_gdf["time"].shift()

    features = []

    # for Point in GeoJSON type
    for _, row in point_gdf.iterrows():
        corrent_point_coordinates = [
            row["geometry"].xy[0][0],
            row["geometry"].xy[1][0]
        ]
        current_time = [row["time"].isoformat()]

        if index < robot_num:
            radius = 8  # 125(5 units)
            opacity = 0.05
            popup_html = f'
UAV {int(row["id"])}
' + f'
raw coord: {corrent_point_coordinates}
' \
                         + f'
grid coord: ({row["x"]},{row["y"]})
' \
                         + f'
dist coord: ({row["x_distance"]}m, {row["y_distance"]}m)
' \
                         + f'
energy: {row["energy"]}J
'
        else:
            radius = 2
            opacity = 1
            popup_html = f'
Human {int(row["id"])}
' + f'
raw coord: {corrent_point_coordinates}
' \
                         + f'
grid coord: ({row["x"]},{row["y"]})
' \
                         + f'
dist coord: ({row["x_distance"]}m, {row["y_distance"]}m)
' \
                         + f'
aoi: {int(row["aoi"])}
'

        # for Point in GeoJSON type  (Temporally Deprecated)
        features.append(
            {
                "type": "Feature",
                "geometry": {
                    "type": "Point",
                    "coordinates": corrent_point_coordinates,
                },
                "properties": {
                    "times": current_time,
                    'popup': popup_html,
                    "icon": 'circle',  # point
                    "iconstyle": {
                        'fillColor': color,
                        'fillOpacity': opacity,  # 透明度
                        'stroke': 'true',
                        'radius': radius,
                        'weight': 1,
                    },

                    "style": {  # line
                        "color": color,
                    },
                    "code": 11,

                },
            }
        )
    return features

if __name__ == "__main__":
    print(judge_collision(new_robot_px=6505, new_robot_py=5130,
                          old_robot_px=6925, old_robot_py=5130))