"""This module contains Langmuir area calculations."""

import logging

logger = logging.getLogger('pygaps')
import textwrap
import warnings

import numpy

from .. import scipy
from ..core.adsorbate import Adsorbate
from ..graphing.calc_graphs import langmuir_plot
from ..utilities.exceptions import CalculationError
from ..utilities.exceptions import ParameterError
from ..utilities.exceptions import pgError

def area_langmuir(isotherm, branch='ads', limits=None, verbose=False):
    r"""
    Calculate the Langmuir-determined surface area of an isotherm.

    Parameters
    ----------
    isotherm : PointIsotherm
        The isotherm of which to calculate the Langmuir surface area.
    branch : {'ads', 'des'}, optional
        Branch of the isotherm to use. It defaults to adsorption.
    limits : [float, float], optional
        Manual limits for region selection.
    verbose : bool, optional
        Prints extra information and plots graphs of the calculation.

    Returns
    -------
    dict
        A dictionary of results with the following components. The basis of these
        results will be derived from the basis of the isotherm (per mass or per
        volume of material):

        - ``area`` (float) : calculated Langmuir surface area, in m2/unit of material
        - ``langmuir_const`` (float) : the constant in the Langmuir fit
        - ``n_monolayer`` (float) : the amount adsorbed at the monolayer
        - ``langmuir_slope`` (float) : slope of the Langmuir plot
        - ``langmuir_intercept`` (float) : intercept of the Langmuir plot
        - ``corr_coef`` (float) : correlation coefficient of the linear region in the Langmuir plot

    Notes
    -----
    *Description*

    The Langmuir theory [#]_, proposed at the start of the 20th century, states that
    adsorption happens on active sites on a surface in a single layer. It is
    derived based on several assumptions.

        * All sites are equivalent and have the same chance of being occupied
        * Each adsorbate molecule can occupy one adsorption site
        * There are no interactions between adsorbed molecules
        * The rates of adsorption and desorption are proportional to the number
          of sites currently free and currently occupied, respectively
        * Adsorption is complete when all sites are filled.

    The Langmuir equation is then:

    .. math::

        n = n_m\frac{KP}{1+KP}

    The equation can be rearranged as:

    .. math::

        \frac{P}{n} = \frac{1}{K n_m} + \frac{P}{n_m}

    Assuming the data can be fitted with a Langmuir model, by plotting
    :math:`\frac{P}{n}` against pressure, a line will be obtained. The slope and
    intercept of this line can then be used to calculate :math:`n_{m}`,
    the amount adsorbed at the monolayer, as well as K, the Langmuir constant.

    .. math::

        n_m = \frac{1}{s}

        K = \frac{1}{i * n_m}

    The surface area can then be calculated by using the moles adsorbed at the
    monolayer. If the specific area taken by one of the adsorbate molecules on the surface
    is known, it is inserted in the following equation together with Avogadro's number:

    .. math::

        a(Langmuir) = n_m A_N \sigma

    *Limitations*

    The Langmuir method for determining surface area assumes that only one single
    layer is adsorbed on the surface of the material. As most adsorption processes
    (except chemisorption) don't follow this behaviour, it is important to regard
    the Langmuir surface area as an estimate.

    References
    ----------
    .. [#] I. Langmuir, J. American Chemical Society 38, 2219(1916); 40, 1368(1918)

    """
    # get adsorbate properties
    adsorbate = Adsorbate.find(isotherm.adsorbate)
    cross_section = adsorbate.get_prop("cross_sectional_area")

    # Read data in
    loading = isotherm.loading(
        branch=branch, loading_unit='mol', loading_basis='molar'
    )
    try:
        pressure = isotherm.pressure(
            branch=branch,
            pressure_mode='relative',
        )
    except pgError:
        raise CalculationError(
            "The isotherm cannot be converted to a relative basis. "
            "Is your isotherm supercritical?"
        )

    # use the langmuir function
    (
        langmuir_area,
        langmuir_const,
        n_monolayer,
        slope,
        intercept,
        minimum,
        maximum,
        corr_coef,
    ) = area_langmuir_raw(pressure, loading, cross_section, limits=limits)

    if verbose:

        logger.info(
            textwrap.dedent(
                f"""\
            Langmuir surface area: a = {langmuir_area:.2e} m2/{isotherm.material_unit}
            Minimum pressure point is {pressure[minimum]:.3f} and maximum is {pressure[maximum -1]:.3f}
            The slope of the Langmuir line: s = {slope:.2e}
            The intercept of the Langmuir line: i = {intercept:.2e}
            The Langmuir constant is: K = {langmuir_const:.1f}
            Amount for a monolayer: n = {n_monolayer:.2e} mol/{isotherm.material_unit}"""
            )
        )

        # Generate plot of the langmuir points chosen
        langmuir_plot(
            pressure, langmuir_transform(pressure, loading), minimum, maximum,
            slope, intercept
        )

    return {
        'area': langmuir_area,
        'langmuir_const': langmuir_const,
        'n_monolayer': n_monolayer,
        'langmuir_slope': slope,
        'langmuir_intercept': intercept,
        'corr_coef': corr_coef,
        'limits': [minimum, maximum]
    }

def area_langmuir_raw(pressure, loading, cross_section, limits=None):
    """
    Calculate Langmuir-determined surface area.

    This is a 'bare-bones' function to calculate Langmuir surface area which is
    designed as a low-level alternative to the main function.
    Designed for advanced use, its parameters have to be manually specified.

    Parameters
    ----------
    pressure : array
        Pressures, relative.
    loading : array
        Loadings, in mol/basis.
    cross_section : float
        Adsorbed cross-section of the molecule of the adsorbate, in nm.
    limits : [float, float], optional
        Manual limits for region selection.

    Returns
    -------
    langmuir_area : float
        Calculated Langmuir surface area.
    langmuir_const : float
        K constant from the Langmuir equation.
    n_monolayer : float
        Adsorbed quantity in the monolayer.
    slope : float
        Calculated slope of the Langmuir plot.
    intercept : float
        Calculated intercept of the Langmuir plot.
    minimum : float
        Minimum point taken for the linear region.
    maximum : float
        Maximum point taken for the linear region.
    corr_coef : float
        Correlation coefficient of the straight line in the Langmuir plot.

    """
    if len(pressure) != len(loading):
        raise ParameterError(
            "The length of the pressure and loading arrays do not match."
        )
    # Ensure numpy arrays, if not already
    loading = numpy.asarray(loading)
    pressure = numpy.asarray(pressure)

    # select the maximum and minimum of the points and the pressure associated
    if limits is None:
        limits = [0.05, 0.9]

    maximum = len(pressure) - 1
    minimum = 0

    if limits[1]:
        maximum = numpy.searchsorted(pressure, limits[1])

    if limits[0]:
        minimum = numpy.searchsorted(pressure, limits[0])

    if maximum - minimum < 3:  # (for 2 point minimum)
        raise CalculationError(
            "The isotherm does not have enough points (at least 2) "
            "in selected region. Unable to calculate Langmuir area."
        )

    # calculate the Langmuir slope and intercept
    langmuir_array = langmuir_transform(
        pressure[minimum:maximum], loading[minimum:maximum]
    )
    slope, intercept, corr_coef = langmuir_optimisation(
        pressure[minimum:maximum], langmuir_array
    )

    # calculate the Langmuir parameters
    n_monolayer, langmuir_const, langmuir_area = langmuir_parameters(
        slope, intercept, cross_section
    )

    # Checks for consistency
    if langmuir_const < 0:
        warnings.warn("The Langmuir constant is negative.")
    if corr_coef < 0.99:
        warnings.warn("The correlation is not linear.")

    return (
        langmuir_area, langmuir_const, n_monolayer, slope, intercept, minimum,
        maximum, corr_coef
    )

def langmuir_transform(pressure, loading):
    """Langmuir transform function."""
    return pressure / loading

def langmuir_optimisation(pressure, langmuir_points):
    """Finds the slope and intercept of the Langmuir region."""
    slope, intercept, corr_coef, p, stderr = scipy.stats.linregress(
        pressure, langmuir_points
    )
    return slope, intercept, corr_coef

def langmuir_parameters(slope, intercept, cross_section):
    """Calculates the Langmuir parameters from the slope and intercept."""
    n_monolayer = 1 / slope
    langmuir_const = 1 / (intercept * n_monolayer)
    langmuir_area = n_monolayer * cross_section * \
        (10**(-18)) * scipy.const.Avogadro
    return n_monolayer, langmuir_const, langmuir_area


import os
import logging
from datetime import datetime, timezone
from ibm_botocore.credentials import DefaultTokenManager

from lithops.utils import is_lithops_function
from lithops.config import CACHE_DIR, load_yaml_config, dump_yaml_config

logger = logging.getLogger(__name__)

class IBMIAMAPIKeyManager:
    def __init__(self, component_name, iam_api_key, token=None, token_expiry_time=None):
        self.component_name = component_name
        self.iam_api_key = iam_api_key

        self._token_manager = DefaultTokenManager(api_key_id=self.iam_api_key)
        self._token_filename = os.path.join(CACHE_DIR, self.component_name, 'iam_token')

        if token:
            logger.debug("Using IBM IAM API Key - Reusing Token from config")
            self._token_manager._token = token
            self._token_manager._expiry_time = datetime.strptime(token_expiry_time, '%Y-%m-%d %H:%M:%S.%f%z')
            logger.debug("Token expiry time: {} - Minutes left: {}".format(self._token_manager._expiry_time, self._get_token_minutes_diff()))
        elif os.path.exists(self._token_filename):
            logger.debug("Using IBM IAM API Key - Reusing Token from local cache")
            token_data = load_yaml_config(self._token_filename)
            self._token_manager._token = token_data['token']
            self._token_manager._expiry_time = datetime.strptime(token_data['token_expiry_time'], '%Y-%m-%d %H:%M:%S.%f%z')
            logger.debug("Token expiry time: {} - Minutes left: {}".format(self._token_manager._expiry_time, self._get_token_minutes_diff()))

    def _get_token_minutes_diff(self):
        return int((self._token_manager._expiry_time - datetime.now(timezone.utc)).total_seconds() / 60.0)

    def _generate_new_token(self):
        self._token_manager._token = None
        self._token_manager.get_token()
        token_data = {}
        token_data['token'] = self._token_manager._token
        token_data['token_expiry_time'] = self._token_manager._expiry_time.strftime('%Y-%m-%d %H:%M:%S.%f%z')
        dump_yaml_config(self._token_filename, token_data)

    def get_token(self):
        if (self._token_manager._is_expired() or self._get_token_minutes_diff() < 11) and not is_lithops_function():
            logger.debug("Using IBM IAM API Key - Token expired. Requesting new token")
            self._generate_new_token()

        token = self._token_manager._token
        token_expiry_time = self._token_manager._expiry_time.strftime('%Y-%m-%d %H:%M:%S.%f%z')

        return token, token_expiry_time


import random
from functools import wraps

def RandomChance(chance: float):
	def RandomChanceDecorator(fn):
		chance_min = random.random()
		chance_max = chance_min + chance

		if chance_max > 1:
			chance_min -= chance_max - 1

		@wraps(fn)
		async def new_fn(*args, **kwargs):
			if chance_min <= random.random() <= chance_max:
				#print('im doin it')
				return await fn(*args, **kwargs)
			else:
				return False

		return new_fn

	return RandomChanceDecorator


#!/usr/bin/env python
# coding: utf-8

# # Python Poll

# **Polling Analysis**

# In[31]:

# File path across OS
import os

# Module for reading CSV files
import csv

#parameters
candidates_list=[]
total_dict={}
winner=0
total_votes=0
winnername=''

#text file

file = './Analysis/Analysis.txt'

#specifying the path
csvpath = os.path.join('.', 'Resources', 'election_data.csv')

with open(csvpath) as csvfile:
    # CSV reader specifies delimiter and variable that holds contents
    csvreader = csv.reader(csvfile, delimiter=',')

    # checking the read by printing the header
    csv_header = next(csvreader)
    #print(f"CSV Header: {csv_header}")

    # iterating over the rows to find all the unique candidates
    for row in csvreader:
        total_votes=total_votes+1
        if row[2] not in candidates_list:
            candidates_list.append(row[2])
            total_dict[row[2]]=0
        # finding the votes count for the candidates
        total_dict[row[2]]=total_dict[row[2]]+1
    # printing the results and writing to text file
    with open(file, 'w') as text:
        # printing total votes
        Total_votes_print=("Election Results\n"+"-------------------------\n"+"Total Votes: "+str(total_votes)+"\n-------------------------")
        print(Total_votes_print)
        text.write(Total_votes_print)
        # iterating over the dictionary to print the results
        for x,y in total_dict.items():
            candidates=("\n"+x+": "+str(round((y/total_votes)*100,4))+"00% ("+str(y)+")")
            print(candidates)
            text.write(candidates)
            # finding the winner
            if(y>winner):
                winner=y
                winnername=x
        # printing the winner
        winnerprint=("\n-------------------------\nWinner: "+ winnername + "\n-------------------------")
        print(winnerprint)
        text.write(winnerprint)

# In[ ]:


import datetime
from explorer.api import util
from flask import request
from explorer.services.message import TipsetGasService, TipsetService, MessageService
from base.response import response_json

def sync_tipset_gas():
    """
    同步单个区块的全网gas汇总
    :return:
    """

    end_index = request.form.get('end_index')
    start_index = request.form.get('start_index')
    TipsetGasService.sync_overview_tipset_gas()
    return response_json(True)

def sync_messages_stat():
    """

    :return:
    """
    TipsetGasService.sync_messages_stat()
    return response_json(True)

def get_gas_trends():
    """
    获取gas费用趋势图
    :return:
    """
    search_type = request.form.get('search_type', 'day')
    start_date = request.form.get('start_date')
    end_date = request.form.get('end_date')
    step, start_date, end_date = util.trends_parm(search_type, start_date, end_date)
    data = TipsetService.get_gas_trends(start_date=start_date, end_date=end_date, step=step)
    return response_json(data)

def get_gas_stat_all():
    """
    获取gas24小时统计
    :return:
    """
    data = TipsetService.get_gas_stat_all()
    return response_json(data)

def get_message_list():
    miner_id = request.form.get('miner_id')
    block_hash = request.form.get('block_hash')
    msg_method = request.form.get('msg_method')
    msgrct_exit_code = request.form.get('msgrct_exit_code')
    page_index = int(request.form.get('page_index', 1))
    page_size = min(int(request.form.get('page_size', 20)), 50)
    data = MessageService.get_message_list(miner_id, block_hash, msg_method, msgrct_exit_code,
                                           page_index=page_index, page_size=page_size)
    return response_json(data)

def get_message_method_list():
    """
    获取方法详情
    :return:
    """
    miner_id = request.form.get('miner_id')
    block_hash = request.form.get('block_hash')
    data = MessageService.get_message_method_list(miner_id, block_hash)
    return response_json(data)

def get_message_info():
    """
    获取消息详情
    :return:
    """
    msg_cid = request.form.get('msg_cid')
    height = int(request.form.get('height'))
    result = MessageService.get_message_info(msg_cid, height)
    return response_json(result)

def get_mpool_list():
    key_words = request.form.get('key_words')
    msg_method = request.form.get('msg_method')
    page_index = int(request.form.get('page_index', 1))
    page_size = min(int(request.form.get('page_size', 10)), 50)
    data = MessageService.get_mpool_list(key_words, msg_method, page_index=page_index, page_size=page_size)
    return response_json(data)

def get_mpool_info():
    cid = request.form.get('cid')
    result = MessageService.get_mpool_info(cid)
    return response_json(result)

def get_transfer_list():
    """
    获取转账列表
    :return:
    """
    msg_method = request.form.get('msg_method')
    miner_id = request.form.get('miner_id')
    start_date = request.form.get('start_date')
    end_date = request.form.get('end_date')
    page_size = int(request.form.get('page_size', 20))
    page_index = int(request.form.get('page_index', 1))
    data = MessageService.get_message_list(miner_id, msg_method=msg_method, start_date=start_date, end_date=end_date,
                                           is_transfer=True, page_index=page_index, page_size=page_size)
    return response_json(data)

def get_transfer_method_list():
    """
    获取转账列表方法详情
    :return:
    """
    miner_id = request.form.get('miner_id')
    start_date = request.form.get('start_date')
    end_date = request.form.get('end_date')
    data = MessageService.get_message_method_list(miner_id, start_date=start_date, end_date=end_date,
                                                  is_transfer=True)
    return response_json(data)


#calss header
class _BIOFEEDBACK():
	def __init__(self,):
		self.name = "BIOFEEDBACK"
		self.definitions = [u'a method by which a person learns to control their heart rate or other physical or mental processes by using information from recordings of those processes']

		self.parents = []
		self.childen = []
		self.properties = []
		self.jsondata = {}

		self.specie = 'nouns'

	def run(self, obj1 = [], obj2 = []):
		return self.jsondata


"""
Author: Johan Lundstroem
Email: lundstrom.se at gmail.com
Date: 12/9 2011
License: GPLv2
"""

import __init__
import subprocess
from datetime import datetime
from datetime import timedelta
import settings
import logging
import time

class Testcase(object):

  def init(self, thread=None, log_name=None):
    self.log_name = log_name
    if not thread == None:
      self.name = self.name+"_thread:"+str(thread)
    if self.log == None:
      if thread == None:
        # Creating a log file
        self.log_name = caller = self.name
        self.file_path = settings.log_file_path
        time = datetime.now()
        time = str(time)
        time = time.replace(' ','')
        time = time.replace(':','-')
        self.file_name = "%s_%s_%s_%s.log" % (caller, caller, caller, time)
        self.file = self.file_path+self.file_name
        f = open(self.file, "w")
        f.close()
        logging.basicConfig(filename=self.file, level=logging.DEBUG,format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
      log = logging.getLogger(self.log_name)
      log.setLevel(logging.INFO)
      # create console handler and set level to debug
      ch = logging.StreamHandler()
      ch.setLevel(logging.INFO)
      # create formatter
      formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
      # add formatter to ch
      ch.setFormatter(formatter)
      # add ch to logger
      log.addHandler(ch)
      self.log=log
    self.log.info("%s initiated" % (self.name))

  def run(self):
    pass

  def exe(self, what, max_time=None):
    if max_time == None: max_time = timedelta(seconds=10, microseconds=0)
    start = datetime.now()
    #self.log.info("start %s..." % (inspect.stack()[1][3]))
    self.log.info("%s start %s..." % (self.name, what.__name__))
    what()
    #self.log.info("end %s..." % (inspect.stack()[1][3]))
    #self.log.info("end %s..." % (what.__name__))
    stop = datetime.now()
    exe_time = stop-start
    if exe_time > max_time:
      self.log.info('The maximum response time was exseeded. %s' % (str(exe_time)))
      # Handle timeout here
    #self.log.info("%s duration: %s" % (inspect.stack()[1][3], str(stop-start)))
    self.log.info("%s end %s duration: %s" % (self.name, what.__name__, str(stop-start)))

# Manual execution
#if __name__ == "__main__":
#  t = testcase()
#  t.do_test()


import csv
import tensorflow as tf
from tensorflow.keras import layers
import numpy as np
from numpy.random import RandomState
import pandas as pd

#get data from raw url, read from csv file & store in dummy var

orig_data = tf.keras.utils.get_file('aug_train.csv', 'https://raw.githubusercontent.com/thunder789066/neural-networks-461/main/aug_train.csv')
orig_data = pd.read_csv(orig_data)
clean_data = pd.get_dummies(orig_data)

randState = RandomState()

#data divided 70% training, 15% test, 15% validation
training_dataset = clean_data.sample(frac = 0.7, random_state = randState)
other_dataset = clean_data.loc[~clean_data.index.isin(training_dataset.index)]

test_dataset = other_dataset.sample(frac = 0.5, random_state = randState)
validation_dataset = clean_data.loc[~clean_data.index.isin(test_dataset.index)]

training_target = training_dataset.pop('target')
test_target = test_dataset.pop('target')
validation_target = validation_dataset.pop('target')

train_data_subset = tf.data.Dataset.from_tensor_slices((training_dataset.values, training_target.values))
test_data_subset = tf.data.Dataset.from_tensor_slices((test_dataset.values, test_target.values))
validate_data_subset = tf.data.Dataset.from_tensor_slices((validation_dataset.values, validation_target.values))

#compile model
orig_model = tf.keras.Model
orig_model = tf.keras.Sequential([
        tf.keras.layers.Dense(64, activation="tanh"),
        tf.keras.layers.Dense(128, activation="tanh"),
        tf.keras.layers.Dense(1, activation="sigmoid")])

orig_model.compile(optimizer = 'Adam', loss = 'mean_squared_error', metrics = ['accuracy'])
orig_model.fit(training_dataset, training_target, epochs = 20)
orig_model.evaluate(test_data_subset.batch(128))


from flask import Flask
from flask_sockets import Sockets

import numpy as np
import torch
from torchvision import transforms
from torchvision.datasets import MNIST
from torchvision.datasets import ImageFolder
from tqdm import tqdm

# from utils import AddGaussianNoise  # pylint: disable=import-error

KEEP_LABELS_DICT = {
    "alice": [0, 1, 2, 3],
    "bob": [4, 5, 6],
    "charlie": [7, 8, 9],
    None: list(range(10)),
}

def create_app(node_id, debug=False, database_url=None, data_dir: str = None):
    """ Create / Configure flask socket application instance.

        Args:
            node_id (str) : ID of Grid Node.
            debug (bool) : debug flag.
            test_config (bool) : Mock database environment.
        Returns:
            app : Flask application instance.
    """
    app = Flask(__name__)
    app.debug = debug

    app.config["SECRET_KEY"] = "justasecretkeythatishouldputhere"

    # Enable persistent mode
    # Overwrite syft.object_storage methods to work in a persistent way
    # Persist models / tensors
    if database_url:
        app.config["REDISCLOUD_URL"] = database_url
        from .main.persistence import database, object_storage

        db_instance = database.set_db_instance(database_url)
        object_storage.set_persistent_mode(db_instance)

    from .main import html, ws, hook, local_worker, auth

    # Global socket handler
    sockets = Sockets(app)

    # set_node_id(id)
    local_worker.id = node_id
    hook.local_worker._known_workers[node_id] = local_worker
    local_worker.add_worker(hook.local_worker)

    # add data
    if data_dir:
        print("register data")
        if "mnist" in data_dir.lower():
            dataset = MNIST(
                root="./data",
                train=True,
                download=True,
                transform=transforms.Compose(
                    [transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))]
                ),
            )
            if node_id in KEEP_LABELS_DICT:
                indices = np.isin(dataset.targets, KEEP_LABELS_DICT[node_id]).astype(
                    "uint8"
                )
                selected_data = (
                    torch.native_masked_select(  # pylint:disable=no-member
                        dataset.data.transpose(0, 2),
                        torch.tensor(indices),  # pylint:disable=not-callable
                    )
                    .view(28, 28, -1)
                    .transpose(2, 0)
                )
                selected_targets = torch.native_masked_select(  # pylint:disable=no-member
                    dataset.targets,
                    torch.tensor(indices),  # pylint:disable=not-callable
                )
            """ dataset = sy.BaseDataset(
                    data=selected_data,
                    targets=selected_targets,
                    transform=dataset.transform,
                )
            dataset_name = "mnist"
            """
        else:

            train_tf = [
                transforms.RandomVerticalFlip(p=0.5),
                transforms.RandomAffine(
                    degrees=30,
                    translate=(0, 0),
                    scale=(0.85, 1.15),
                    shear=10,
                    #    fillcolor=0.0,
                ),
                transforms.Resize(224),
                transforms.RandomCrop(224),
                transforms.ToTensor(),
                transforms.Normalize((0.57282609,), (0.17427578,)),
                # transforms.RandomApply([AddGaussianNoise(mean=0.0, std=0.05)], p=0.5),
            ]
            """train_tf.append(
                transforms.Lambda(
                    lambda x: torch.repeat_interleave(  # pylint: disable=no-member
                        x, 3, dim=0
                    )
                )
            )"""
            target_dict_pneumonia = {0: 1, 1: 0, 2: 2}
            dataset = ImageFolder(
                data_dir,
                transform=transforms.Compose(train_tf),
                target_transform=lambda x: target_dict_pneumonia[x],
            )
            data, targets = [], []
            for d, t in tqdm(dataset, total=len(dataset)):
                data.append(d)
                targets.append(t)
            selected_data = torch.stack(data)  # pylint:disable=no-member
            selected_targets = torch.from_numpy(np.array(targets))  # pylint:disable=no-member
            #dataset = sy.BaseDataset(data=data, targets=targets)
            """dataset = PPPP(
                "data/Labels.csv",
                train=True,
                transform=transforms.Compose(train_tf),
                seed=1
            )"""
            dataset_name = "pneumonia"

        local_worker.register_obj(selected_data, "data")
        local_worker.register_obj(selected_targets, "targets")

        print("registered data")

    # Register app blueprints
    app.register_blueprint(html, url_prefix=r"/")
    sockets.register_blueprint(ws, url_prefix=r"/")

    # Set Authentication configs
    app = auth.set_auth_configs(app)

    return app


from django.shortcuts import render, redirect, reverse, get_object_or_404
from products.models import Product
from .forms import ReviewForm
from django.contrib.auth.decorators import login_required
from django.contrib import messages
from django import forms

@login_required
def add_review(request, product_id):
    product = get_object_or_404(Product, pk=product_id)
    if request.method == 'POST':
        form = ReviewForm(request.POST, request.FILES)
        form.instance.user = request.user
        if form.is_valid():
            form.save()
            messages.success(request, 'Successfully added review!')
            return redirect(reverse('product_detail', args=[product.id]))
        else:
            messages.error(
                request, 'Review not added. Please check your form.')
    else:
        initial = {'product': product}
        form = ReviewForm(initial=initial)
        form.fields['product'].widget = forms.HiddenInput()

    template = 'review/add_review.html'
    context = {
        'form': form,
        'product': product,
    }
    return render(request, template, context)


import logging
import time
import warnings
from typing import Optional, Dict, Union, Callable, List

import cloudpickle
import numpy as np
import pandas as pd
import xarray as xr

from pywatts.callbacks import BaseCallback
from pywatts.core.base import Base, BaseEstimator
from pywatts.core.base_condition import BaseCondition
from pywatts.core.base_step import BaseStep
from pywatts.core.base_summary import BaseSummary
from pywatts.core.run_setting import RunSetting
from pywatts.core.computation_mode import ComputationMode
from pywatts.core.exceptions.not_fitted_exception import NotFittedException
from pywatts.core.filemanager import FileManager
from pywatts.core.result_step import ResultStep
from pywatts.utils._xarray_time_series_utils import _get_time_indexes

logger = logging.getLogger(__name__)

class Step(BaseStep):
    """
    This step encapsulates modules and manages all information for executing a pipeline step.
    Including fetching the input from the input and target step.

    :param module: The module which is wrapped by the step-
    :type module: Base
    :param input_step: The input_step of the module.
    :type input_step: Step
    :param file_manager: The file_manager which is used for storing data.
    :type file_manager: FileManager
    :param target: The step against which's output the module of the current step should be fitted. (Default: None)
    :type target: Optional[Step]
    :param computation_mode: The computation mode which should be for this step. (Default: ComputationMode.Default)
    :type computation_mode: ComputationMode
    :param callbacks: Callbacks to use after results are processed.
    :type callbacks: List[Union[BaseCallback, Callable[[Dict[str, xr.DataArray]], None]]]
    :param condition: A callable which checks if the step should be executed with the current data.
    :type condition: Callable[xr.DataArray, xr.DataArray, bool]
    :param refit_conditions: A List of Callables of BaseConditions, which contains a condition that indicates if
                                 the module should be trained or not
    :type refit_conditions: List[Union[BaseCondition, Callable]]
    :param lag: Needed for online learning. Determines what data can be used for retraining.
            E.g., when 24 hour forecasts are performed, a lag of 24 hours is needed, else the retraining would
            use future values as target values.
    :type lag: pd.Timedelta
    """

    def __init__(self, module: Base, input_steps: Dict[str, BaseStep], file_manager, *,
                 targets: Optional[Dict[str, "BaseStep"]] = None,
                 computation_mode=ComputationMode.Default,
                 callbacks: List[Union[BaseCallback, Callable[[Dict[str, xr.DataArray]], None]]] = [],
                 condition=None,
                 batch_size: Optional[None] = None,
                 refit_conditions=[],
                 lag=pd.Timedelta(hours=24),
                 config_summary=[]):
        super().__init__(input_steps, targets, condition=condition,
                         computation_mode=computation_mode, name=module.name)
        self.file_manager = file_manager
        self.module = module
        self.callbacks = callbacks
        self.batch_size = batch_size
        if self.current_run_setting.computation_mode is not ComputationMode.Refit and len(refit_conditions) > 0:
            message = "You added a refit_condition without setting the computation_mode to refit." \
                      " The condition will be ignored."
            warnings.warn(message)
            logger.warning(message)
        self.lag = lag
        self.config_summary = config_summary
        self.refit_conditions = refit_conditions
        self.result_steps: Dict[str, ResultStep] = {}
        self.recalculated = False

    def _fit(self, inputs: Dict[str, BaseStep], target_step):
        # Fit the encapsulate module, if the input and the target is not stopped.
        self.module.fit(**inputs, **target_step)

    def _callbacks(self):
        # plots and writs the data if the step is finished.
        for callback in self.callbacks:
            dim = _get_time_indexes(self.result_buffer)[0]

            if self.current_run_setting.online_start is not None:
                to_plot = {k: self.result_buffer[k][self.result_buffer[k][dim] >= self.current_run_setting.online_start]
                           for k in
                           self.result_buffer.keys()}
            else:
                to_plot = self.result_buffer
            if isinstance(callback, BaseCallback):
                callback.set_filemanager(self.file_manager)
            if isinstance(self.result_buffer, xr.DataArray) or isinstance(self.result_buffer, xr.Dataset):
                # DEPRECATED: direct DataArray or Dataset passing is depricated
                callback({"deprecated": self.result_buffer})
            else:
                callback(to_plot)

    def _transform(self, input_step):
        if isinstance(self.module, BaseEstimator) and not self.module.is_fitted:
            message = f"Try to call transform in {self.name} on not fitted module {self.module.name}"
            logger.error(message)
            raise NotFittedException(message, self.name, self.module.name)
        result = self.module.transform(**input_step)
        return self._post_transform(result)

    @classmethod
    def load(cls, stored_step: Dict, inputs, targets, module, file_manager):
        """
        Load a stored step.

        :param stored_step: Informations about the stored step
        :param inputs: The input step of the stored step
        :param targets: The target step of the stored step
        :param module: The module wrapped by this step
        :return: Step
        """
        if stored_step["condition"]:
            with open(stored_step["condition"], 'rb') as pickle_file:
                condition = cloudpickle.load(pickle_file)
        else:
            condition = None
        refit_conditions = []
        for refit_condition in stored_step["refit_conditions"]:
            with open(refit_condition, 'rb') as pickle_file:
                refit_conditions.append(cloudpickle.load(pickle_file))

        callbacks = []
        for callback_path in stored_step["callbacks"]:
            with open(callback_path, 'rb') as pickle_file:
                callback = cloudpickle.load(pickle_file)
            callback.set_filemanager(file_manager)
            callbacks.append(callback)

        step = cls(module, inputs, targets=targets, file_manager=file_manager, condition=condition,
                   refit_conditions=refit_conditions, callbacks=callbacks, batch_size=stored_step["batch_size"])
        step.default_run_setting = RunSetting.load(stored_step["default_run_setting"])
        step.current_run_setting = step.default_run_setting.clone()
        step.id = stored_step["id"]
        step.name = stored_step["name"]
        step.last = stored_step["last"]

        return step

    def _compute(self, start, end, minimum_data):
        input_data = self._get_input(start, end, minimum_data)
        target = self._get_target(start, end, minimum_data)
        if self.current_run_setting.computation_mode in [ComputationMode.Default, ComputationMode.FitTransform,
                                                         ComputationMode.Train]:
            # Fetch input_data and target data
            if self.batch_size:
                input_batch = self._get_input(end - self.batch_size, end, minimum_data)
                target_batch = self._get_target(end - self.batch_size, end, minimum_data)
                start_time = time.time()
                self._fit(input_batch, target_batch)
                self.training_time.set_kv("", time.time() - start_time)
                config_summary = {}
                for config in self.config_summary:
                    if hasattr(self.module, "module"):
                        if hasattr(self.module.module, config):
                            config_summary.update({config: getattr(self.module.module, config)})
                    elif hasattr(self.module, config):
                        config_summary.update({config: getattr(self.module, config)})
                if config_summary:
                    self.fit_config.set_kv(f"", config_summary)
            else:
                start_time = time.time()
                self._fit(input_data, target)
                for refit_condition in self.refit_conditions:
                    if isinstance(refit_condition, BaseCondition):
                        if refit_condition.is_fitted == False:
                            # the refit_condition could already be fitted in a previous step
                            refit_condition.fit(start, end)
                self.training_time.set_kv("", time.time() - start_time)
                config_summary = {}
                for config in self.config_summary:
                    if hasattr(self.module, "module"):
                        if hasattr(self.module.module, config):
                            config_summary.update({config: getattr(self.module.module, config)})
                    elif hasattr(self.module, config):
                        config_summary.update({config: getattr(self.module, config)})
                if config_summary:
                    self.fit_config.set_kv(f"", config_summary)

        elif self.module is BaseEstimator:
            logger.info("%s not fitted in Step %s", self.module.name, self.name)

        start_time = time.time()
        result = self._transform(input_data)
        self.transform_time.set_kv("", time.time() - start_time)
        result_dict = {}
        for key, res in result.items():
            index = res.indexes[_get_time_indexes(result)[0]]
            start_ = max(index[0], start.to_numpy()) if start is not None else index[0]
            result_dict[key] = res.sel(**{_get_time_indexes(res)[0]: index[(index >= start_)]})
        return result_dict

    def _get_target(self, start, batch, minimum_data=(0, pd.Timedelta(0))):
        min_data_module = self.module.get_min_data()
        if isinstance(min_data_module, (int, np.integer)):
            minimum_data = minimum_data[0] + min_data_module, minimum_data[1]
        else:
            minimum_data = minimum_data[0], minimum_data[1] + min_data_module
        return {
            key: target.get_result(start, batch, minimum_data=minimum_data)
            for key, target in self.targets.items()
        }

    def _get_input(self, start, batch, minimum_data=(0, pd.Timedelta(0)), use_result_buffer=False):
        min_data_module = self.module.get_min_data()
        if isinstance(min_data_module, (int, np.integer)):
            minimum_data = minimum_data[0] + min_data_module, minimum_data[1]
        else:
            minimum_data = minimum_data[0], minimum_data[1] + min_data_module
        return {
            key: input_step.get_result(start, batch, minimum_data=minimum_data, use_result_buffer=use_result_buffer) for
            key, input_step in self.input_steps.items()
        }

    def get_json(self, fm: FileManager):
        json = super().get_json(fm)
        condition_path = None
        refit_conditions_paths = []
        callbacks_paths = []
        if self.condition:
            condition_path = fm.get_path(f"{self.name}_condition.pickle")
            with open(condition_path, 'wb') as outfile:
                cloudpickle.dump(self.condition, outfile)
        for refit_condition in self.refit_conditions:
            refit_conditions_path = fm.get_path(f"{self.name}_refit_conditions.pickle")
            with open(refit_conditions_path, 'wb') as outfile:
                cloudpickle.dump(refit_condition, outfile)
            refit_conditions_paths.append(refit_conditions_path)
        for callback in self.callbacks:
            callback_path = fm.get_path(f"{self.name}_callback.pickle")
            with open(callback_path, 'wb') as outfile:
                cloudpickle.dump(callback, outfile)
            callbacks_paths.append(callback_path)
        json.update({"callbacks": callbacks_paths,
                     "condition": condition_path,
                     "refit_conditions": refit_conditions_paths,
                     "batch_size": self.batch_size})
        return json

    def refit(self, start: pd.Timestamp, end: pd.Timestamp):
        """
        Refits the module of the step.
        :param start: The date of the first data used for retraining.
        :param end: The date of the last data used for retraining.
        """
        if self.current_run_setting.computation_mode in [ComputationMode.Refit] and isinstance(self.module,
                                                                                BaseEstimator):
            refitted = False
            # this flag is required if multiple refit conditions evaluate True at the same time, e.g.,
            # [PeriodicCondition(28, refit_params_A), PeriodicCondition(7, refit_params_B).
            # In this case, the first condition is dominant, applies the refit_params, and refits,
            # but we also have to evaluate the second condition to increase its counter
            for refit_condition in self.refit_conditions:
                if isinstance(refit_condition, BaseCondition):
                    eval_ = refit_condition.evaluate(start, end)
                    if eval_ and not refitted:
                        refit_batch = refit_condition.refit_batch.get(self.name) \
                            if isinstance(refit_condition.refit_batch, dict) \
                            else refit_condition.refit_batch
                        refit_params = refit_condition.refit_params.get(self.name) \
                            if all(isinstance(value, dict) for value in refit_condition.refit_params.values()) \
                            else refit_condition.refit_params
                        self._refit(end, refit_batch, refit_params)
                        self._recalculate(end)
                        refitted = True
                elif isinstance(refit_condition, Callable):
                    input_data = self._get_input(start, end)
                    target = self._get_target(start, end)
                    if refit_condition(input_data, target):
                        self._refit(end)
                        break
        else:
            recalculated_inputs = [input_step.recalculated for input_step in self.input_steps.values()
                                   if hasattr(input_step, 'recalculated')]
            recalculated_targets = [input_step.recalculated for input_step in self.input_steps.values()
                                    if hasattr(input_step, 'recalculated')]
            recalculated = recalculated_inputs + recalculated_targets
            if any(recalculated):
                self._recalculate(end)

    def _refit(self, end, refit_batch, refit_params=None):
        if isinstance(refit_batch, pd.Timedelta):
            refit_input = self._get_input(end - refit_batch, end)
            refit_target = self._get_target(end - refit_batch, end)
        elif isinstance(refit_batch, (tuple, list)):
            s_, e_ = refit_batch[0], refit_batch[1]
            refit_input = self._get_input(s_, e_)
            refit_target = self._get_target(s_, e_)
        else:
            raise TypeError("refit_batch is not a Timedelta or a tuple or list of Timestamps!")

        if refit_params is not None:
            self.module.set_params(**refit_params)
        start_time = time.time()
        self.module.refit(**refit_input, **refit_target)
        self.refit_time.set_kv(f"refit at position {end}", time.time() - start_time)
        config_summary = {}
        for config in self.config_summary:
            if hasattr(self.module, "module"):
                if hasattr(self.module.module, config):
                    config_summary.update({config: getattr(self.module.module, config)})
            elif hasattr(self.module, config):
                config_summary.update({config: getattr(self.module, config)})
        if config_summary:
            self.refit_config.set_kv(f"refit at position {end}", config_summary)

    def _recalculate(self, end):
        if isinstance(self.module, BaseSummary):
            self.recalculated = True
            return None  # summaries have no buffer
        index = _get_time_indexes(self.result_buffer)
        start = pd.Timestamp(list(self.result_buffer.values())[0][index[0]].data[0])
        recalculate_input = self._get_input(start, end)

        # We need to call the transform already here, otherwise following steps would not get the recalculated data.
        # Move data from the current buffer to the result buffer and fill the current buffer with the recalculated data.
        self.renew_current_buffer()
        self._transform(recalculate_input)
        self.recalculated = True

    def get_result_step(self, item: str):
        if item not in self.result_steps:
            self.result_steps[item] = ResultStep(input_steps={"result": self}, buffer_element=item)
        return self.result_steps[item]


import time

class Graph:

    def __init__(self, edge_list, n):
        self._visited = set()
        self._graph = {key:[] for key in range(1, n+1)}
        self.add_edge(edge_list)
        self._reach = [0]*n

    def add_edge(self, edge_list):
        for u, v in edge_list:
            self._graph[u].append(v)
            self._graph[v].append(u)

    def bfs(self, start):
        queue = []
        queue.append(start)
        self._visited.add(start)

        while queue:
            s = queue.pop(0)
            print(self._graph[s])
            print(self._visited)
            print("===")
            for v in self._graph[s]:
                if v not in self._visited:
                    self._visited.add(v)
                    self._reach[v - 1]  = self._reach[s - 1] + 6
                    queue.append(v)

        del self._reach[start - 1]
        return [(-1 if i == 0 else i) for i in self._reach]

def bfs(s, edges, n):
    weight = 6
    dist = [0] * (n + 1)
    dist[s] = 0
    for _ in range(n):
        # A flag indicating whether update happens.
        updated = False
        for edge in edges:
            edge_src, edge_dest = edge
            print("{} - {}".format(edge_src, edge_dest))
            if dist[edge_src] + weight < dist[edge_dest]:
                updated = True
                dist[edge_dest] = dist[edge_src] + weight

            print(dist)

        if not updated:
            # Early exit since distances are now stable.
            break

    # Perform certain transformations on the output.
    del dist[s]
    del dist[0]
    return [(-1 if i == 0 else i) for i in dist]

if __name__ == "__main__":

    n1 = 4
    s1 = 1
    edges1 = [[1, 2],
             [1, 3]]

    n2 = 3
    s2 = 2
    edges2 = [[2, 3]]

    n3 = 5
    s3 = 1
    edges3 = [[1, 2],
              [1, 3],
              [3, 4]]

    start = time.time()
    g = Graph(edges1, n1)
    #g.bfs(s1)
    print(g.bfs(s1))

    print("-----------")
    g = Graph(edges2, n2)
    #g.bfs(s1)
    print(g.bfs(s2))

    print("-----------")
    g = Graph(edges3, n3)
    #g.bfs(s1)
    print(g.bfs(s3))


import os
import fnmatch
import sys
import shutil

def _collect_collection_met(collection_path, target_path):
    collection_base_dir = os.path.dirname(collection_path)

    # matches = []
    for root, dirnames, filenames in os.walk(collection_path):
        for filename in fnmatch.filter(filenames, '*.cfg'):
            # matches.append(os.path.join(root, filename))
            rel_path = os.path.relpath(os.path.join(root, filename), start=collection_base_dir)
            print('copying', os.path.join(root, filename), os.path.join(target_path, rel_path))
            os.makedirs(os.path.dirname(os.path.join(target_path, rel_path)), exist_ok=True)
            shutil.copyfile(os.path.join(root, filename), os.path.join(target_path, rel_path))

if __name__ == "__main__":

    archive_path=None
    for arg_strng in sys.argv:
        if 'collection_path' in arg_strng:
            collection_path=arg_strng.split('=')[1]
        if 'archive_path' in arg_strng:
            archive_path=arg_strng.split('=')[1]
        if 'target_path' in arg_strng:
            target_path = arg_strng.split('=')[1]

    if archive_path!=None:
        collection_paths=[os.path.join(archive_path,item) for item in os.listdir(archive_path)]
    else:
        collection_paths=[collection_path]

    for collection_path in collection_paths:
        print('Publishing collection at:',collection_path)
        _collect_collection_met(collection_path, target_path)

# # to copy all collections and their metadata files from the archive dir to a new location:
# python3 collect_metadata_files.py --archive_path=/usr/local/labcas/backend/archive --target_path=/home/edrn-fn/EDRN-metadata

# # to copy a single collection and its metadata files to a new location:
# python3 collect_metadata_files.py --collection_path=/usr/local/labcas/backend/archive/<collection_dir_name> --target_path=/home/edrn-fn/EDRN-metadata

# # to update metadata files for a single collection from github's metadata repository:
# python3 collect_metadata_files.py --collection_path=/home/edrn-fn/EDRN-metadata/<collection_dir_name> --target_path=/usr/local/labcas/backend/archive


# Copyright 2014, 2015 IBM Corp.
#
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Wrappers for VirtualIOServer and virtual storage mapping elements."""

import abc
import copy
import re
import six

from oslo_log import log as logging

import pypowervm.entities as ent
import pypowervm.util as u
import pypowervm.wrappers.base_partition as bp
import pypowervm.wrappers.entry_wrapper as ewrap
import pypowervm.wrappers.logical_partition as lpar
import pypowervm.wrappers.managed_system as ms
import pypowervm.wrappers.network as net
import pypowervm.wrappers.storage as stor

LOG = logging.getLogger(__name__)

# VIO Constants
_VIO_API_CAP = 'APICapable'
_VIO_SVR_INST_CFG = 'ServerInstallConfiguration'
_VIO_LNAGGS = 'LinkAggregations'
_VIO_MGR_PASSTHRU_CAP = 'ManagerPassthroughCapable'
_VIO_MEDIA_REPOS = 'MediaRepositories'
_VIO_MVR_SVC_PARTITION = 'MoverServicePartition'
_VIO_NET_BOOT_DEVS = 'NetworkBootDevices'
_VIO_PAGING_SVC_PARTITION = 'PagingServicePartition'
_VIO_PVS = stor.PVS
_VIO_SEAS = net.NB_SEAS
_VIO_SSP_CAP = 'SharedStoragePoolCapable'
_VIO_SSP_VER = 'SharedStoragePoolVersion'
_VIO_STOR_POOLS = 'StoragePools'
_VIO_TRUNK_ADPTS = net.SEA_TRUNKS
_VIO_LICENSE = 'VirtualIOServerLicense'
_VIO_LICENSE_ACCEPTED = 'VirtualIOServerLicenseAccepted'
_VIO_VFC_MAPPINGS = 'VirtualFibreChannelMappings'
_VIO_VSCSI_MAPPINGS = 'VirtualSCSIMappings'
_VIO_FREE_IO_ADPTS_FOR_LNAGG = 'FreeIOAdaptersForLinkAggregation'
# "FreeEthernetBackingDevicesForSEA" is really misspelled in the schema.
_VIO_FREE_ETH_BACKDEVS_FOR_SEA = 'FreeEthenetBackingDevicesForSEA'
_VIO_VNIC_BACKDEVS = 'VirtualNICBackingDevices'

_VOL_UID = 'VolumeUniqueID'
_VOL_NAME = 'VolumeName'
_RESERVE_POLICY = 'ReservePolicy'

_IO_ADPT_CHOICE = 'IOAdapterChoice'
_IO_ADPT = 'IOAdapter'
_IO_LINK_AGG_ADPT_ID = 'AdapterID'
_IO_LINK_AGG_DESC = 'Description'
_IO_LINK_AGG_DEV_NAME = 'DeviceName'
_IO_LINK_AGG_DEV_TYPE = 'DeviceType'
_IO_LINK_AGG_DRC_NAME = 'DynamicReconfigurationConnectorName'
_IO_LINK_AGG_PHYS_LOC = 'PhysicalLocation'
_IO_LINK_AGG_UDID = 'UniqueDeviceID'

_VIRT_MEDIA_REPOSITORY_PATH = u.xpath(_VIO_MEDIA_REPOS,
                                      'VirtualMediaRepository')
_IF_ADDR = u.xpath('IPInterface', 'IPAddress')
_ETHERNET_BACKING_DEVICE = u.xpath(_VIO_FREE_ETH_BACKDEVS_FOR_SEA,
                                   'IOAdapterChoice', net.ETH_BACK_DEV)
_SEA_PATH = u.xpath(net.NB_SEAS, net.SHARED_ETH_ADPT)

# Mapping Constants
_MAP_STORAGE = 'Storage'
_MAP_CLIENT_LPAR = 'AssociatedLogicalPartition'
_MAP_PORT = 'Port'
_MAP_ORDER = (_MAP_CLIENT_LPAR, stor.CLIENT_ADPT, stor.SERVER_ADPT,
              _MAP_STORAGE)

_WWPNS_PATH = u.xpath(_VIO_VFC_MAPPINGS, 'VirtualFibreChannelMapping',
                      stor.CLIENT_ADPT, 'WWPNs')
_PVS_PATH = u.xpath(stor.PVS, stor.PHYS_VOL)

_VIOS_EL_ORDER = bp.BP_EL_ORDER + (
    _VIO_API_CAP, _VIO_SVR_INST_CFG, _VIO_LNAGGS, _VIO_MGR_PASSTHRU_CAP,
    _VIO_MEDIA_REPOS, _VIO_MVR_SVC_PARTITION, _VIO_NET_BOOT_DEVS,
    _VIO_PAGING_SVC_PARTITION, _VIO_PVS, _VIO_SEAS, _VIO_SSP_CAP,
    _VIO_SSP_VER, _VIO_STOR_POOLS, _VIO_TRUNK_ADPTS, _VIO_LICENSE,
    _VIO_LICENSE_ACCEPTED, _VIO_VFC_MAPPINGS, _VIO_VSCSI_MAPPINGS,
    _VIO_FREE_IO_ADPTS_FOR_LNAGG, _VIO_FREE_ETH_BACKDEVS_FOR_SEA,
    _VIO_VNIC_BACKDEVS)

@ewrap.EntryWrapper.pvm_type('VirtualIOServer', child_order=_VIOS_EL_ORDER)
class VIOS(bp.BasePartition):

    # Extended Attribute Groups
    xags = ent.XAG(NETWORK='ViosNetwork',
                   STORAGE='ViosStorage',
                   SCSI_MAPPING='ViosSCSIMapping',
                   FC_MAPPING='ViosFCMapping')

    @classmethod
    def bld(cls, adapter, name, mem_cfg, proc_cfg, io_cfg=None):
        """Creates a new VIOS wrapper."""
        return super(VIOS, cls)._bld_base(adapter, name, mem_cfg, proc_cfg,
                                          env=bp.LPARType.VIOS, io_cfg=io_cfg)

    @property
    def media_repository(self):
        return self.element.find(_VIRT_MEDIA_REPOSITORY_PATH)

    def get_vfc_wwpns(self):
        """Returns a list of the virtual FC WWPN pairs for the vios.

        The response is a List of Lists.
        Ex. (('c05076065a8b005a', 'c05076065a8b005b'),
             ('c05076065a8b0060', 'c05076065a8b0061'))
        """
        return set([frozenset(x.split()) for x in
                    self._get_vals(_WWPNS_PATH)])

    def get_pfc_wwpns(self):
        """Returns a set of the Physical FC Adapter WWPNs on this VIOS."""
        path = u.xpath(bp.IO_CFG_ROOT, bp.IO_SLOTS_ROOT,
                       bp.IO_SLOT_ROOT, bp.ASSOC_IO_SLOT_ROOT,
                       bp.RELATED_IO_ADPT_ROOT, bp.IO_PFC_ADPT_ROOT,
                       bp.PFC_PORTS_ROOT, bp.PFC_PORT_ROOT,
                       bp.PFC_PORT_WWPN)
        return set(self._get_vals(path))

    def get_active_pfc_wwpns(self):
        """Returns a set of Physical FC Adapter WWPNs of 'active' ports."""
        # The logic to check for active ports is poor.  Right now it only
        # checks if the port has NPIV connections available.  If there is a
        # FC, non-NPIV card...then this logic fails.
        #
        # This will suffice until the backing API adds more granular logic.
        return [pfc.wwpn for pfc in self.pfc_ports if pfc.npiv_total_ports > 0]

    @property
    def pfc_ports(self):
        """The physical Fibre Channel ports assigned to the VIOS."""
        path = u.xpath(bp.IO_CFG_ROOT, bp.IO_SLOTS_ROOT,
                       bp.IO_SLOT_ROOT, bp.ASSOC_IO_SLOT_ROOT,
                       bp.RELATED_IO_ADPT_ROOT, bp.IO_PFC_ADPT_ROOT,
                       bp.PFC_PORTS_ROOT, bp.PFC_PORT_ROOT)
        elems = self._find(path, use_find_all=True)
        resp = []
        for elem in elems:
            resp.append(bp.PhysFCPort.wrap(elem))
        return resp

    @property
    def is_license_accepted(self):
        return self._get_val_bool(_VIO_LICENSE_ACCEPTED, default=True)

    def hdisk_reserve_policy(self, disk_uuid):
        """Get the reserve policy for an hdisk.

        :param disk_uuid: The uuid of the hdisk.
        :returns: The reserve policy or None if the disk isn't found.
        """
        policy = None

        # Get all the physical volume elements and look for a diskname match
        volumes = self.element.findall(_PVS_PATH)
        for volume in volumes:
            vol_uuid = volume.findtext(_VOL_UID)
            match = re.search(r'^[0-9]{5}([0-9A-F]{32}).+$', vol_uuid)

            if match and match.group(1) == disk_uuid:
                policy = volume.findtext(_RESERVE_POLICY)
                break

        return policy

    def hdisk_from_uuid(self, disk_uuid):
        """Get the hdisk name from the volume uuid.

        :param disk_uuid: The uuid of the hdisk.
        :returns: The associated hdisk name.
        """
        name = None

        # Get all the physical volume elements and look for a diskname match
        volumes = self.element.findall(_PVS_PATH)
        for volume in volumes:
            vol_uuid = volume.findtext(stor.UDID)
            if vol_uuid:
                LOG.debug('get_hdisk_from_uuid match: %s' % vol_uuid)
                LOG.debug('get_hdisk_from_uuid disk_uuid: %s' % disk_uuid)
                if vol_uuid == disk_uuid:
                    name = volume.findtext(_VOL_NAME)
                    break

        return name

    @property
    def is_mover_service_partition(self):
        return self._get_val_bool(_VIO_MVR_SVC_PARTITION, False)

    @property
    def ip_addresses(self):
        """Returns a list of IP addresses assigned to the VIOS.

        Will only return the IP Addresses that can be made known to the system.
        This only includes online Shared Ethernet Adapters and Ethernet Backing
        Devices.  It will not include, for example, a VLAN adapter.

        This is a READ-ONLY list.
        """
        ip_list = []

        # Get all the shared ethernet adapters and free
        # ethernet devices and pull the IPs
        seas = self.element.findall(_SEA_PATH)
        free_eths = self.element.findall(_ETHERNET_BACKING_DEVICE)
        for eth in seas + free_eths:
            ip = eth.findtext(_IF_ADDR)
            if ip and ip not in ip_list:
                ip_list.append(ip)

        return tuple(ip_list)

    @property
    def vfc_mappings(self):
        """Returns a WrapperElemList of the VFCMapping objects."""
        def_attrib = self.xags.FC_MAPPING.attrs
        es = ewrap.WrapperElemList(
            self._find_or_seed(_VIO_VFC_MAPPINGS, attrib=def_attrib),
            VFCMapping)
        return es

    @vfc_mappings.setter
    def vfc_mappings(self, new_mappings):
        self.replace_list(_VIO_VFC_MAPPINGS, new_mappings,
                          attrib=self.xags.FC_MAPPING.attrs)

    @property
    def scsi_mappings(self):
        """Returns a WrapperElemList of the VSCSIMapping objects."""
        def_attrib = self.xags.SCSI_MAPPING.attrs
        # TODO(efried): remove parent_entry once VIOS has pg83 in Events
        es = ewrap.WrapperElemList(
            self._find_or_seed(_VIO_VSCSI_MAPPINGS, attrib=def_attrib),
            VSCSIMapping, parent_entry=self)
        return es

    @scsi_mappings.setter
    def scsi_mappings(self, new_mappings):
        self.replace_list(_VIO_VSCSI_MAPPINGS, new_mappings,
                          attrib=self.xags.SCSI_MAPPING.attrs)

    @property
    def seas(self):
        def_attrib = self.xags.NETWORK.attrs
        es = ewrap.WrapperElemList(
            self._find_or_seed(net.NB_SEAS, attrib=def_attrib), net.SEA)
        return es

    @property
    def trunk_adapters(self):
        def_attrib = self.xags.NETWORK.attrs
        es = ewrap.WrapperElemList(
            self._find_or_seed(net.SEA_TRUNKS, attrib=def_attrib),
            net.TrunkAdapter)
        return es

    def derive_orphan_trunk_adapters(self):
        """Builds a list of trunk adapters not attached to a SEA."""
        sea_trunks = []
        for sea in self.seas:
            sea_trunks.append(sea.primary_adpt)
            sea_trunks.extend(sea.addl_adpts)

        # Subtract the list of our adapters from there.
        orig_trunks = copy.copy(self.trunk_adapters)
        orphan_trunks = copy.copy(self.trunk_adapters)
        for sea_trunk in sea_trunks:
            # We can't just remove because the trunk adapters from the SEA
            # have the vswitch ref instead of id...  So we have to compare
            # based off anchors.
            for ta in orig_trunks:
                if ta.dev_name == sea_trunk.dev_name:
                    orphan_trunks.remove(ta)
                    break
        return orphan_trunks

    @property
    def phys_vols(self):
        """Will return a list of physical volumes attached to this VIOS.

        This list is READ-ONLY.
        """
        # TODO(efried): remove parent_entry once VIOS has pg83 in Events
        es = ewrap.WrapperElemList(
            self._find_or_seed(stor.PVS, attrib=self.xags.STORAGE.attrs),
            stor.PV, parent_entry=self)
        es_list = [es_val for es_val in es]
        return tuple(es_list)

    @property
    def io_adpts_for_link_agg(self):
        es = ewrap.WrapperElemList(self._find_or_seed(
            _VIO_FREE_IO_ADPTS_FOR_LNAGG, attrib=self.xags.NETWORK.attrs),
            LinkAggrIOAdapterChoice)
        return es

@six.add_metaclass(abc.ABCMeta)
class VStorageMapping(ewrap.ElementWrapper):
    """Base class for VSCSIMapping and VFCMapping."""

    @staticmethod
    def crt_related_href(adapter, host_uuid, client_lpar_uuid):
        """Creates the Element for the 'AssociatedLogicalPartition'."""
        return adapter.build_href(ms.System.schema_type, host_uuid,
                                  lpar.LPAR.schema_type, client_lpar_uuid,
                                  xag=[])

    @property
    def client_lpar_href(self):
        """Returns the Client LPAR (if any) URI.

        If None - then no client is connected.
        """
        return self.get_href(_MAP_CLIENT_LPAR, one_result=True)

    def _client_lpar_href(self, href):
        self.set_href(_MAP_CLIENT_LPAR, href)

    @property
    def client_adapter(self):
        """Returns the Client side VSCSIClientAdapterElement.

        If None - then no client is connected.
        """
        elem = self.element.find(stor.CLIENT_ADPT)
        if elem is not None:
            return self._client_adapter_cls.wrap(elem)
        return None

    def _client_adapter(self, ca):
        elem = self._find_or_seed(stor.CLIENT_ADPT)
        self.element.replace(elem, ca.element)

    @property
    def server_adapter(self):
        """Returns the Virtual I/O Server side VSCSIServerAdapterElement."""
        return self._server_adapter_cls.wrap(
            self.element.find(stor.SERVER_ADPT))

    def _server_adapter(self, sa):
        elem = self._find_or_seed(stor.SERVER_ADPT)
        self.element.replace(elem, sa.element)

@ewrap.ElementWrapper.pvm_type('VirtualSCSIMapping', has_metadata=True,
                               child_order=_MAP_ORDER)
class VSCSIMapping(VStorageMapping):
    """The mapping of a VIOS SCSI adapter to the Client LPAR SCSI adapter.

    PowerVM provides a mechanism for Server/Client adapters to provide storage
    connectivity (for LPARs that do not have dedicated hardware).  This mapping
    describes the Virtual I/O Server's Server SCSI Adapter and the Client
    LPAR's Client SCSI Adapter.

    To create a new Client SCSI Adapter, create a new mapping and update the
    Virtual I/O Server.  This will be an atomic operation that creates the
    adapters on the Virtual I/O Server and Client LPAR, and then maps them
    properly.  There is no need to pre-create the adapters before creating a
    new mapping.
    """

    _client_adapter_cls = stor.VSCSIClientAdapterElement
    _server_adapter_cls = stor.VSCSIServerAdapterElement

    @classmethod
    def bld(cls, adapter, host_uuid, client_lpar_uuid, stg_ref):
        s_map = super(VSCSIMapping, cls)._bld(adapter)
        # Create the 'Associated Logical Partition' element of the mapping.
        s_map._client_lpar_href(
            cls.crt_related_href(adapter, host_uuid, client_lpar_uuid))
        s_map._client_adapter(stor.VClientStorageAdapterElement.bld(adapter))
        s_map._server_adapter(stor.VServerStorageAdapterElement.bld(adapter))
        s_map._backing_storage(stg_ref)
        return s_map

    @classmethod
    def bld_from_existing(cls, existing_map, stg_ref):
        """Clones the existing mapping, but swaps in the new storage elem."""
        # We do NOT want the TargetDevice element, so we explicitly copy the
        # pieces we want from the original mapping.
        new_map = super(VSCSIMapping, cls)._bld(existing_map.adapter)
        if existing_map.client_lpar_href is not None:
            new_map._client_lpar_href(existing_map.client_lpar_href)
        if existing_map.client_adapter is not None:
            new_map._client_adapter(copy.deepcopy(existing_map.client_adapter))
        if existing_map.server_adapter is not None:
            new_map._server_adapter(copy.deepcopy(existing_map.server_adapter))
        new_map._backing_storage(copy.deepcopy(stg_ref))
        return new_map

    @property
    def backing_storage(self):
        """The backing storage element (if applicable).

        Refer to the 'volume_group' wrapper.  This element may be a
        VirtualDisk or VirtualOpticalMedia.  May return None.
        """
        elem = self.element.find(_MAP_STORAGE)
        if elem is None:
            return None
        # If backing storage exists, it comprises a single child of elem.  But
        # type is unknown immediately, so call all children and then wrap.
        stor_elems = elem.getchildren()
        if len(stor_elems) != 1:
            return None
        # TODO(efried): parent_entry not needed once VIOS has pg83 in Events
        parent_entry = getattr(self, 'parent_entry', None)
        # The storage element may be any one of VDisk, VOptMedia, PV, or LU.
        # Allow ElementWrapper to detect (from the registry) and wrap correctly
        return ewrap.ElementWrapper.wrap(stor_elems[0],
                                         parent_entry=parent_entry)

    def _backing_storage(self, stg):
        """Sets the backing storage of this mapping to a VDisk, VOpt, LU or PV.

        :param stg: Either a VDisk, VOpt, LU or PV wrapper representing the
                    backing storage to assign.
        """
        # Always replace.  Because while the storage has one element, it can't
        # inject properly if the backing type changes (ex. cloning from vOpt to
        # vDisk).
        stor_elem = ent.Element(_MAP_STORAGE, self.adapter, attrib={},
                                children=[])
        stor_elem.inject(stg.element)
        self.inject(stor_elem)

@ewrap.ElementWrapper.pvm_type('VirtualFibreChannelMapping', has_metadata=True)
class VFCMapping(VStorageMapping):
    """The mapping of a VIOS FC adapter to the Client LPAR FC adapter.

    PowerVM provides a mechanism for Server/Client adapters to provide storage
    connectivity (for LPARs that do not have dedicated hardware).  This mapping
    describes the Virtual I/O Server's Server Fibre Channel (FC) Adapter and
    the Client LPAR's Client FC Adapter.

    To create a new Client FC Adapter, create a new mapping and update the
    Virtual I/O Server.  This will be an atomic operation that creates the
    adapters on the Virtual I/O Server and Client LPAR, and then maps them
    properly.  There is no need to pre-create the adapters before creating a
    new mapping.
    """

    _client_adapter_cls = stor.VFCClientAdapterElement
    _server_adapter_cls = stor.VFCServerAdapterElement

    @classmethod
    def bld(cls, adapter, host_uuid, client_lpar_uuid, backing_phy_port,
            client_wwpns=None):
        """Creates the VFCMapping object to connect to a Physical FC Port.

        This is used when creating a new mapping between a Client LPAR and the
        VirtualIOServer.  This creates a Fibre Channel connection between an
        LPAR and a physical Fibre Port.

        The response object should be used for creating the mapping via an
        adapter.update() to the Virtual I/O Server.  The response object
        will not have the UUIDs (as those are not assigned until the update is
        done).  This holds true for certain other elements as well.

        :param adapter: The pypowervm Adapter that will be used to create the
                        mapping.
        :param host_uuid: The host system's UUID.
        :param client_lpar_uuid: The client LPAR's UUID that the disk should be
                                 connected to.
        :param backing_phy_port: The name of the physical FC port that backs
                                 the virtual adapter.
        :param client_wwpns: An optional set of two WWPNs that can be set upon
                             the mapping.  These represent the client VM's
                             WWPNs on the client FC adapter.  If not set, the
                             system will dynamically generate them.
        :returns: The new VFCMapping Wrapper.
        """
        s_map = super(VFCMapping, cls)._bld(adapter)
        # Create the 'Associated Logical Partition' element of the mapping.
        s_map._client_lpar_href(
            cls.crt_related_href(adapter, host_uuid, client_lpar_uuid))
        s_map._client_adapter(stor.VFCClientAdapterElement.bld(
            adapter, wwpns=client_wwpns))

        # Create the backing port and change label.  API requires it be
        # Port, even though it is a Physical FC Port
        backing_port = bp.PhysFCPort.bld_ref(adapter, backing_phy_port)
        backing_port.element.tag = 'Port'
        s_map._backing_port(backing_port)

        s_map._server_adapter(stor.VFCServerAdapterElement.bld(adapter))
        return s_map

    def _backing_port(self, value):
        """Sets the backing port."""
        elem = self._find_or_seed(_MAP_PORT)
        self.element.replace(elem, value.element)

    @property
    def backing_port(self):
        """The Virtual I/O Server backing PhysicalFCPort.

        If None - then the vfcmap isn't done and no physical port is backing
        it.
        """
        elem = self.element.find(_MAP_PORT)
        if elem is not None:
            return bp.PhysFCPort.wrap(elem)
        return None

@ewrap.ElementWrapper.pvm_type(_IO_ADPT_CHOICE, has_metadata=False)
class LinkAggrIOAdapterChoice(ewrap.ElementWrapper):
    """A free I/O Adapter link aggregation choice.

    Flattens this two step heirarchy to pull the information needed directly
    from the IOAdapter element.
    """
    def __get_prop(self, func):
        """Thin wrapper to get the IOAdapter and get a property."""
        elem = self._find('IOAdapter')
        if elem is None:
            return None

        io_adpt = bp.IOAdapter.wrap(elem)
        return getattr(io_adpt, func)

    @property
    def id(self):
        return self.__get_prop('id')

    @property
    def description(self):
        return self.__get_prop('description')

    @property
    def dev_name(self):
        return self.__get_prop('dev_name')

    @property
    def dev_type(self):
        return self.__get_prop('dev_type')

    @property
    def drc_name(self):
        return self.__get_prop('drc_name')

    @property
    def phys_loc_code(self):
        return self.__get_prop('phys_loc_code')

    @property
    def udid(self):
        return self.__get_prop('udid')


from server.db.schema.sentences import GET_ALL_SENTENCES_BY_DATASET, GET_RANDOM_CORPA_AFTER, GET_STARTING_INDEX_SENTENCES_BY_DATASET, GET_TOTAL_CORPA_LEN_FORDB
from server.main.utils import get_session
import calendar
import datetime
import json
import os
import random
from server.config import Config
import shutil
import sys
import time
from pathlib import Path

from flask import (current_app, jsonify, redirect, render_template, request,
                   send_from_directory, session, url_for)
from flask_jwt_extended import jwt_required
from werkzeug.utils import secure_filename

import server.tasks_split.save as save
from server import cache
from server.db import UserFileSystem
from server.db.user import *
from server.main import main

DB = Database("data/data.db")
SENTENCEDB = Database("data/sentences.db")

# here it should be parents[2]
up_one = Path(__file__).parents[2]
# print(up_one)
folder = up_one / 'web_app' / 'src'

@main.route('/')
def index():
    return render_template('index.html')

def get_self_info():
    # get cookie by sending username & pass
    # TODO add a register server page
    # then redirect to the central server register page
    # save those here
    try:
        session = get_session()
        r = session.get(Config.CENTRAL_SERVER_INFO_URL)
        return r.json()
    except Exception:
        pass
    return {}

@main.route('/home')
def public_home():
    # TODO get request to the central server and get our own info
    # TODO self public info
    self_info = get_self_info()

    dataset_types = []
    # dataset_types = ['eng', 'hin', 'tel', 'mix-hin-tel']
    for dire in (up_one / 'corpora' / 'uploads').iterdir():
        if dire.is_dir():
            dataset_types.append(dire.name)

    collections = []
    # TODO store in cache
    config_data = {}
    with open('config.json', 'r') as config:
        config_data = json.load(config)

    print(config_data)

    for dt in dataset_types:
        datasets = []
        selected_index = 0
        index = 0
        for file in (up_one / 'corpora' / 'uploads' / dt).iterdir():
            if file.is_file():
                new_file = file.stat().st_ctime
                selected = False
                if dt in config_data['categories'] and config_data['categories'][dt]:
                    current = config_data['categories'][dt]['current']
                    if current:
                        current = current.strip()
                        if current == str(file.name):
                            selected = True
                            selected_index = index
                datasets.append(
                    {
                        'name': str(file.name),
                        'current': selected,
                        'new': new_file
                    })
                index += 1

        collections.append({'dset': datasets, 'type': dt})

    # TODO show languages and datasets
    # TODO login on server startup and keep session, use the same session on every request
    print(self_info)

    # return render_template('home.html', collections=collections, types=",".join(dataset_types))
    return render_template('home.html', collections=collections, types=",".join(dataset_types), self_info=self_info)

@main.route('/central')
def central_home():
    return render_template("central.html")

@main.route('/login')
def login_alt():
    return redirect(url_for('auth.login'))

@main.route('/signup')
def signup_alt():
    return redirect(url_for('auth.new_user'))

# TODO
# https://stackoverflow.com/a/51013358/8608146

@main.route('/files')
@jwt_required
def all_files():
    items = []
    for i in Path(current_app.config['UPLOAD_FOLDER']).iterdir():
        path = str(i)
        items.append({
            'date': i.split('.')[0][4:],
            'size': os.stat(path).st_size,
            'id': i
        })
    return render_template("files.html", items=items)  # , as_attachment=True)

@main.route('/logout')
def logout():
    return redirect('/')

@main.route('/dashboard')
def dashboard_home():
    # print('x' not in cache)
    running = cache.get('running_zip_tasks')
    print(running)

    session['x'] = (0, 1, 2, 3)
    print(session['x'])
    users = DB.get_users()
    for x in users:
        audiofiles = UserFileSystem(x['username']).get_audio_files()
        x['count'] = len(audiofiles)
    newlist = sorted(users, key=lambda k: k['count'], reverse=True)
    return render_template('dashboard.html', users=newlist)

@main.route('/manage/del', methods=['DELETE'])
def dashboard_settings_del_lang():
    langname = request.json['langcode']
    print(f"Warning: {langname} is scheduled for a deletion")
    dirpath = up_one / 'corpora' / 'uploads' / langname
    dest = up_one / 'corpora' / 'trash'
    try:
        os.makedirs(dest)
    except Exception as e:
        print(e)
    try:
        shutil.move(str(dirpath), str(dest))
    except:
        shutil.move(str(dirpath), str(dest) +
                    str(random.randint(100, 2832989)))
    return jsonify({'success': True, 'error': None})

@main.route('/manage/add', methods=['POST'])
def dashboard_settings_add_lang():
    langname = request.json['langname']
    langs = request.json['langs']
    langtype = request.json['type']
    if not langname or not langs:
        return jsonify({'success': False, 'error': 'Values cannot be empty please try again'}), 403

    langs = [x.strip() for x in langs.split(',')]
    if len(langs) < 1 and langtype == 'mixed':
        return jsonify({'success': False, 'error': 'Select single if need to have only one language, or a comma is missing please try again'}), 403

    # TODO do something with the langname

    dirname = langs[0]
    if len(langs) > 1:
        dirname = "-".join(['mix', *langs])

    dirpath = up_one / 'corpora' / 'uploads' / dirname

    try:
        os.makedirs(dirpath)
    except Exception as e:
        return jsonify({'success': False, 'error': str(e)}), 403
    print(langname, langs, dirname)

    return jsonify({'success': True})

@main.route('/manage/upload', methods=['POST'])
def dashboard_settings_upload():
    langcode = (request.form['langcode'])
    if not langcode:
        return jsonify({
            'status': 'failed',
            'msg': 'no langcode was specified'
        }), 403
    if 'file' not in request.files:
        # return redirect(request.url)
        return jsonify({'status': 'failed', 'msg': 'No file uploaded try again'}), 404
    file = request.files['file']
    if file.filename == '':
        return jsonify({'status': 'failed', 'msg': 'No file selected for uploading'}), 404
    if file and allowed_settings_file(file.filename):
        filename = secure_filename(file.filename)
        filepath = up_one / 'corpora' / 'uploads' / langcode / filename

        try:
            os.makedirs(filepath.parents[0])
        except Exception:
            pass

        file.save(filepath)
        size = os.stat(filepath).st_size
        # TODO shedule celery task to split new uploaded dataset

        print(filepath.absolute(), type(filepath.absolute()))

        task = save.split_corpora.apply_async(
            kwargs={
                "file_path": str(filepath.absolute()),
                "split_path": f'split-{langcode}',
                "langcode": langcode,
                "filename": filename,
            },
            queue="split_queue",
        )
        return jsonify({
            'status': 'ok',
            'msg': None,
            'path': filename,
            'size': size,
            'taskid': task.id
        })
    else:
        return jsonify({
            'status': 'failed',
            'msg': f'invalid file type {file.filename} ' + 'only ' + ", ".join(ALLOWED_SETTINGS_EXTENSIONS) + ' file types are allowed'
        }), 403

@main.route('/manage', methods=['GET', 'POST'])
def dashboard_settings():
    if request.method == 'GET':
        dataset_types = []
        # dataset_types = ['eng', 'hin', 'tel', 'mix-hin-tel']
        for dire in (up_one / 'corpora' / 'uploads').iterdir():
            if dire.is_dir():
                dataset_types.append(dire.name)

        collections = []
        # TODO store in cache
        config_data = {}
        with open('config.json', 'r') as config:
            config_data = json.load(config)

        print(config_data)

        for dt in dataset_types:
            datasets = []
            selected_index = 0
            index = 0
            for file in (up_one / 'corpora' / 'uploads' / dt).iterdir():
                if file.is_file():
                    new_file = file.stat().st_ctime
                    selected = False
                    if dt in config_data['categories'] and config_data['categories'][dt]:
                        current = config_data['categories'][dt]['current']
                        if current:
                            current = current.strip()
                            if current == str(file.name):
                                selected = True
                                selected_index = index
                    datasets.append(
                        {
                            'name': str(file.name),
                            'current': selected,
                            'new': new_file
                        })
                    index += 1

            collections.append({'dset': datasets, 'type': dt})

        return render_template('settings.html', collections=collections, types=",".join(dataset_types))
    else:
        try:
            selected = request.json['selected']
            langcode = request.json['langcode']
        except:
            return jsonify({
                'success': False,
                'error': 'One of selected and langcode was not provided'
            }), 403
        config_data = {}
        with open('config.json', 'r') as config:
            config_data = json.load(config)

        if 'categories' not in config_data:
            config_data['categories'] = {}

        # for k in config_data['categories'].keys():
        #     config_data['categories'][k]['current'] = None

        config_data['categories'][langcode] = {
            'current': selected
        }

        with open('config.json', 'w+') as config:
            json.dump(config_data, config)

        print(config_data, selected, langcode)

        return jsonify({'success': True})

@main.route('/manage/<string:langcat>', methods=['GET', 'POST'])
def dashboard_settings_lang(langcat: str):
    # TODO check if a valid lang category
    if request.method == 'GET':
        config_data = {}
        with open('config.json', 'r') as config:
            config_data = json.load(config)

        datasets = []
        default_index = None
        selected_index = default_index
        index = 0
        for file in (up_one / 'corpora' / 'uploads' / langcat).iterdir():
            if file.is_file():
                selected = False
                if langcat in config_data['categories'] and config_data['categories'][langcat]:
                    current = config_data['categories'][langcat]['current']
                    if current:
                        current = current.strip()
                        if current == str(file.name):
                            selected = True
                            selected_index = index
                index += 1
                datasets.append({'name': str(file.name), 'current': selected})

        selected = None
        if len(datasets) > 0 and selected_index is not None:
            selected = datasets[selected_index]
            selected['index'] = selected_index

        print(datasets)
        print(selected)
        return render_template(
            'lang-settings.html',
            datasets=datasets,
            langcode=langcat,
            selected=selected
        )
    else:
        langcode = langcat
        if 'file' not in request.files:
            # return redirect(request.url)
            return jsonify({'status': 'failed', 'msg': 'No file uploaded try again'}), 404
        file = request.files['file']
        if file.filename == '':
            return jsonify({'status': 'failed', 'msg': 'No file selected for uploading'}), 404
        if file and allowed_settings_file(file.filename):
            filename = secure_filename(file.filename)
            filepath = up_one / 'corpora' / 'uploads' / langcode / filename

            try:
                os.makedirs(filepath.parents[0])
            except Exception as e:
                print(e)

            file.save(filepath)
            size = os.stat(filepath).st_size
            return jsonify({
                'status': 'ok',
                'msg': None, 'path': filename, 'size': size
            })
        else:
            return jsonify({
                'status': 'failed',
                'msg': f'invalid file type {file.filename} ' + 'only ' + ", ".join(ALLOWED_SETTINGS_EXTENSIONS) + ' file types are allowed'
            }), 403

def is_jsonable(x):
    try:
        json.dumps(x)
        return True
    except (TypeError, OverflowError):
        return False

@main.route('/_cache/clear')
def clear_cache():
    cache.clear()
    return jsonify(read_cache())

# TODO cahce experimentaion
# Use cache to cache file results (?)
@main.route('/_cache', methods=['GET', 'POST'])
def export_cache():
    if request.method == 'GET':
        return jsonify(read_cache())
    else:
        try:
            print('try')
            session['x']
        except:
            print('except')
            session['x'] = {}
        cache.set('time', datetime.datetime.now())
        cache.set(f'time-{random.randint(1, 100000)}', datetime.datetime.now())
        cache_dat = read_cache()
        ret = {}
        print(session['x'], cache_dat)
        if (len(session['x'].keys()) > len(cache_dat.keys())):
            ret['changes'] = {'x': session['x'], 'new': cache_dat}

        session['x'] = cache_dat
        ret['cache'] = cache_dat
        ret['x'] = session['x']
        return jsonify(ret)

def read_cache():
    k_prefix = cache.cache.key_prefix
    keys = cache.cache._write_client.keys(k_prefix + '*')
    keys = [k.decode('utf8') for k in keys]
    keys = [k.replace(k_prefix, '') for k in keys]
    values = cache.get_many(*keys)
    ret = {}
    for k, v in zip(keys, values):
        if is_jsonable(v):
            ret[k] = v
        else:
            ret[k] = str(v)
    return ret

@main.route('/users')
def users_home():
    users = DB.get_users()
    for x in users:
        audiofiles = UserFileSystem(x['username']).get_audio_files()
        x['count'] = len(audiofiles)
    newlist = sorted(users, key=lambda k: k['count'], reverse=True)
    return render_template('users.html', users=newlist)

@main.route('/user/<string:username>')
def user_page(username: str):
    userinfo = DB.get_user(username)
    sens = UserFileSystem(username).get_sentences()
    return render_template('user.html', user=userinfo, sentences=sens)

@main.route('/search', methods=['POST'])
def search():
    print(request.json)

    return jsonify({})

@main.route('/userfile/<string:filename>')
def user_file(filename: str):
    # filename is username_sid.wav
    # username can be x_yayya_xx
    username = "_".join(filename.split('_')[:-1])
    dirname = (up_one / 'data')
    print("User name is ", username, secure_filename(username))
    for d in dirname.iterdir():
        d = os.path.basename(d)
        safeusername = secure_filename(str(d))
        if safeusername == username:
            print(safeusername)
            dirname = (up_one / 'data' / d)
            break
    return send_from_directory(str(dirname), filename)

@main.route('/upload', methods=['POST'])
@jwt_required
def upload_file():
    # TODO in the app when logged out
    # 401 code is sent so this method is not called
    # Show failed message
    # user = (User(session['user'][0]).attach_DB(DB))
    print(session['user'])  # tuple (username, age, gender)
    print("-"*(200))
    sys.stdout.flush()

    if request.method == 'POST':
        # check if the post request has the file part
        # print(request)
        if 'file' not in request.files:
            # return redirect(request.url)
            return jsonify({'status': 'failed', 'msg': 'No file uploaded try again'})
        file = request.files['file']
        if file.filename == '':
            return jsonify({'status': 'failed', 'msg': 'No file selected for uploading'})
        if file and allowed_file(file.filename):
            filename = secure_filename(file.filename)
            filepath = Path(current_app.config['UPLOAD_FOLDER']
                            ) / session['user'][0] / filename
            try:
                os.makedirs(filepath.parents[0])
            except Exception as e:
                print(e)

            file.save(filepath)
            size = os.stat(filepath).st_size
            return jsonify({
                'status': 'ok',
                'msg': None, 'path': filename, 'size': size
            })
        else:
            return jsonify({'status': 'failed', 'msg': f'invalid file type {file.filename}'})

# TODO @jwt_required
@main.route('/data/<string:langcat>/<string:dbname>')
def random_corpa(langcat: str, dbname: str):
    results = SENTENCEDB.execute(GET_TOTAL_CORPA_LEN_FORDB, (langcat, dbname))
    count = results[0]['COUNT(*)']
    start_x = SENTENCEDB.execute(
        GET_STARTING_INDEX_SENTENCES_BY_DATASET, (langcat, dbname))
    start_idx = start_x[0]['id']
    start_x = random.randint(1, count) + start_idx
    results = SENTENCEDB.execute(
        GET_RANDOM_CORPA_AFTER, (langcat, dbname, start_x, 100))
    return jsonify(results)

@main.route('/data/list')
def public_list():
    dataset_types = []
    # dataset_types = ['eng', 'hin', 'tel', 'mix-hin-tel']
    for dire in (up_one / 'corpora' / 'uploads').iterdir():
        if dire.is_dir():
            dataset_types.append(dire.name)

    collections = []
    # TODO store in cache
    config_data = {}
    with open('config.json', 'r') as config:
        config_data = json.load(config)

    print(config_data)

    for dt in dataset_types:
        datasets = []
        selected_index = 0
        index = 0
        for file in (up_one / 'corpora' / 'uploads' / dt).iterdir():
            if file.is_file():
                new_file = file.stat().st_ctime
                selected = False
                if dt in config_data['categories'] and config_data['categories'][dt]:
                    current = config_data['categories'][dt]['current']
                    if current:
                        current = current.strip()
                        if current == str(file.name):
                            selected = True
                            selected_index = index
                datasets.append(
                    {
                        'name': str(file.name),
                        'current': selected,
                        'new': new_file
                    })
                index += 1

        collections.append({'dset': datasets, 'type': dt})

    return jsonify(collections=collections, dataset_types=dataset_types)

@main.route('/skipped', methods=['POST'])
@jwt_required
def handle_skip():

    sentence_ids = request.json['ids']
    username = session['user'][0]

    with open('corpora/split/skipped.txt', 'a+') as skipper:
        for sentence_id in sentence_ids:
            skipper.write(f"{sentence_id}||{username}\n")

    return 'ok', 200

@main.route('/files/<path:filename>', methods=['GET', 'DELETE', 'PUT'])
@jwt_required
def download_file(filename):
    print(session['user'])
    if request.method == 'GET':
        print("accessing file: " + filename, time.asctime())
        print((request.headers))
        return send_from_directory(current_app.config['UPLOAD_FOLDER'],
                                   filename)  # , as_attachment=True)
    elif request.method == 'DELETE':
        # TODO
        # check if client has the ownership
        print("Delete", filename)
        try:
            os.remove(os.path.join(
                current_app.config['UPLOAD_FOLDER'], filename))
        except FileNotFoundError as e:
            return jsonify({'status': 'failed', 'msg': str(e)})
        return jsonify({'status': 'ok'})
    elif request.method == 'PUT':
        # https://pythonise.com/series/learning-flask/flask-http-methods
        return jsonify({'status': 'ok'})

# file upload
ALLOWED_EXTENSIONS = set(['wav', 'mp3', 'ogg', 'webm', 'aac'])

# Settings dashboard file upload
ALLOWED_SETTINGS_EXTENSIONS = set(['txt', 'csv'])

def allowed_file(filename):
    return '.' in filename and filename.rsplit('.', 1)[-1].lower() in ALLOWED_EXTENSIONS

def allowed_settings_file(filename):
    return '.' in filename and filename.rsplit('.', 1)[-1].lower() in ALLOWED_SETTINGS_EXTENSIONS