from argparse import ArgumentParser
from config_parser import get_config
import os
import yaml
import matplotlib.pyplot as plt
import time

import torch
from torch import nn, optim
import wandb
from typing import Callable, Tuple
from utils.loss import LabelSmoothingLoss
from utils.opt import get_optimizer, get_adversarial_optimizer
from utils.scheduler import WarmUpLR, get_scheduler
from utils.trainer import train, evaluate, evaluate_sliding_window, train_single_batch
from utils.dataset import get_loader, get_noisy_loader
from utils.misc import seed_everything, count_params, get_model, calc_step, log
#from data2vec.data2vec_utils.trainer import train_single_batch
from utils.misc import log, save_model
from torch.utils.data import DataLoader
from data2vec.masking import AudioMaskingGenerator
from models.Data2Vec import Data2Vec
from tqdm import tqdm
import copy
from collections import OrderedDict
import data2vec.data2vec_utils.trainer
import utils.trainer

def adv_pretrain(config_kwt, config_d2v, k, alpha):
    """Adverserial pretraining with noisy data.

    Args:
        config_kwt (dict): Keyword transformer config for noise type prediction
        config_d2v (dict): Data2vec config for masked prediction (regression)
        k (int): Lower K common layers that the two models share.
    """

    ######################################
    # save hyperparameters for current run
    ######################################

    config_kwt["exp"]["save_dir"] = os.path.join(config_kwt["exp"]["exp_dir"], config_kwt["exp"]["exp_name"])
    os.makedirs(config_kwt["exp"]["save_dir"], exist_ok=True)

    config_d2v["exp"]["save_dir"] = os.path.join(config_d2v["exp"]["exp_name"])
    os.makedirs(config_d2v["exp"]["save_dir"], exist_ok=True)

    config_str = yaml.dump(config_kwt)
    print("Using settings:\n", config_str)

    with open(os.path.join(config_kwt["exp"]["save_dir"], "settings.txt"), "w+") as f:
        f.write(config_str)

    ######################################
    # data loaders
    ######################################

    # training
    # adversarial
    print("Loading adversarial training dataset...")
    with open(config_kwt["train_list_file"], "r") as f:
        train_lista = f.read().rstrip().split("\n")
    trainloadera = get_noisy_loader(train_lista, config_kwt, train=True)

    # friendly
    print("Loading friendly training dataset...")
    with open(config_d2v["train_list_file"], "r") as f:
        train_listf = f.read().rstrip().split("\n")
    trainloaderf = get_loader(train_listf, config_d2v, train=True)

    # validation
    # adversarial
    print("Loading adversarial validation dataset...")
    with open(config_kwt["val_list_file"], "r") as f:
        val_lista = f.read().rstrip().split("\n")
    valloadera = get_noisy_loader(val_lista, config_kwt, train=False)

    #friendly
    print("Loading friendly validation dataset...")
    with open(config_d2v["val_list_file"], "r") as f:
        val_listf = f.read().rstrip().split("\n")
    valloaderf = get_loader(val_listf, config_d2v, train=False)

    mask_generator = AudioMaskingGenerator(mask_prob=config_d2v["hparams"]["model"]["mask_prob"],
                                           mask_length=config_d2v["hparams"]["model"]["mask_length"],
                                           attention_mask=None,
                                           min_masks=config_d2v["hparams"]["model"]["min_masks"])

    ######################################
    # models
    ######################################

    # KWT model
    model_kwt = get_model(config_kwt["hparams"]["model"])
    model_kwt = model_kwt.to(config_kwt["hparams"]["device"])

    model_kwt_copy = copy.deepcopy(model_kwt)

    # data2vec model
    model_d2v = Data2Vec(encoder=model_kwt_copy,
                        modality=config_d2v["modality"],
                        model_embed_dim=config_d2v["hparams"]["model"]["dim"],
                        ema_decay=config_d2v["hparams"]["model"]["ema_decay"],
                        ema_end_decay=config_d2v["hparams"]["model"]["ema_end_decay"],
                        ema_anneal_end_step=config_d2v["hparams"]["model"]["ema_anneal_end_step"],
                        average_top_k_layers=config_d2v["hparams"]["model"]["average_top_k_layers"],
                        normalize_targets=config_d2v["hparams"]["model"]["normalize_targets"])
    model_d2v= model_d2v.to(config_d2v["hparams"]["device"])

    criterion_kwt = nn.CrossEntropyLoss()
    criterion_d2v = nn.MSELoss(reduction="none")

    parameters_kwt = model_kwt.parameters()
    parameters_d2v = model_d2v.parameters()

    # optimizer for KWT
    optimizer_kwt = get_adversarial_optimizer(model_kwt, config_kwt["hparams"]["optimizer"], k, alpha)

    # optimizer for data2vec
    optimizer_d2v = optim.Adam(parameters_d2v, lr=config_d2v["hparams"]["optimizer"]["opt_kwargs"]["lr"],
                           betas=config_d2v["hparams"]["optimizer"]["opt_kwargs"]["betas"],
                           eps=config_d2v["hparams"]["optimizer"]["opt_kwargs"]["eps"],
                           weight_decay=config_d2v["hparams"]["optimizer"]["opt_kwargs"]["weight_decay"])

    #for group in optimizer_kwt.param_groups:
    #    print(group['lr'])

    #return

    # Learning rate scheduler for data2vec
    epochs = config_d2v["hparams"]["n_epochs"]
    steps_per_epoch = len(trainloaderf)
    lr_scheduler = optim.lr_scheduler.OneCycleLR(
        optimizer_d2v,
        max_lr=config_d2v["hparams"]["optimizer"]["opt_kwargs"]["lr"],
        epochs=epochs,
        steps_per_epoch=steps_per_epoch,
        anneal_strategy="cos")
    schedulers_d2v = {"scheduler": lr_scheduler,
                  "warmup": 0}

    # Learning rate scheduler for KWT
    schedulers_kwt = {
        "warmup": None,
        "scheduler": None
    }

    # Setting up the learning rate scheduler for data2vec and KWT
    if config_d2v["hparams"]["scheduler"]["n_warmup"]:
        schedulers_d2v["warmup"] = WarmUpLR(optimizer_kwt, total_iters=len(trainloaderf) * config_d2v["hparams"]["scheduler"]["n_warmup"])

    if config_d2v["hparams"]["scheduler"]["scheduler_type"] is not None:
        total_iters = len(trainloaderf) * max(1, (config_d2v["hparams"]["scheduler"]["max_epochs"] - config_d2v["hparams"]["scheduler"]["n_warmup"]))
        schedulers_d2v["scheduler"] = get_scheduler(optimizer_kwt, config_d2v["hparams"]["scheduler"]["scheduler_type"], total_iters)

    if config_kwt["hparams"]["scheduler"]["n_warmup"]:
        schedulers_kwt["warmup"] = WarmUpLR(optimizer_kwt, total_iters=len(trainloadera) * config_kwt["hparams"]["scheduler"]["n_warmup"])

    if config_kwt["hparams"]["scheduler"]["scheduler_type"] is not None:
        total_iters = len(trainloadera) * max(1, (config_kwt["hparams"]["scheduler"]["max_epochs"] - config_kwt["hparams"]["scheduler"]["n_warmup"]))
        schedulers_kwt["scheduler"] = get_scheduler(optimizer_kwt, config_kwt["hparams"]["scheduler"]["scheduler_type"], total_iters)

    # Saving directory for the data2vec model
    config_d2v["exp"]["save_dir"] = os.path.join(config_d2v["exp"]["exp_dir"], config_d2v["exp"]["exp_name"])
    os.makedirs(config_d2v["exp"]["save_dir"], exist_ok=True)

    ######################################
    # train models
    ######################################

    step = 0
    best_acc = 0.0
    device = config_d2v["hparams"]["device"]
    log_file = os.path.join(config_d2v["exp"]["exp_dir"], "training_log.txt")
    best_avg_loss = 0.0
    n_batches = len(trainloaderf)

    for epoch in range(config_d2v["hparams"]["n_epochs"]):
        t0 = time.time()
        running_loss_d2v = 0.0
        running_target_var_d2v = 0.0
        running_prediction_var_d2v = 0.0
        running_loss_kwt = 0.0
        correct_kwt = 0

        for (dataf, targetsf), (dataa, targetsa) in zip(trainloaderf, trainloadera):
            batch_size = dataf.size(dim=0)
            audio_length = dataf.size(dim=-1)

            ######################################
            # data2vec step - friendly step
            ######################################

            # masking
            mask = mask_generator(shape=(batch_size, audio_length)).to(device)
            mask = torch.cat([torch.zeros(batch_size, 1, device=mask.device), mask], dim=1).bool()

            # loading first K transformer layers of KWT
            kwt_partial_state_dict = load_partial_state_dict(model_kwt.state_dict(), k)
            model_d2v.load_state_dict(kwt_partial_state_dict, strict=False)

            # train single batch
            loss_d2v, target_var_d2v, prediction_var_d2v = data2vec.data2vec_utils.trainer.train_single_batch(model_d2v, dataf, mask, optimizer_d2v, criterion_d2v, device)
            model_d2v.ema_step()
            running_loss_d2v += loss_d2v
            running_target_var_d2v += target_var_d2v
            running_prediction_var_d2v += prediction_var_d2v

            # learning rate scheduler
            if schedulers_d2v["warmup"] is not None and epoch < config_d2v["hparams"]["scheduler"]["n_warmup"]:
                schedulers_d2v["warmup"].step()
            elif schedulers_d2v["scheduler"] is not None:
                schedulers_d2v["scheduler"].step()

            # logging data2vec step
            if not step % config_d2v["exp"]["log_freq"]:
                log_dict = {"epoch": epoch, "loss": loss_d2v, "lr": optimizer_d2v.param_groups[0]["lr"],
                            "target_var": target_var_d2v, "prediction_var": prediction_var_d2v}
                log(log_dict, step, config_d2v)

            ######################################
            # kwt step - adversarial step
            ######################################

            # loading first K layers of data2vec encoder
            d2v_partial_state_dict = load_partial_state_dict(model_d2v.state_dict(), k)
            model_kwt.load_state_dict(d2v_partial_state_dict, strict=False)

            # train single batch
            loss_kwt, corr_kwt = utils.trainer.train_single_batch(model_kwt, dataa, targetsa, optimizer_kwt, criterion_kwt, device)
            running_loss_kwt += loss_kwt
            correct_kwt += corr_kwt

            # logging KWT step
            if schedulers_kwt["warmup"] is not None and epoch < config_kwt["hparams"]["scheduler"]["n_warmup"]:
                schedulers_kwt["warmup"].step()
            elif schedulers_kwt["scheduler"] is not None:
                schedulers_kwt["scheduler"].step()

            if not step % config_kwt["exp"]["log_freq"]:
                log_dict = {"epoch": epoch, "loss": loss_kwt, "lr": optimizer_kwt.param_groups[0]["lr"]}

            step += 1

        #################################################
        # epoch complete - log, validation and save model
        #################################################

        # data2vec log, validation and save model
        log_dict = {"epoch": epoch, "time_per_epoch": time.time() - t0,
                    "avg_train_target_var": running_target_var_d2v / n_batches,
                    "avg_train_prediction_var": running_prediction_var_d2v / n_batches,
                    "avg_loss_per_ep": running_loss_d2v / len(trainloaderf.dataset)}
        log(log_dict, step, config_d2v)

        if not epoch % config_d2v["exp"]["val_freq"]:
            avg_val_loss, avg_val_target_var, avg_val_prediction_var = data2vec.data2vec_utils.trainer.evaluate(model_d2v, mask_generator, criterion_d2v,
                                                                                valloaderf, device)
            log_dict = {"epoch": epoch, "val_loss": avg_val_loss,
                        "avg_val_target_var": avg_val_target_var, "avg_val_prediction_var": avg_val_prediction_var}
            #log(log_dict, step, config_d2v)

            # save best validation checkpoint
            if avg_val_loss < best_avg_loss or epoch == config_d2v["exp"]["val_freq"]:
                best_avg_loss = avg_val_loss
                save_path = os.path.join(config_d2v["exp"]["save_dir"], "best.pth")
                save_model(epoch, avg_val_loss, save_path, model_d2v, optimizer_d2v, log_file)
                save_path = os.path.join(config_d2v["exp"]["save_dir"], "best_encoder.pth")
                save_model(epoch, avg_val_loss, save_path, model_d2v.encoder, optimizer_d2v, log_file)

        # kwt log, validation and save model
        log_dict = {"epoch": epoch, "time_per_epoch": time.time() - t0, "train_acc": correct_kwt/(len(trainloadera.dataset)), "avg_loss_per_ep": running_loss_kwt/len(trainloaderf)}
        log(log_dict, step, config_kwt)

        if not epoch % config_kwt["exp"]["val_freq"]:
            val_acc, avg_val_loss = utils.trainer.evaluate(model_kwt, criterion_kwt, valloadera, device)
            log_dict = {"epoch": epoch, "val_loss": avg_val_loss, "val_acc": val_acc}
            log(log_dict, step, config_kwt)

            # save best val ckpt
            if val_acc > best_acc:
                best_acc = val_acc
                save_path = os.path.join(config_kwt["exp"]["exp_dir"], "best.pth")
                save_model(epoch, val_acc, save_path, model_kwt, optimizer_kwt, log_file)

    # training complete

    # data2vec evaluation
    avg_val_loss, avg_val_target_var, avg_val_prediction_var = data2vec.data2vec_utils.trainer.evaluate(model_d2v, mask_generator, criterion_d2v, valloaderf,
                                                                        device)
    log_dict = {"epoch": epoch, "val_loss": avg_val_loss,
                "avg_val_target_var": avg_val_target_var, "avg_val_prediction_var": avg_val_prediction_var}

    log(log_dict, step, config_d2v)

    # data2vec save final checkpoint
    save_path = os.path.join(config_d2v["exp"]["exp_dir"], "last.pth")
    save_model(epoch, avg_val_loss, save_path, model_d2v, optimizer_d2v, log_file)
    save_path = os.path.join(config_d2v["exp"]["exp_dir"], "last_encoder.pth")
    save_model(epoch, avg_val_loss, save_path, model_d2v.encoder, optimizer_d2v, log_file)

    # kwt evaluation

    val_acc, avg_val_loss = evaluate(model_kwt, criterion_kwt, valloadera, device)
    log_dict = {"epoch": epoch, "val_loss": avg_val_loss, "val_acc": val_acc}
    log(log_dict, step, config_kwt)

    # kwt save final checkpoint
    save_path = os.path.join(config_kwt["exp"]["exp_dir"], "last.pth")
    save_model(epoch, val_acc, save_path, model_kwt, optimizer_kwt, log_file)

def load_partial_state_dict(state_dict, K):
    """_summary_

    Args:
        state_dict (_type_): state_dict to load
        K (_type_): first K layers to load
    """
    before_transformer = True
    custom_dict = OrderedDict()
    for param in state_dict:
        if param.split('.')[0] == "transformer":
            before_transformer = False
            layer_num = int(param.split('.')[2])
            if layer_num >= K:
                continue
            custom_dict[param] = state_dict[param]
        #elif before_transformer:
        #    custom_dict[param] = state_dict[param]
    #print("###################################")
    #for param in custom_dict:
    #    print(param, '\t\t', custom_dict[param].size())
    return custom_dict

def main(args):
    config_kwt = get_config(args.confk)
    config_data2vec = get_config(args.confd)
    seed_everything(config_kwt['hparams']['seed'])
    alpha = 1
    adv_pretrain(config_kwt, config_data2vec, args.k, alpha)

if __name__ == "__main__":
    parser = ArgumentParser("Adversarial pretraining")
    parser.add_argument("--confk", type=str, required=True, help="Path to config.yaml file for KWT.")
    parser.add_argument("--confd", type=str, required=True, help="Path to config.yaml file for data2vec.")
    parser.add_argument("--k", type=int, required=True, help="First K transformer layers to update")
    args = parser.parse_args()
    main(args)


import xmltodict
import json

# Loading and parsing the xml file.
with open (r'q1.xml', "r") as xml_file:
    xml_data = xml_file.read()
    print(xml_data)

# Convert xml to json
json_data = json.dumps(xmltodict.parse(xml_data), indent=4)

print(json_data)

with open("output.json", "w") as json_file:
    json_file.write(json_data)


#grid = [[0 for j in range(9)] for i in range(9)]
import time

def not_complete(board):
    for i in board:
        if 0 in i:
            return True
    return False

#will return True if the inputted number is not in the row
def check_row(board, row_num, num):
    if num in board[row_num]:
        return False
    else:
        return True

#will return True if inputted number is not in the column
def check_col(board, col_num, num):
    not_in_col = True
    for i in range(9):
        if num == board[i][col_num]:
            not_in_col = False
    return not_in_col

def get_box_values(board, row, col):
    values = []
    if row<3:
        if col<3:
            for i in range(3):
                values.append(board[i][0:3])
        elif col<6:
            for i in range(3):
                values.append(board[i][3:6])
        elif col<9:
            for i in range(3):
                values.append(board[i][6:9])
    elif row<6:
        if col<3:
            for i in range(3,6):
                values.append(board[i][0:3])
        elif col<6:
            for i in range(3,6):
                values.append(board[i][3:6])
        elif col<9:
            for i in range(3,6):
                values.append(board[i][6:9])
    elif row<9:
        if col<3:
            for i in range(6,9):
                values.append(board[i][0:3])
        elif col<6:
            for i in range(6,9):
                values.append(board[i][3:6])
        elif col<9:
            for i in range(6,9):
                values.append(board[i][6:9])
    return values

#returns True if inputted number not in box
def check_box(board, row, col, num):
    box_values = get_box_values(board, row, col)

    for row in box_values:
        if num in row:

            return False

    return True

def find_possibilities(board,pos):
    possibilities = []
    for i in range(1,10):
        if check_box(board, pos[0], pos[1], i) and check_row(board, pos[0], i) and check_col(board, pos[1], i):
            possibilities.append(i)

    return possibilities

def solver(board):
    if not not_complete(board):
        return board
    else:
        for x in range(9):
            for y in range(9):
                if board[x][y] == 0:
                    i = x
                    j = y
                    break
            else:
                continue
            break

        possibilities = find_possibilities(board, [i,j])
        for choice in possibilities:
            board[i][j] = choice

            value = solver(board)
            if value != None:
                return board
        board[i][j] = 0


# Definition for a binary tree node.
# class TreeNode:
#     def __init__(self, val=0, left=None, right=None):
#         self.val = val
#         self.left = left
#         self.right = right
class Solution:
    def flatten(self, root: Optional[TreeNode]) -> None:
        """
        Do not return anything, modify root in-place instead.
        """
        if not root:
            return None

        left = self.flatten(root.left)
        right = self.flatten(root.right)

        root.left = None
        ptr = root

        if (left):
            ptr.right = left
            while (ptr.right):
                ptr = ptr.right
        ptr.right = right

        return root


import logging
import threading
import numpy as np
import pandas as pd
from timeit import default_timer as timer
from datetime import datetime, timedelta, timezone
from time import sleep
import time
import matplotlib.pyplot as mpl
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import ShuffleSplit
from sklearn.metrics import make_scorer, r2_score
from sklearn.model_selection import cross_val_score
from sklearn.neighbors import KNeighborsRegressor
from utils.history import Historical_Caching
import warnings

warnings.filterwarnings('ignore')
logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO)
logger = logging.getLogger(__name__)

# Gives a list of timestamps from the start date to the end date
#
# startDate:     The start date as a string xxxx-xx-xx
# endDate:       The end date as a string year-month-day
# period:		 'minute', 'daily', 'weekly', or 'monthly'
# weekends:      True if weekends should be included; false otherwise
# return:        A numpy array of timestamps
def DateRange(startDate, endDate, period='minute', weekends=True):
    # The start and end date
    sd = datetime.fromtimestamp(startDate)
    ed = datetime.fromtimestamp(endDate)
    # print(startDate, endDate)
    # Invalid start and end dates
    if (sd > ed):
        raise ValueError("The start date cannot be later than the end date.")
    # One time period is a day
    if (period == 'minute'):
        prd = timedelta(minutes=1)
    if (period == 'daily'):
        prd = timedelta(1)
    # One prediction per week
    if (period == 'weekly'):
        prd = timedelta(7)
    # one prediction every 30 days ("month")
    if (period == 'monthly'):
        prd = timedelta(30)
    # The final list of timestamp data
    dates = []
    cd = sd
    while (cd <= ed):
        # If weekdays are included or it's a weekday append the current ts
        if (weekends or (cd.date().weekday() != 5 and cd.date().weekday() != 6)):
            dates.append(cd.timestamp())
        # Onto the next period
        cd = cd + prd
    # print(np.array(dates))
    return np.array(dates)

# Given a date, returns the previous day
#
# startDate:     The start date as a datetime object
# weekends:      True if weekends should counted; false otherwise
def DatePrevDay(startDate):
    # One day
    day = timedelta(minutes=1)
    cd = startDate - day
    return cd

# Load data from the CSV file. Note: Some systems are unable
# to give timestamps for dates before 1970. This function may
# fail on such systems.
#
# path:      The path to the file
# return:    A data frame with the parsed timestamps
def ParseData(path):
    # Read the csv file into a dataframe
    df = None
    while df is None:
        try:
            sleep(0.5)
            df = pd.read_csv(path)
        except Exception as e:
            sleep(0.5)
            pass

    df['Timestamp'] = df['Date']
    # Remove any unused columns (axis = 1 specifies fields are columns)
    df = df.drop('Date', axis=1)
    # df = df.iloc[::-1]  # CHECK THIS
    return df

# Given dataframe from ParseData
# plot it to the screen
#
# df:        Dataframe returned from
# p:         The position of the predicted data points
def PlotData(df, p=None):
    if (p is None):
        p = np.array([])
    # Timestamp data
    ts = df.Timestamp.values
    # Number of x tick marks
    nTicks = 10
    # Left most x value
    s = np.min(ts)
    # Right most x value
    e = np.max(ts)
    # Total range of x values
    r = e - s
    # Add some buffer on both sides
    s -= r / 5
    e += r / 5
    # These will be the tick locations on the x axis
    tickMarks = np.arange(s, e, (e - s) / nTicks)
    # Convert timestamps to strings
    strTs = [datetime.fromtimestamp(i).strftime('%Y-%m-%d %H:%M:%S') for i in tickMarks]
    mpl.figure()
    # Plots of the high and low values for the day
    mpl.plot(ts, df.High.values, color='#727272', linewidth=1.618, label='Actual')
    # Predicted data was also provided
    if (len(p) > 0):
        mpl.plot(ts[p], df.High.values[p], color='#7294AA', linewidth=1.618, label='Predicted')
    # Set the tick marks
    mpl.xticks(tickMarks, strTs, rotation='vertical')
    # Set y-axis label
    mpl.ylabel('Crypto Price (USD)')
    # Add the label in the upper left
    mpl.legend(loc='upper left')
    mpl.show()

# A class that predicts stock prices based on historical stock data
class Predictor:
    # Constructor
    # nPrevDays:     The number of past days to include
    #               in a sample.
    # rmodel:        The regressor model to use (sklearn)
    # nPastDays:     The number of past days in each feature
    # scaler:        The scaler object used to scale the data (sklearn)
    def __init__(self, rmodel, nPastDays, scaler=StandardScaler()):
        self.npd = nPastDays
        self.R = rmodel
        self.S = scaler
        self.D = None
        self.D_orig = None
        self.DTS = None
        self.A = None
        self.y = None
        self.targCols = None

    # Extracts features from stock market data
    #
    # D:         A dataframe from ParseData
    # ret:       The data matrix of samples
    def _ExtractFeat(self, D):
        # One row per day of stock data
        m = D.shape[0]
        # Open, High, Low, and Close for past n days + timestamp and volume
        n = self._GetNumFeatures()
        B = np.zeros([m, n])
        # Preserve order of spreadsheet
        for i in range(m - 1, -1, -1):
            self._GetSample(B[i], i, D)
        # Return the internal numpy array
        return B

    # Extracts the target values from stock market data
    #
    # D:         A dataframe from ParseData
    # ret:       The data matrix of targets and the

    def _ExtractTarg(self, D):
        # Timestamp column is not predicted
        tmp = D.drop('Timestamp', axis=1)
        # Return the internal numpy array
        return tmp.values, tmp.columns

    # Get the number of features in the data matrix
    #
    # n:         The number of previous days to include
    #           self.npd is  used if n is None
    # ret:       The number of features in the data matrix
    def _GetNumFeatures(self, n=None):
        if (n is None):
            n = self.npd
        return n * 7 + 1

    # Get the sample for a specific row in the dataframe.
    # A sample consists of the current timestamp and the data from
    # the past n rows of the dataframe
    #
    # r:         The array to fill with data
    # i:         The index of the row for which to build a sample
    # df:        The dataframe to use
    # return;    r
    def _GetSample(self, r, i, df):
        # First value is the timestamp
        r[0] = df['Timestamp'].values[i]
        # The number of columns in df
        n = df.shape[1]
        # The last valid index
        lim = df.shape[0]
        # Each sample contains the past n days of stock data; for non-existing data
        # repeat last available sample
        # Format of row:
        # Timestamp Volume Open[i] High[i] ... Open[i-1] High[i-1]... etc
        for j in range(0, self.npd):
            # Subsequent rows contain older data in the spreadsheet
            ind = i + j + 1
            # If there is no older data, duplicate the oldest available values
            if (ind >= lim):
                ind = lim - 1
            # Add all columns from row[ind]
            for k, c in enumerate(df.columns):
                # + 1 is needed as timestamp is at index 0
                r[k + 1 + n * j] = df[c].values[ind]
        return r

    # Attempts to learn the stock market data
    # given a dataframe taken from ParseData
    #
    # D:         A dataframe from ParseData
    def Learn(self, D):
        # Keep track of the currently learned data
        self.D = D.copy()
        self.D_orig = D.copy()
        # self.S = StandardScaler(with_mean=True, with_std=True)
        # Keep track of old timestamps for indexing
        self.DTS = np.asarray(self.D.Timestamp.values)
        # Scale the data
        self.S.fit(self.D)
        self.D[self.D.columns] = self.S.transform(self.D)
        # Get features from the data frame
        self.A = self._ExtractFeat(self.D)
        # Get the target values and their corresponding column names
        self.y, self.targCols = self._ExtractTarg(self.D)
        # Create the regressor model and fit it
        self.R.fit(self.A, self.y)
        return True

    # Predicts values for each row of the dataframe. Can be used to
    # estimate performance of the model
    #
    # df:            The dataframe for which to make prediction
    # return:        A dataframe containing the predictions
    def PredictDF(self, df):
        # Make a local copy to prevent modifying df
        D = df.copy()
        # Scale the input data like the training data
        D[D.columns] = self.S.transform(D)
        # Get features
        A = self._ExtractFeat(D)
        # Construct a dataframe to contain the predictions
        # Column order was saved earlier
        P = pd.DataFrame(index=range(A.shape[0]), columns=self.targCols)
        # Perform prediction
        P[P.columns] = self.R.predict(A)
        # Add the timestamp (already scaled from above)
        P['Timestamp'] = D['Timestamp'].values
        # Scale the data back to original range
        P[P.columns] = self.S.inverse_transform(P)
        return P

    # Predict the stock price during a specified time
    #
    # startDate:     The start date as a string in yyyy-mm-dd format
    # endDate:       The end date as a string yyyy-mm-dd format
    # period:		 'daily', 'weekly', or 'monthly' for the time period
    #				 between predictions
    # return:        A dataframe containing the predictions or
    def PredictDate(self, startDate, endDate, period='minute'):
        # Create the range of timestamps and reverse them
        ts = DateRange(startDate, endDate, period)[::-1]
        m = ts.shape[0]
        # Prediction is based on data prior to start date
        # Get timestamp of previous day
        prevts_ = datetime.fromtimestamp(ts[-1]) - timedelta(minutes=1)
        prevts = np.asarray(prevts_.timestamp())
        # Test if there is enough data to continue
        try:
            ind = np.where(self.DTS <= prevts)[0][0]
        except IndexError:
            logger.info('Safety ON')
            ind = 0
            pass
        # There is enough data to perform prediction; allocate new data frame
        P = pd.DataFrame(np.zeros((self.D.shape[0], self.D.shape[1])), index=range(self.D.shape[0]), columns=self.D.columns)
        # Add in the timestamp column so that it can be scaled properly
        P.loc[int(m):int(self.D.shape[0]), 'Timestamp'] = self.D.loc[0:(int(self.D.shape[0] - m)), 'Timestamp']
        P.loc[0:int(m - 1), 'Timestamp'] = ts
        for i in range(self.D.shape[0] - m):
            # If the current index does not exist, repeat the last valid data
            curInd = ind + i
            if (curInd >= self.D.shape[0]):
                curInd = curInd - 1
            # Copy over the past data (already scaled)
            P.iloc[int(m + i)] = self.D_orig.xs(int(curInd))
        # for i in range(len(P)):
        #     print(datetime.datetime.fromtimestamp(P.loc[i, 'Timestamp']))
        # Scale the timestamp (other fields are 0)
        self.S.fit(P)
        P[P.columns] = self.S.transform(P)
        P = P[0:int(m * 2)]
        # B is to be the data matrix of features
        B = np.zeros((1, self._GetNumFeatures()))
        # Add extra last entries for past existing data
        # Loop until end date is reached
        # print(P)
        for i in range(m - 1, -1, -1):
            # Create one sample
            self._GetSample(B[0], i, P)
            # Predict the row of the dataframe and save it
            pred = self.R.predict(B).ravel()
            for j, k in zip(self.targCols, pred):
                P.at[i, j] = k
        # Discard extra rows needed for prediction
        # Scale the dataframe back to the original range
        P[P.columns] = self.S.inverse_transform(P)

        '''for i in range(len(P)):
            print(datetime.fromtimestamp(P.loc[i, 'Timestamp']))
        print(P)'''

        '''j = 0
        for i in P.Timestamp:
            print(dt.fromtimestamp(i))
            j += 1
            if j > 10:
                break'''

        # PlotData(P)
        P = P[0:m]
        return P

    # Test the predictors performance and
    # displays results to the screen
    #
    # D:             The dataframe for which to make prediction
    def Performance(self, df=None):
        # If no dataframe is provided, use the currently learned one
        if df is None:
            D = self.D.copy()
        else:
            self.S.fit(df)
            D = self.S.transform(df)
        # Get features from the data frame
        A = self._ExtractFeat(D)
        # Get the target values and their corresponding column names
        y, _ = self._ExtractTarg(D)
        # Begin cross validation
        ss = ShuffleSplit(n_splits=1, test_size=0.1, train_size=0.9, random_state=0)

        for trn, tst in ss.split(A):
            s2 = cross_val_score(self.R, A[tst], y[tst], cv=5, scoring=make_scorer(r2_score), n_jobs=-1)

        if len(s2) > 1:
            return s2.mean()
        elif len(s2) == 1:
            logger.info(str(s2))
            return s2
        else:
            return 0

class ML_Calculus:

    def __init__(self, ws_bmex, rest, instrument, history_count, per_pred, API_key, API_secret):
        self.client = rest
        self.instrument_bmex = instrument
        self.API_key_bmex = API_key
        self.API_secret_bmex = API_secret
        self.periods_pred = per_pred - 1
        self.p_verdict = 0
        self.D = None
        self.ready = False
        self.history = Historical_Caching(ws_bmex, rest, instrument, history_count)
        self.thread = threading.Thread(target=self.Engine)
        self.R = KNeighborsRegressor(n_neighbors=10, weights='distance', algorithm='auto', leaf_size=25, n_jobs=-1)
        self.sp_Classic = Predictor(rmodel=self.R, nPastDays=50)
        self.logger = logging.getLogger(__name__)

    def Main(self, args):
        if (len(args) != 3 and len(args) != 4):
            return
        # Test if file exists
        try:
            open(args[0])
        except Exception as e:
            logger.error('Error opening args: ' + args[0])
            logger.error(str(e))
            return

        if (len(args) == 4):
            predPrd = args[3]
            if predPrd == 'm':
                predPrd = 'minute'
            if predPrd == 'D':
                predPrd = 'daily'
            if predPrd == 'W':
                predPrd = 'weekly'
            if predPrd == 'M':
                predPrd = 'monthly'

        try:
            # Everything looks okay; proceed with program
            # Grab the data frame
            # self.D = pand.DataFrame(index=range(self.hc))
            self.D = None
            self.D = ParseData(args[0])
            # The number of previous days of data used
            # when making a prediction

            # PlotData(D)
            s2_mean = 0
            P = None
            i = 0
            res = 0

            while res < 1:
                self.sp_Classic.Learn(self.D)
                res += 1
            while s2_mean < 0.70 and i < 3:
                # Learn the dataset and then display performance statistics
                # sp.TestPerformance()
                # Perform prediction for a specified date range
                P = self.sp_Classic.PredictDate(args[1], args[2])
                if P is None:
                    logger.info(self.instrument_bmex + ': TYPE 2 Reboot')
                    return 0, 0, 0
                s2_mean = self.sp_Classic.Performance()
                # Keep track of number of predicted results for plot
                # n = P.shape[0]
                # Append the predicted results to the actual results
                # D = P.append(D)
                # Predicted results are the first n rows
                # D.to_csv(r'xbt_m1_treated.csv', index=False)
                # PlotData(D, range(n + 1))'''
                i += 1
            # print(P)
            return i, P, s2_mean
        except Exception as e:
            logger.error(str(e))
            sleep(1)
            return 0, 0, 0

    def Engine(self):
        datetime_minute_cached = None
        fails = 0
        self.history.start_supplychain()
        while self.history.get_data_loaded() is False or self.history.get_run_completed() is False:
            logger.info(self.instrument_bmex + ': Waiting for historical data... ')
            sleep(5)
            continue
        logger.info(self.instrument_bmex + ': Starting machine learning computation...')

        while True:
            try:
                if datetime_minute_cached != datetime.utcnow().minute:  # and self.history.get_run_completed() is True:
                    start_timer = timer()

                    timestamp_ = datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:00')

                    timestamp = datetime.strptime(timestamp_, '%Y-%m-%d %H:%M:%S').replace(tzinfo=timezone.utc)
                    timestamp = timestamp.timetuple()
                    start_ts = time.mktime(timestamp)

                    timestamp = datetime.strptime(timestamp_, '%Y-%m-%d %H:%M:%S').replace(tzinfo=timezone.utc) + timedelta(minutes=(self.periods_pred))
                    timestamp = timestamp.timetuple()
                    end_ts = time.mktime(timestamp)

                    naming = str(self.instrument_bmex + "_pair_m1.csv")
                    p = None
                    counter = 0

                    while isinstance(p, pd.DataFrame) is False and counter < 5:
                        i, p, s2_mean = self.Main([naming, start_ts, end_ts, 'm'])
                        counter += 1

                    if counter >= 5:
                        logger.warning(self.instrument_bmex + ': Predictor: Error p format')
                        continue

                    elif s2_mean < 0.70:
                        self.p_verdict = 0
                        logger.info(self.instrument_bmex + ' Processing Time: ' + str(round(timer() - start_timer, 5)))
                        logger.info(self.instrument_bmex + ': Machine learning : UNCONCLUSIVE !')
                        self.ready = True
                        fails += 1
                        datetime_minute_cached = datetime.utcnow().minute
                    # print('shape: ', p.shape)
                    # print(p)
                    # print(datetime.datetime.fromtimestamp(p.loc[0, 'Timestamp']))
                    # print(datetime.datetime.fromtimestamp(p.loc[1, 'Timestamp']))
                    # print(datetime.fromtimestamp(p.loc[self.periods_pred, 'Timestamp']))
                    # print(p)
                    else:
                        if np.isnan(s2_mean):
                            self.p_verdict = 0
                        else:
                            temp = self.periods_pred - 1
                            temp_1 = p.loc[temp + 1, 'Close']
                            temp_2 = p.loc[temp + 1, 'Close']
                            j = 0
                            k = 0
                            while temp >= 0:
                                p_close_tx = p.loc[temp, 'Close']
                                if temp_1 < p_close_tx:
                                    j += 1
                                elif temp_2 > p_close_tx:
                                    k += 1
                                temp_1 = temp_2 = p_close_tx
                                temp -= 1
                            if j >= self.periods_pred:
                                self.p_verdict = -1
                            elif k >= self.periods_pred:
                                self.p_verdict = 1
                            else:
                                self.p_verdict = 0
                        logger.info(self.instrument_bmex + ' Processing Time: ' + str(round(timer() - start_timer, 5)))
                        if self.p_verdict == 0:
                            logger.info(self.instrument_bmex + ' -> Machine learning : NEUTRAL !')
                        elif self.p_verdict > 0:
                            logger.info(self.instrument_bmex + ' -> Machine learning : UP !')
                        else:
                            logger.info(self.instrument_bmex + ' -> Machine learning : DOWN !')
                        self.ready = True
                        datetime_minute_cached = datetime.utcnow().minute
                        fails = 0
                    logger.info(self.instrument_bmex + ' -> Machine learning / Non-gaussian metric : ' + str(round(s2_mean * 100, 2)) + "% (iter: " + str(i) + ")")

                sleep(0.1)
            except Exception as e:
                logger.error(str(e))
                sleep(1)
                pass

    def start_ml(self):
        self.thread.daemon = True
        self.thread.start()

    def get_p_verdict(self):
        if self.p_verdict > 0:
            verdict = 1
        elif self.p_verdict < 0:
            verdict = -1
        else:
            verdict = 0
        return verdict


"""Django Models for tracking the configuration compliance per feature and device."""

import json
import logging

from deepdiff import DeepDiff
from django.core.exceptions import ValidationError
from django.db import models
from django.utils.module_loading import import_string
from hier_config import Host as HierConfigHost
from nautobot.core.models.generics import PrimaryModel
from nautobot.core.models.utils import serialize_object, serialize_object_v2
from nautobot.dcim.models import Device
from nautobot.extras.models import ObjectChange
from nautobot.extras.models.statuses import StatusField
from nautobot.extras.utils import extras_features
from netutils.config.compliance import feature_compliance

from nautobot_golden_config.choices import ComplianceRuleConfigTypeChoice, ConfigPlanTypeChoice, RemediationTypeChoice
from nautobot_golden_config.utilities.constant import ENABLE_SOTAGG, PLUGIN_CFG

LOGGER = logging.getLogger(__name__)
GRAPHQL_STR_START = "query ($device_id: ID!)"

ERROR_MSG = (
    "There was an issue with the data that was returned by your get_custom_compliance function. "
    "This is a local issue that requires the attention of your systems administrator and not something "
    "that can be fixed within the Golden Config plugin. "
)
MISSING_MSG = (
    ERROR_MSG + "Specifically the `{}` key was not found in value the get_custom_compliance function provided."
)
VALIDATION_MSG = (
    ERROR_MSG + "Specifically the key {} was expected to be of type(s) {} and the value of {} was not that type(s)."
)

CUSTOM_FUNCTIONS = {
    "get_custom_compliance": "custom",
    "get_custom_remediation": RemediationTypeChoice.TYPE_CUSTOM,
}

def _is_jsonable(val):
    """Check is value can be converted to json."""
    try:
        json.dumps(val)
        return True
    except (TypeError, OverflowError):
        return False

def _null_to_empty(val):
    """Convert to empty string if the value is currently null."""
    if not val:
        return ""
    return val

def _get_cli_compliance(obj):
    """This function performs the actual compliance for cli configuration."""
    feature = {
        "ordered": obj.rule.config_ordered,
        "name": obj.rule,
    }
    feature.update({"section": obj.rule.match_config.splitlines()})
    value = feature_compliance(
        feature, obj.actual, obj.intended, obj.device.platform.network_driver_mappings.get("netutils_parser")
    )
    compliance = value["compliant"]
    if compliance:
        compliance_int = 1
        ordered = value["ordered_compliant"]
    else:
        compliance_int = 0
        ordered = value["ordered_compliant"]
    missing = _null_to_empty(value["missing"])
    extra = _null_to_empty(value["extra"])
    return {
        "compliance": compliance,
        "compliance_int": compliance_int,
        "ordered": ordered,
        "missing": missing,
        "extra": extra,
    }

def _get_json_compliance(obj):
    """This function performs the actual compliance for json serializable data."""

    def _normalize_diff(diff, path_to_diff):
        """Normalizes the diff to a list of keys and list indexes that have changed."""
        dictionary_items = list(diff.get(f"dictionary_item_{path_to_diff}", []))
        list_items = list(diff.get(f"iterable_item_{path_to_diff}", {}).keys())
        values_changed = list(diff.get("values_changed", {}).keys())
        type_changes = list(diff.get("type_changes", {}).keys())
        return dictionary_items + list_items + values_changed + type_changes

    diff = DeepDiff(obj.actual, obj.intended, ignore_order=obj.ordered, report_repetition=True)
    if not diff:
        compliance_int = 1
        compliance = True
        ordered = True
        missing = ""
        extra = ""
    else:
        compliance_int = 0
        compliance = False
        ordered = False
        missing = _null_to_empty(_normalize_diff(diff, "added"))
        extra = _null_to_empty(_normalize_diff(diff, "removed"))

    return {
        "compliance": compliance,
        "compliance_int": compliance_int,
        "ordered": ordered,
        "missing": missing,
        "extra": extra,
    }

def _verify_get_custom_compliance_data(compliance_details):
    """This function verifies the data is as expected when a custom function is used."""
    for val in ["compliance", "compliance_int", "ordered", "missing", "extra"]:
        try:
            compliance_details[val]
        except KeyError:
            raise ValidationError(MISSING_MSG.format(val)) from KeyError
    for val in ["compliance", "ordered"]:
        if compliance_details[val] not in [True, False]:
            raise ValidationError(VALIDATION_MSG.format(val, "Boolean", compliance_details[val]))
    if compliance_details["compliance_int"] not in [0, 1]:
        raise ValidationError(VALIDATION_MSG.format("compliance_int", "0 or 1", compliance_details["compliance_int"]))
    for val in ["missing", "extra"]:
        if not isinstance(compliance_details[val], str) and not _is_jsonable(compliance_details[val]):
            raise ValidationError(VALIDATION_MSG.format(val, "String or Json", compliance_details[val]))

def _get_hierconfig_remediation(obj):
    """Returns the remediating config."""
    hierconfig_os = obj.device.platform.network_driver_mappings["hier_config"]
    if not hierconfig_os:
        raise ValidationError(f"platform {obj.network_driver} is not supported by hierconfig.")

    try:
        remediation_setting_obj = RemediationSetting.objects.get(platform=obj.rule.platform)
    except Exception as err:  # pylint: disable=broad-except:
        raise ValidationError(f"Platform {obj.network_driver} has no Remediation Settings defined.") from err

    remediation_options = remediation_setting_obj.remediation_options

    try:
        hc_kwargs = {"hostname": obj.device.name, "os": hierconfig_os}
        if remediation_options:
            hc_kwargs.update(hconfig_options=remediation_options)
        host = HierConfigHost(**hc_kwargs)

    except Exception as err:  # pylint: disable=broad-except:
        raise Exception(  # pylint: disable=broad-exception-raised
            f"Cannot instantiate HierConfig on {obj.device.name}, check Device, Platform and Hier Options."
        ) from err

    host.load_generated_config(obj.intended)
    host.load_running_config(obj.actual)
    host.remediation_config()
    remediation_config = host.remediation_config_filtered_text(include_tags={}, exclude_tags={})

    return remediation_config

# The below maps the provided compliance types
FUNC_MAPPER = {
    ComplianceRuleConfigTypeChoice.TYPE_CLI: _get_cli_compliance,
    ComplianceRuleConfigTypeChoice.TYPE_JSON: _get_json_compliance,
    RemediationTypeChoice.TYPE_HIERCONFIG: _get_hierconfig_remediation,
}
# The below conditionally add the custom provided compliance type
for custom_function, custom_type in CUSTOM_FUNCTIONS.items():
    if PLUGIN_CFG.get(custom_function):
        try:
            FUNC_MAPPER[custom_type] = import_string(PLUGIN_CFG[custom_function])
        except Exception as error:  # pylint: disable=broad-except
            msg = (
                "There was an issue attempting to import the custom function of"
                f"{PLUGIN_CFG[custom_function]}, this is expected with a local configuration issue "
                "and not related to the Golden Configuration Plugin, please contact your system admin for further details"
            )
            raise Exception(msg).with_traceback(error.__traceback__)

@extras_features(
    "custom_fields",
    "custom_validators",
    "export_templates",
    "graphql",
    "relationships",
    "webhooks",
)
class ComplianceFeature(PrimaryModel):  # pylint: disable=too-many-ancestors
    """ComplianceFeature details."""

    name = models.CharField(max_length=100, unique=True)
    slug = models.SlugField(max_length=100, unique=True)
    description = models.CharField(max_length=200, blank=True)

    class Meta:
        """Meta information for ComplianceFeature model."""

        ordering = ("slug",)

    def __str__(self):
        """Return a sane string representation of the instance."""
        return self.slug

@extras_features(
    "custom_fields",
    "custom_validators",
    "export_templates",
    "graphql",
    "relationships",
    "webhooks",
)
class ComplianceRule(PrimaryModel):  # pylint: disable=too-many-ancestors
    """ComplianceRule details."""

    feature = models.ForeignKey(to="ComplianceFeature", on_delete=models.CASCADE, related_name="feature")

    platform = models.ForeignKey(
        to="dcim.Platform",
        on_delete=models.CASCADE,
        related_name="compliance_rules",
    )
    description = models.CharField(
        max_length=200,
        blank=True,
    )
    config_ordered = models.BooleanField(
        verbose_name="Configured Ordered",
        help_text="Whether or not the configuration order matters, such as in ACLs.",
        default=False,
    )

    config_remediation = models.BooleanField(
        default=False,
        verbose_name="Config Remediation",
        help_text="Whether or not the config remediation is executed for this compliance rule.",
    )

    match_config = models.TextField(
        blank=True,
        verbose_name="Config to Match",
        help_text="The config to match that is matched based on the parent most configuration. E.g.: For CLI `router bgp` or `ntp`. For JSON this is a top level key name.",
    )
    config_type = models.CharField(
        max_length=20,
        default=ComplianceRuleConfigTypeChoice.TYPE_CLI,
        choices=ComplianceRuleConfigTypeChoice,
        help_text="Whether the configuration is in CLI or JSON/structured format.",
    )
    custom_compliance = models.BooleanField(
        default=False, help_text="Whether this Compliance Rule is proceeded as custom."
    )

    @property
    def remediation_setting(self):
        """Returns remediation settings for a particular platform."""
        return RemediationSetting.objects.filter(platform=self.platform).first()

    class Meta:
        """Meta information for ComplianceRule model."""

        ordering = ("platform", "feature__name")
        unique_together = (
            "feature",
            "platform",
        )

    def __str__(self):
        """Return a sane string representation of the instance."""
        return f"{self.platform} - {self.feature.name}"

    def clean(self):
        """Verify that if cli, then match_config is set."""
        if self.config_type == ComplianceRuleConfigTypeChoice.TYPE_CLI and not self.match_config:
            raise ValidationError("CLI configuration set, but no configuration set to match.")

@extras_features(
    "custom_fields",
    "custom_links",
    "custom_validators",
    "export_templates",
    "graphql",
    "relationships",
    "webhooks",
)
class ConfigCompliance(PrimaryModel):  # pylint: disable=too-many-ancestors
    """Configuration compliance details."""

    device = models.ForeignKey(to="dcim.Device", on_delete=models.CASCADE, help_text="The device")
    rule = models.ForeignKey(to="ComplianceRule", on_delete=models.CASCADE, related_name="rule")
    compliance = models.BooleanField(blank=True)
    actual = models.JSONField(blank=True, help_text="Actual Configuration for feature")
    intended = models.JSONField(blank=True, help_text="Intended Configuration for feature")
    # these three are config snippets exposed for the ConfigDeployment.
    remediation = models.JSONField(blank=True, help_text="Remediation Configuration for the device")
    missing = models.JSONField(blank=True, help_text="Configuration that should be on the device.")
    extra = models.JSONField(blank=True, help_text="Configuration that should not be on the device.")
    ordered = models.BooleanField(default=False)
    # Used for django-pivot, both compliance and compliance_int should be set.
    compliance_int = models.IntegerField(blank=True)

    def to_objectchange(
        self, action, *, related_object=None, object_data_extra=None, object_data_exclude=None
    ):  # pylint: disable=arguments-differ
        """Remove actual and intended configuration from changelog."""
        if not object_data_exclude:
            object_data_exclude = ["actual", "intended"]
        return ObjectChange(
            changed_object=self,
            object_repr=str(self),
            action=action,
            object_data=serialize_object(self, extra=object_data_extra, exclude=object_data_exclude),
            object_data_v2=serialize_object_v2(self),
            related_object=related_object,
        )

    class Meta:
        """Set unique together fields for model."""

        ordering = ["device", "rule"]
        unique_together = ("device", "rule")

    def __str__(self):
        """String representation of a the compliance."""
        return f"{self.device} -> {self.rule} -> {self.compliance}"

    def compliance_on_save(self):
        """The actual configuration compliance happens here, but the details for actual compliance job would be found in FUNC_MAPPER."""
        if self.rule.custom_compliance:
            if not FUNC_MAPPER.get("custom"):
                raise ValidationError(
                    "Custom type provided, but no `get_custom_compliance` config set, please contact system admin."
                )
            compliance_details = FUNC_MAPPER["custom"](obj=self)
            _verify_get_custom_compliance_data(compliance_details)
        else:
            compliance_details = FUNC_MAPPER[self.rule.config_type](obj=self)

        self.compliance = compliance_details["compliance"]
        self.compliance_int = compliance_details["compliance_int"]
        self.ordered = compliance_details["ordered"]
        self.missing = compliance_details["missing"]
        self.extra = compliance_details["extra"]

    def remediation_on_save(self):
        """The actual remediation happens here, before saving the object."""
        if self.compliance:
            self.remediation = ""
            return

        if not self.rule.config_remediation:
            self.remediation = ""
            return

        if not self.rule.remediation_setting:
            self.remediation = ""
            return

        remediation_config = FUNC_MAPPER[self.rule.remediation_setting.remediation_type](obj=self)
        self.remediation = remediation_config

    def save(self, *args, **kwargs):
        """The actual configuration compliance happens here, but the details for actual compliance job would be found in FUNC_MAPPER."""
        self.compliance_on_save()
        self.remediation_on_save()
        self.full_clean()

        super().save(*args, **kwargs)

@extras_features(
    "custom_fields",
    "custom_links",
    "custom_validators",
    "export_templates",
    "graphql",
    "relationships",
    "webhooks",
)
class GoldenConfig(PrimaryModel):  # pylint: disable=too-many-ancestors
    """Configuration Management Model."""

    device = models.OneToOneField(
        to="dcim.Device",
        on_delete=models.CASCADE,
        help_text="device",
        blank=False,
    )
    backup_config = models.TextField(blank=True, help_text="Full backup config for device.")
    backup_last_attempt_date = models.DateTimeField(null=True, blank=True)
    backup_last_success_date = models.DateTimeField(null=True, blank=True)

    intended_config = models.TextField(blank=True, help_text="Intended config for the device.")
    intended_last_attempt_date = models.DateTimeField(null=True, blank=True)
    intended_last_success_date = models.DateTimeField(null=True, blank=True)

    compliance_config = models.TextField(blank=True, help_text="Full config diff for device.")
    compliance_last_attempt_date = models.DateTimeField(null=True, blank=True)
    compliance_last_success_date = models.DateTimeField(null=True, blank=True)

    def to_objectchange(
        self, action, *, related_object=None, object_data_extra=None, object_data_exclude=None
    ):  # pylint: disable=arguments-differ
        """Remove actual and intended configuration from changelog."""
        if not object_data_exclude:
            object_data_exclude = ["backup_config", "intended_config", "compliance_config"]
        return ObjectChange(
            changed_object=self,
            object_repr=str(self),
            action=action,
            object_data=serialize_object(self, extra=object_data_extra, exclude=object_data_exclude),
            object_data_v2=serialize_object_v2(self),
            related_object=related_object,
        )

    @staticmethod
    def get_dynamic_group_device_pks():
        """Get all Device PKs associated with GoldenConfigSetting DynamicGroups."""
        gc_dynamic_group_device_queryset = Device.objects.none()
        for setting in GoldenConfigSetting.objects.all():
            # using "|" should not require calling distinct afterwards
            gc_dynamic_group_device_queryset = gc_dynamic_group_device_queryset | setting.dynamic_group.members

        return set(gc_dynamic_group_device_queryset.values_list("pk", flat=True))

    @classmethod
    def get_golden_config_device_ids(cls):
        """Get all Device PKs associated with GoldenConfig entries."""
        return set(cls.objects.values_list("device__pk", flat=True))

    class Meta:
        """Set unique together fields for model."""

        ordering = ["device"]

    def __str__(self):
        """String representation of a the compliance."""
        return f"{self.device}"

@extras_features(
    "graphql",
)
class GoldenConfigSetting(PrimaryModel):  # pylint: disable=too-many-ancestors
    """GoldenConfigSetting Model definition. This provides global configs instead of via configs.py."""

    name = models.CharField(max_length=100, unique=True)
    slug = models.SlugField(max_length=100, unique=True)
    weight = models.PositiveSmallIntegerField(default=1000)
    description = models.CharField(
        max_length=200,
        blank=True,
    )
    backup_repository = models.ForeignKey(
        to="extras.GitRepository",
        on_delete=models.PROTECT,
        null=True,
        blank=True,
        related_name="backup_repository",
        limit_choices_to={"provided_contents__contains": "nautobot_golden_config.backupconfigs"},
    )
    backup_path_template = models.CharField(
        max_length=255,
        blank=True,
        verbose_name="Backup Path in Jinja Template Form",
        help_text="The Jinja path representation of where the backup file will be found. The variable `obj` is available as the device instance object of a given device, as is the case for all Jinja templates. e.g. `{{obj.location.name|slugify}}/{{obj.name}}.cfg`",
    )
    intended_repository = models.ForeignKey(
        to="extras.GitRepository",
        on_delete=models.PROTECT,
        null=True,
        blank=True,
        related_name="intended_repository",
        limit_choices_to={"provided_contents__contains": "nautobot_golden_config.intendedconfigs"},
    )
    intended_path_template = models.CharField(
        max_length=255,
        blank=True,
        verbose_name="Intended Path in Jinja Template Form",
        help_text="The Jinja path representation of where the generated file will be places. e.g. `{{obj.location.name|slugify}}/{{obj.name}}.cfg`",
    )
    jinja_repository = models.ForeignKey(
        to="extras.GitRepository",
        on_delete=models.PROTECT,
        null=True,
        blank=True,
        related_name="jinja_template",
        limit_choices_to={"provided_contents__contains": "nautobot_golden_config.jinjatemplate"},
    )
    jinja_path_template = models.CharField(
        max_length=255,
        blank=True,
        verbose_name="Template Path in Jinja Template Form",
        help_text="The Jinja path representation of where the Jinja template can be found. e.g. `{{obj.platform.network_driver}}.j2`",
    )
    backup_test_connectivity = models.BooleanField(
        default=True,
        verbose_name="Backup Test",
        help_text="Whether or not to pretest the connectivity of the device by verifying there is a resolvable IP that can connect to port 22.",
    )
    sot_agg_query = models.ForeignKey(
        to="extras.GraphQLQuery",
        on_delete=models.PROTECT,
        null=True,
        blank=True,
        related_name="sot_aggregation",
    )
    dynamic_group = models.OneToOneField(
        to="extras.DynamicGroup",
        on_delete=models.PROTECT,
        related_name="golden_config_setting",
    )

    def __str__(self):
        """Return a simple string if model is called."""
        return f"Golden Config Setting - {self.name}"

    class Meta:
        """Set unique fields for model.

        Provide ordering used in tables and get_device_to_settings_map.
        Sorting on weight is performed from the highest weight value to the lowest weight value.
        This is to ensure only one plugin settings could be applied per single device based on priority and name.
        """

        verbose_name = "Golden Config Setting"
        ordering = ["-weight", "name"]  # Refer to weight comment in class docstring.

    def clean(self):
        """Validate the scope and GraphQL query."""
        super().clean()

        if ENABLE_SOTAGG and not self.sot_agg_query:
            raise ValidationError("A GraphQL query must be defined when `ENABLE_SOTAGG` is True")

        if self.sot_agg_query:
            LOGGER.debug("GraphQL - test  query start with: `%s`", GRAPHQL_STR_START)
            if not str(self.sot_agg_query.query.lstrip()).startswith(GRAPHQL_STR_START):
                raise ValidationError(f"The GraphQL query must start with exactly `{GRAPHQL_STR_START}`")

    def get_queryset(self):
        """Generate a Device QuerySet from the filter."""
        return self.dynamic_group.members

    def device_count(self):
        """Return the number of devices in the group."""
        return self.dynamic_group.count

    def get_url_to_filtered_device_list(self):
        """Get url to all devices that are matching the filter."""
        return self.dynamic_group.get_group_members_url()

@extras_features(
    "custom_fields",
    "custom_links",
    "custom_validators",
    "export_templates",
    "graphql",
    "relationships",
    "webhooks",
)
class ConfigRemove(PrimaryModel):  # pylint: disable=too-many-ancestors
    """ConfigRemove for Regex Line Removals from Backup Configuration Model definition."""

    name = models.CharField(max_length=255)
    platform = models.ForeignKey(
        to="dcim.Platform",
        on_delete=models.CASCADE,
        related_name="backup_line_remove",
    )
    description = models.CharField(
        max_length=200,
        blank=True,
    )
    regex = models.CharField(
        max_length=200,
        verbose_name="Regex Pattern",
        help_text="Regex pattern used to remove a line from the backup configuration.",
    )

    clone_fields = ["platform", "description", "regex"]

    class Meta:
        """Meta information for ConfigRemove model."""

        ordering = ("platform", "name")
        unique_together = ("name", "platform")

    def __str__(self):
        """Return a simple string if model is called."""
        return self.name

@extras_features(
    "custom_fields",
    "custom_links",
    "custom_validators",
    "export_templates",
    "graphql",
    "relationships",
    "webhooks",
)
class ConfigReplace(PrimaryModel):  # pylint: disable=too-many-ancestors
    """ConfigReplace for Regex Line Replacements from Backup Configuration Model definition."""

    name = models.CharField(max_length=255)
    platform = models.ForeignKey(
        to="dcim.Platform",
        on_delete=models.CASCADE,
        related_name="backup_line_replace",
    )
    description = models.CharField(
        max_length=200,
        blank=True,
    )
    regex = models.CharField(
        max_length=200,
        verbose_name="Regex Pattern to Substitute",
        help_text="Regex pattern that will be found and replaced with 'replaced text'.",
    )
    replace = models.CharField(
        max_length=200,
        verbose_name="Replaced Text",
        help_text="Text that will be inserted in place of Regex pattern match.",
    )

    clone_fields = ["platform", "description", "regex", "replace"]

    class Meta:
        """Meta information for ConfigReplace model."""

        ordering = ("platform", "name")
        unique_together = ("name", "platform")

    def __str__(self):
        """Return a simple string if model is called."""
        return self.name

@extras_features(
    "custom_fields",
    "custom_links",
    "custom_validators",
    "export_templates",
    "graphql",
    "relationships",
    "webhooks",
)
class RemediationSetting(PrimaryModel):  # pylint: disable=too-many-ancestors
    """RemediationSetting details."""

    # Remediation points to the platform
    platform = models.OneToOneField(
        to="dcim.Platform",
        on_delete=models.CASCADE,
        related_name="remediation_settings",
    )

    remediation_type = models.CharField(
        max_length=50,
        default=RemediationTypeChoice.TYPE_HIERCONFIG,
        choices=RemediationTypeChoice,
        help_text="Whether the remediation setting is type HierConfig or custom.",
    )

    # takes options.json.
    remediation_options = models.JSONField(
        blank=True,
        default=dict,
        help_text="Remediation Configuration for the device",
    )

    csv_headers = [
        "platform",
        "remediation_type",
    ]

    class Meta:
        """Meta information for RemediationSettings model."""

        ordering = ("platform", "remediation_type")

    def to_csv(self):
        """Indicates model fields to return as csv."""
        return (
            self.platform,
            self.remediation_type,
        )

    def __str__(self):
        """Return a sane string representation of the instance."""
        return str(self.platform)

@extras_features(
    "custom_fields",
    "custom_links",
    "custom_validators",
    "export_templates",
    "graphql",
    "relationships",
    "webhooks",
    "statuses",
)
class ConfigPlan(PrimaryModel):  # pylint: disable=too-many-ancestors
    """ConfigPlan for Golden Configuration Plan Model definition."""

    plan_type = models.CharField(max_length=20, choices=ConfigPlanTypeChoice, verbose_name="Plan Type")
    device = models.ForeignKey(
        to="dcim.Device",
        on_delete=models.CASCADE,
        related_name="config_plan",
    )
    config_set = models.TextField(help_text="Configuration set to be applied to device.")
    feature = models.ManyToManyField(
        to=ComplianceFeature,
        related_name="config_plan",
        blank=True,
    )
    plan_result = models.ForeignKey(
        to="extras.JobResult",
        on_delete=models.CASCADE,
        related_name="config_plan",
        verbose_name="Plan Result",
    )
    deploy_result = models.ForeignKey(
        to="extras.JobResult",
        on_delete=models.PROTECT,
        related_name="config_plan_deploy_result",
        verbose_name="Deploy Result",
        blank=True,
        null=True,
    )
    change_control_id = models.CharField(
        max_length=50,
        blank=True,
        verbose_name="Change Control ID",
        help_text="Change Control ID for this configuration plan.",
    )
    change_control_url = models.URLField(blank=True, verbose_name="Change Control URL")
    status = StatusField(blank=True, null=True, on_delete=models.PROTECT)

    class Meta:
        """Meta information for ConfigPlan model."""

        ordering = ("-created", "device")
        unique_together = (
            "plan_type",
            "device",
            "created",
        )

    def __str__(self):
        """Return a simple string if model is called."""
        return f"{self.device.name}-{self.plan_type}-{self.created}"


import socket

if __name__ == '__main__':
    # 创建TCP套接字
    server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

    # 设置套接字选项
    server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

    # 绑定 监听
    server_socket.bind(('',8080))

    server_socket.listen(128)

    while True:
        client_socket, client_addr = server_socket.accept()
        print("接受到来自%s的连接请求" % str(client_addr))

        recv_data = client_socket.recv(4096)
        if not recv_data:
            print("对方已经断开连接")
        else:
            print(recv_data)
        client_socket.close()

Day01-15 Python基础课程/Day01-15课程与项目/07 [项目]web服务器：自己动手实现web服务器/03-代码/00-模拟web服务器接收浏览器的HTTP请求.py

00-模拟web服务器接收浏览器的HTTP请求.py


def nextSmaller(arr):
    # result array
    res = [-1] * len(arr)
    # stak
    stack = []

    # traversing from end
    for i in range(len(arr)-1, -1, -1):
        # current element
        curr = arr[i]

        # keep popping element from stack untill you're getting a bigger element
        while len(stack) and arr[stack[-1]] >= curr:
            stack.pop()

        # we got the smaller element, store it as answer
        if len(stack):
            res[i] = stack[-1]
        else:
            res[i] = len(arr)
        # add the current number
        stack.append(i)
    # print(res)
    return res

def prevSmaller(arr):
    st = []
    res = [-1] * len(arr)

    for i in range(len(arr)):
        curr = arr[i]

        while len(st) and arr[st[-1]] >= curr:
            st.pop()

        if len(st):
            res[i] = st[-1]
        else:
            res[i] = -1

        st.append(i)

    # print(res)
    return res

def maxAreaHistogram(arr):
    ns = nextSmaller(arr)
    ps = prevSmaller(arr)

    area = 0

    for i in range(len(arr)):
        area = max((ns[i] - ps[i] - 1) * arr[i], area)

    # print(area)
    return area

def maxAreaMatrix(arr):
    tempArr = [0] * len(arr)
    area = 0
    for i in range(len(arr)):
        for j in range(len(arr[i])):
            if arr[i][j]:
                tempArr[j] += arr[i][j]
            else:
                tempArr[j] = 0

        print(tempArr)
        area = max(area, maxAreaHistogram(tempArr))

    print(area)

l = [[0, 1, 1, 0], [1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 0, 0]]
maxAreaMatrix(l)


# -*- coding: utf-8 -*-

# Data preparation at one-second level for Ph.D thesis
# @author: Andres L. Suarez-Cetrulo

import glob
import time
import logging
import yaml
import subprocess
import os
import pandas as pd
import numpy as np
import datetime

# Global attributes
SLASH = os.path.sep
EQUIVALENCE = {
    's': 1,
    'min': 60
}

# logging.getLogger().addHandler(logging.StreamHandler())  # for debugging while coding / comment out otherwise
LOGLEVEL = logging.DEBUG  # logging.WARNING  # logging.DEBUG

def get_datetime_format(date):
    return time.strptime(date, "%Y-%m-%d")

def show_missing_values(dates, df_name, df):

    logging.debug('Printing list of dates with missing values in ' + df_name + ' level (e.g. NA and nulls).')
    logging.debug(str(np.unique(dates.index.time)))
    logging.debug('Showing full DF for only rows with missing values in ' + df_name + ' level')
    logging.debug(df[df.isnull().any(axis=1)].to_string())

    logging.debug('Returning the list of columns which have missing values in ' + df_name + ' level')
    nulls = list()
    for col, bool_var in df.isna().any().items():
        if bool_var:
            nulls.append(col)
    logging.debug(df.isna().any().keys())

def set_logging(config):
    # Set logging level
    logname = str(config['efts']).replace('\'','') + ' Tests across files starting with ' + config['levels'][0] + \
              ' level and period (' + config['start'] + '-' + config['end'] + ').log'
    logging.basicConfig(filename=config['path'] + logname)
    logger = logging.getLogger()
    logger.setLevel(LOGLEVEL)
    logging.info('')

def log_new_execution(config):
    logging.info('')
    logging.info('')
    logging.info('')
    logging.info('')
    logging.info('')
    logging.info('#########################')
    logging.info('')
    logging.info('')
    logging.info('NEW EXECUTION AT: '+str(datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')))
    logging.info('')
    logging.info('')
    logging.info('#########################')
    logging.info('Logging config settings:')
    logging.info(config)

def log_iteration(config):
    logging.info(' --------------------------------------------------------------  ')
    logging.info(' START OF TEST FOR ' + str(config['eft']).replace('\'','') +
                 ' ' + config['level'] + ' level on period: ' +
                 config['start'] + ' to ' + config['end'])
    logging.info(' --------------------------------------------------------------  ')

def load_config():
    # Load global parameters as paths, symbols and periods to iterate through
    with open(os.path.sep.join(['.', 'config.yaml']), 'r') as f:
        config = yaml.safe_load(f.read())['merging']

    # Path where every individual folder is located
    config['path'] = config['src_path']  # + config['level'] + '-level' + SLASH + config['eft'] + SLASH
    config['filename_pattern'] = config['eft'] + '_*_indicators.csv.gz'
    config['start_date'] = get_datetime_format(config['start'])
    config['end_date'] = get_datetime_format(config['end'])
    config['end_date'] = get_datetime_format(config['end'])
    config['name'] = '_'.join([config['eft'], config['output_subname'], config['level']])+'-level'
    config['output'] = config['path'] + config['name']

    # The feature set in 15min and 30min levels differ to avoid considering dependencies across days
    config['featureset'] = config['columns'+nconf['level']] if config['level'] in ['15min', '30min'] \
        else config['columns']

    return config

def new_config(nconf):
    """ Only use this for testing and not creating of .arff, as we add the field gapt+1 to the featureset"""
    # Path where every individual folder is located
    nconf['path'] = nconf['src_path'] + nconf['level'] + '-level' + SLASH + nconf['eft'] + SLASH
    nconf['filename_pattern'] = nconf['eft'] + '_*_indicators.csv.gz'
    nconf['start_date'] = get_datetime_format(nconf['start'])
    nconf['end_date'] = get_datetime_format(nconf['end'])
    nconf['name'] = '_'.join([nconf['eft'], nconf['output_subname'], nconf['level']])+'-level'
    nconf['output'] = nconf['path'] + nconf['name']

    # The feature set in 15min and 30min levels differ to avoid considering dependencies across days
    nconf['featureset'] = nconf['columns'+nconf['level']] if nconf['level'] in ['15min', '30min'] \
        else nconf['columns']

    if 'gap_t+1' not in nconf['featureset']:
        nconf['featureset'].append('gap_t+1')

    return nconf

def load_data(conf, min_time=''):
    # Iterate through collection of processed data with indicators
    df_full = pd.DataFrame()

    for filename in sorted(glob.glob(conf['path'] + conf['filename_pattern'])):
        # Extract dates from filename
        aux_dates = filename.replace(conf['path'] + conf['eft']+'_(', '')
        aux_dates = aux_dates.replace(')_indicators.csv.gz', '')
        dates = aux_dates.split('_to_')

        # If file in date range, add it to DF
        if get_datetime_format(dates[0]) >= conf['start_date'] and \
                get_datetime_format(dates[1]) <= conf['end_date']:
            logging.info('Importing data period ' + str(dates[0]) + ' to ' + str(dates[1]) + '.')
            new_df = pd.read_csv(filename, encoding="utf-8", index_col=0, sep=";")
            new_df['datetime'] = new_df.index  # save string readable timestamp as a column
            new_df.index = pd.to_datetime(new_df.index)
            print('Preview of 5 fist rows of file being load.')
            print(new_df.head(5))

            # If required, remove times if there are dependencies from previous days,
            #  or missing values in certain dates that we should avoid
            if min_time == '':
                print('Rows removed at this level: '+str(conf['rows_per_day_to_remove'][str(conf['level'])]))
                print(conf['rows_per_day_to_remove'][str(conf['level'])])
                min_time = '09:30' if int(conf['rows_per_day_to_remove'][str(conf['level'])]) == 0 \
                                   else get_min_time(conf, new_df)
            print('Current minimum time being considered: '+str(min_time))
            new_df = new_df.between_time(min_time, '16:00')
            df_full = df_full.append(new_df)

    # logging.debug('Printing a sample of DF at level: '+conf['level'])
    # logging.debug(df)
    logging.info('Data read.')
    logging.warning('WARNING: Now make sure that the count of rows makes sense for the number of market days:')
    logging.warning(df_full.groupby([df_full.index.year, df_full.index.month, df_full.index.day]).agg({'count'}).to_string())

    return df_full[conf['featureset']], min_time

def get_min_time(conf, df):

    # Return the time of the minimum datetime
    return min(df[int(conf['rows_per_day_to_remove'][str(conf['level'])]):].index.tolist()).time()

def split_level(txt, seps):  # this function would make sense if we had  tick, hourly or daily level also.
    default_sep = seps[0]

    # we skip seps[0] because that's the default separator
    level = ''
    for sep in seps[1:]:
        aux_len = len(txt)
        txt = txt.replace(sep, default_sep)
        level = sep if len(txt) != aux_len else level  # if it has changed, then we keep the level string

    level_split = [i.strip() for i in txt.split(default_sep)]
    level_split[1] = default_sep if level == '' else level
    return level_split

def plot_stats(cl_df, ol_df, scl, sol):
    logging.info('Iterated level DF has a length of: '+str(len(ol_df))+' versus a length of: '+str(len(cl_df)) +
                 '. The equivalence should be of: cdf = '+str(float(scl)/float(sol))+'*odf')
    logging.info('Current DF cdf stats: '+cl_df.describe().to_string())
    logging.info('Compared DF odf stats: '+ol_df.describe().to_string())

def test_subsets(df1, df2):

    # checking that datetime for the current level exists in inferior levels
    key_diff = set(df1.index).difference(df2.index)
    where_diff = df1.index.isin(key_diff)
    logging.info('Logging difference:')
    logging.info(df1[where_diff])

    assert len(df1[where_diff]) == 0
    logging.debug('Test 2 PASSED: No missing subsets (rows based on datetime) when looking at lower levels.')

def test_number_of_rows(cl_df, ol_df, comparison_type, scl, sol):

    if comparison_type == 'eft':
        cl_df.index = pd.to_datetime(cl_df.index)
        logging.debug(len(np.unique(cl_df.index.date)))

        len_cdf = len(cl_df)
        len_odf = len(ol_df)

        logging.debug('Size of c_eft: '+str(len_cdf))
        logging.debug(cl_df.head())
        logging.debug('Size of o_eft: '+str(len_odf))
        logging.debug(ol_df.head())

        cl_df['datetime'] = cl_df.index
        print(cl_df[cl_df['datetime'] == '2015-04-01 09:41:15'])

        test = len_cdf == len_odf
        logging.info('')
        logging.info('Test 0: Length of both EFTs have the same length for the same periods and level: ' +
                     ' PASSED' if test else ' NOT PASSED')
        logging.info('')
        test_subsets(ol_df, cl_df)

        assert test
    else:

        # Test not ready. It only works for single dates.
        from collections import Counter

        cl_df['stridx'] = cl_df.index
        ol_df['stridx'] = ol_df.index

        logging.info(pd.Series(cl_df['stridx'].str.split(' ')[0].map(Counter).sum()))
        logging.info(len(cl_df[cl_df['stridx'].str.contains('2015-01-02')]))
        logging.info(len(ol_df[ol_df['stridx'].str.contains('2015-01-02')]))

        logging.infologging.info(len(cl_df.loc['2015-01-02' in cl_df.index]))
        logging.info(len(ol_df.loc['2015-01-02' in ol_df.index]))

        logging.info('---')
        logging.info(len_cdf * (float(scl)/float(sol)) * len(np.unique(cl_df.index.date)) - (float(scl)/float(sol) * len(np.unique(cl_df.index.date))) + len(np.unique(cl_df.index.date)))
        ## for a single day
        logging.infologging.info(len_cdf * float(scl)/float(sol) - float(scl)/float(sol) + 1)
        logging.info(len_odf * sol)

        # for a single day
        #assert len_cdf * float(scl)/float(sol) - float(scl)/float(sol) + 1 == len_odf * sol

def max_and_min_dates(cl_df, ol_df):
    cl_df['datetime'] = pd.to_datetime(cl_df.index)
    ol_df['datetime'] = pd.to_datetime(ol_df.index)

    logging.info('Max and min dates in current DF are: ' +
                 str(cl_df['datetime'].min())+' - '+str(cl_df['datetime'].max()))
    logging.info('Max and min dates in compared DF are: ' +
                 str(ol_df['datetime'].min())+' - '+str(ol_df['datetime'].max()))

def any_nulls(cl_df, ol_df):

    # Exclude last row from comparisons as it may be null for lack of values after 4pm
    cl_df.drop(cl_df.tail(1).index, inplace=True)  # drop last n rows
    ol_df.drop(ol_df.tail(1).index, inplace=True)  # drop last n rows

    logging.info('Confirming that both currend compared models don\'t have missing values:')
    passed = (len(cl_df[cl_df.isnull().any(axis=1)]) == 0 and len(ol_df[ol_df.isnull().any(axis=1)]) == 0)
    logging.info(('PASSED' if passed else 'NOT PASSED'))

    if ~passed:
        aux_df = cl_df[cl_df.isnull().any(axis=1)]
        aux_df.index = pd.to_datetime(aux_df.index)
        show_missing_values(dates=aux_df, df_name='current', df=cl_df)
        aux_df = ol_df[ol_df.isnull().any(axis=1)]
        aux_df.index = pd.to_datetime(aux_df.index)
        show_missing_values(dates=aux_df, df_name='compared', df=ol_df)

    assert passed

def get_percentage_of_gaps(cl_df, ol_df):

    logging.debug('--------------------------------------------')
    logging.debug('Printing percentage of gaps in current level')
    logging.info(len(cl_df[cl_df['gap_t+1'] == 1])/len(cl_df))

    logging.debug('Printing percentage of gaps in compared level')
    logging.info(len(ol_df[ol_df['gap_t+1'] == 1])/len(ol_df))
    logging.debug('')
    logging.warning('Are the two values above similar enough when compared to the previous testing logs? Manual check.')
    logging.debug('--------------------------------------------')

def compare_levels(cl_df, ol_df, c_level, o_level, featureset):

    logging.info('')
    logging.info('#################################')
    logging.info('')
    logging.info('o_level: '+str(o_level))
    logging.info('c_level: '+str(c_level))
    logging.info('')

    # second equivalence of current level and the other level to compare against in the current iteration
    scl = int(c_level[0]) * EQUIVALENCE[c_level[1]]
    sol = int(o_level[0]) * EQUIVALENCE[o_level[1]]

    # First unit test
    logging.info('Test 1: Stats and counts - Judge manually below to determine if it PASSED. Different number of ' +
                 'columns across levels it\'s ok, as it is a design-made choice.')
    logging.info('Please, check that the difference in amount of rows for the same date period makes sense.')
    logging.info('For a single date, this should be truth: ' +
                 'len_current_df * float(scl)/float(sol) - float(scl)/float(sol) + 1 == len_compared_df * sol')
    logging.info('scl and sol are the amount of seconds of the current and compared level respectively. '+
                 'If current_df level =30 min level, then scl=30*60.')
    plot_stats(cl_df, ol_df, scl, sol)

    # if the other level to compare against is lower than the current one, trigger the following tests
    # added extra checks just in case
    if sol < scl and int(c_level[0]) % int(o_level[0]) == 0 and c_level[1] == o_level[1]:
        logging.info('Test 2: Check if lower levels miss any row that they shouldn\'t')
        test_subsets(cl_df, ol_df)

    logging.info('Test 3: Max and Min dates:')
    max_and_min_dates(cl_df, ol_df)

    logging.info('Test 4: Any null values?')
    any_nulls(cl_df[featureset], cl_df[featureset])

    logging.info('Test 5: Percentage of gaps. ' +
                 'It should map the percentages that we already have in the same files ' +
                 'regarding to the class distribution.')
    get_percentage_of_gaps(cl_df, ol_df)

def compare_efts(ce_df, oe_df, c_eft, o_eft):

    logging.info('')
    logging.info('#################################')
    logging.info('')
    logging.info('o_eft: '+str(o_eft))
    logging.info('c_eft: '+str(c_eft))
    logging.info('')

    logging.info('Test 0: Stats and counts - Judge manually below to determined if it PASSED. ' +
                 'IBEX may be the most different one due to bank holidays in Spain.')
    test_number_of_rows(ce_df, oe_df, 'eft', 1, 1)

def run_tests_for_config(config):

    # Load all the data for the current level and period in a Dataframe
    cl_df, min_time = load_data(config)

    logging.info('First part in testing, comparing to efts at same level:')
    for other_eft in config['compare_efts']:
        logging.info(' ///// ')
        logging.info('Comparing to efts: '+other_eft)

        # Load DF to compare against
        oe_config = config.copy()
        oe_config['eft'] = other_eft
        oe_df, _ = load_data(new_config(oe_config))  # only getting DF, as min_time doesn't need to update
        compare_efts(cl_df, oe_df, config['eft'], oe_config['eft'])
        logging.info(' ///// ')

    logging.info('Second part in testing, comparing different levels of the same EFT and period:')
    c_level = split_level(config['level'], list(EQUIVALENCE.keys()))
    for other_level in config['compare_against']:
        logging.info(' ///// ')
        logging.info('Comparing to level: '+other_level)

        # Load DF to compare against
        ol_config = config.copy()
        ol_config['level'] = other_level
        # new_config updates some config settings that are dependant on the values changed
        ol_df, _ = load_data(new_config(ol_config), min_time)  # only getting DF, as min_time doesn't need to update

        # Tests
        o_level = split_level(other_level, list(EQUIVALENCE.keys()))

        compare_levels(cl_df, ol_df, c_level, o_level, config['featureset'])
        logging.info(' ///// ')

def run_tests():

    # Load settings and set logging config
    config = load_config()
    config['featureset'].append('gap_t+1')  # only for testing (not merging)
    set_logging(config)
    log_new_execution(config)

    for eft in config['efts']:
        print()
        print('EFT: '+str(eft))
        print('------')

        for period in config['periods'].values():
            print('')
            print('Period: '+str(period))
            print('')

            for level in config['levels']:
                print('Level: '+str(level))
                config['eft'] = eft
                config['start'] = period[0]
                config['end'] = period[1]
                config['level'] = level
                log_iteration(config)
                run_tests_for_config(new_config(config))

if __name__ == '__main__':
    run_tests()


from character import Enemy
from battle import Battle
from system import display_message

class LastCastle:
    def __init__(self, player):
        self.player = player
        self.enemy = Enemy()
        self.enemy.define_enemy("last_boss", self.player.stats["level"])
        self.battle = Battle(self.player, self.enemy)

    def boss_lines(self):
        display_message('魔王:「良くぞここまでたどり着いた、勇者よ。誉めてやろう。」')
        display_message('%s:「だろ？そして、俺はお前を倒す！」' % (self.player.stats['name']))
        display_message('魔王:「ほう。面白い。出来るものならやってみせよ！」')

    def encounter(self):
        display_message("%sが現れた！" % (self.enemy.stats["name"]))
        self.battle.start_battle()

    def travel(self):
        display_message("～魔王城～")

        self.boss_lines()

        self.encounter()