#!/usr/bin/python

import socket
import sys
import os
import signal

HEADER_SIZE = 10

s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(("",1235))
s.listen(5)

while True:
    clientsocket, address = s.accept()
    print(f"Connection from {address} established!")
    full_msg = ''
    new_msg = True
    while True:
        msg = clientsocket.recv(16)

        try:

            if new_msg:

                print(f"new message length: {msg[:HEADER_SIZE]}")
                msglen = int(msg[:HEADER_SIZE])
                new_msg = False

            full_msg += msg.decode("utf-8")

            if len(full_msg)-HEADER_SIZE == msglen:
                print("Full message received")
                print(full_msg[HEADER_SIZE:])
                new_msg = True
                full_msg = ''
        except Exception as e:
            continue

    print(full_msg)


# Importing the libraries
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from sklearn.metrics import r2_score

# Generic Settings
dataset = pd.read_csv(r'D:\Learning\Udemy ML\MLPracticeProjects\Classification\Classification Evaluation Tool\Data.csv')
display_outcome = False # shows actual ouput vs predicted output
np.set_printoptions(precision=2) # sets property for output arrays
class color: # class for print in particular format
   CYAN = '\033[96m'
   BLUE = '\033[94m'
   GREEN = '\033[92m'
   YELLOW = '\033[93m'
   RED = '\033[91m'
   BOLD = '\033[1m'
   UNDERLINE = '\033[4m'
   END = '\033[0m'

# Allocating x and y paramters from the dataset
x = dataset.iloc[:, :-1].values
y = dataset.iloc[:, -1].values

# Splitting the dataset into the Training set and Test set
from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size = 0.25, random_state = 0)

# Feature Scaling
from sklearn.preprocessing import StandardScaler
sc = StandardScaler()
x_train = sc.fit_transform(x_train)
x_test = sc.transform(x_test)

########################################## Logistic Regression
print(color.BOLD + color.RED + 'Logistic Regression Start -------------------' + color.END)

# Training
from sklearn.linear_model import LogisticRegression
logistic_classifier = LogisticRegression(random_state = 0)
logistic_classifier.fit(x_train, y_train)

# Predict
logistic_y_pred = logistic_classifier.predict(x_test)

# Print if asked
if display_outcome :
    print('Test Results:', np.concatenate((logistic_y_pred.reshape(len(logistic_y_pred), 1), y_test.reshape(len(y_test), 1)), 1))

# Making the Confusion Matrix
from sklearn.metrics import confusion_matrix, accuracy_score
cm = confusion_matrix(y_test, logistic_y_pred)
print('Confusion Matrix:', cm)
print('Accuracy score:', accuracy_score(y_test, logistic_y_pred) * 100, '%')

print(color.BOLD + color.RED + 'Logistic Regression End ---------------------' + color.END)
##########################################
########################################## K-NN Classifier
print(color.BOLD + color.BLUE + 'K-NN Classifier Start -------------------' + color.END)

# Training
from sklearn.neighbors import KNeighborsClassifier
knn_classifier = KNeighborsClassifier(n_neighbors = 5, metric = 'minkowski', p = 2) # default parameters
knn_classifier.fit(x_train, y_train)

# Predict
knn_y_pred = knn_classifier.predict(x_test)

# Print if asked
if display_outcome :
    print('Test Results:', np.concatenate((knn_y_pred.reshape(len(knn_y_pred), 1), y_test.reshape(len(y_test), 1)), 1))

# Making the Confusion Matrix
from sklearn.metrics import confusion_matrix, accuracy_score
cm = confusion_matrix(y_test, knn_y_pred)
print('Confusion Matrix:', cm)
print('Accuracy score:', accuracy_score(y_test, knn_y_pred) * 100, '%')

print(color.BOLD + color.BLUE + 'K-NN Classifier End ---------------------' + color.END)
##########################################
########################################## SVM Classifier
print(color.BOLD + color.CYAN + 'SVM Classifier Start -------------------' + color.END)

# Training
from sklearn.svm import SVC
svm_classifier = SVC(kernel = 'linear', random_state = 0)
svm_classifier.fit(x_train, y_train)

# Predict
svm_y_pred = svm_classifier.predict(x_test)

# Print if asked
if display_outcome :
    print('Test Results:', np.concatenate((svm_y_pred.reshape(len(svm_y_pred), 1), y_test.reshape(len(y_test), 1)), 1))

# Making the Confusion Matrix
from sklearn.metrics import confusion_matrix, accuracy_score
cm = confusion_matrix(y_test, svm_y_pred)
print('Confusion Matrix:', cm)
print('Accuracy score:', accuracy_score(y_test, svm_y_pred) * 100, '%')

print(color.BOLD + color.CYAN + 'SVM Classifier End ---------------------' + color.END)
##########################################
########################################## Kernel SVM Classifier
print(color.BOLD + color.CYAN + 'Kernel SVM Classifier Start -------------------' + color.END)

# Training
from sklearn.svm import SVC
ksvm_classifier = SVC(kernel = 'rbf', random_state = 0)
ksvm_classifier.fit(x_train, y_train)

# Predict
ksvm_y_pred = ksvm_classifier.predict(x_test)

# Print if asked
if display_outcome :
    print('Test Results:', np.concatenate((ksvm_y_pred.reshape(len(ksvm_y_pred), 1), y_test.reshape(len(y_test), 1)), 1))

# Making the Confusion Matrix
from sklearn.metrics import confusion_matrix, accuracy_score
cm = confusion_matrix(y_test, ksvm_y_pred)
print('Confusion Matrix:', cm)
print('Accuracy score:', accuracy_score(y_test, ksvm_y_pred) * 100, '%')

print(color.BOLD + color.CYAN + 'Kernel SVM Classifier End ---------------------' + color.END)
##########################################
########################################## Naive Bayes Classifier
print(color.BOLD + color.GREEN + 'Naive Bayes Classifier Start -------------------' + color.END)

# Training
from sklearn.naive_bayes import GaussianNB
nb_classifier = GaussianNB()
nb_classifier.fit(x_train, y_train)

# Predict
nb_y_pred = nb_classifier.predict(x_test)

# Print if asked
if display_outcome :
    print('Test Results:', np.concatenate((nb_y_pred.reshape(len(nb_y_pred), 1), y_test.reshape(len(y_test), 1)), 1))

# Making the Confusion Matrix
from sklearn.metrics import confusion_matrix, accuracy_score
cm = confusion_matrix(y_test, nb_y_pred)
print('Confusion Matrix:', cm)
print('Accuracy score:', accuracy_score(y_test, nb_y_pred) * 100, '%')

print(color.BOLD + color.GREEN + 'Naive Bayes Classifier End ---------------------' + color.END)
##########################################
########################################## Decision Tree Classifier
print(color.BOLD + color.YELLOW + 'Decision Tree Classifier Start -------------------' + color.END)

# Training
from sklearn.tree import DecisionTreeClassifier
dt_classifier = DecisionTreeClassifier(criterion = 'entropy', random_state = 0)
dt_classifier.fit(x_train, y_train)

# Predict
dt_y_pred = dt_classifier.predict(x_test)

# Print if asked
if display_outcome :
    print('Test Results:', np.concatenate((dt_y_pred.reshape(len(dt_y_pred), 1), y_test.reshape(len(y_test), 1)), 1))

# Making the Confusion Matrix
from sklearn.metrics import confusion_matrix, accuracy_score
cm = confusion_matrix(y_test, dt_y_pred)
print('Confusion Matrix:', cm)
print('Accuracy score:', accuracy_score(y_test, dt_y_pred) * 100, '%')

print(color.BOLD + color.YELLOW + 'Decision Tree Classifier End ---------------------' + color.END)
##########################################
########################################## Random Forest Classifier
print(color.BOLD + color.YELLOW + 'Random Forest Classifier Start -------------------' + color.END)

# Training
from sklearn.ensemble import RandomForestClassifier
rf_classifier = RandomForestClassifier(n_estimators = 10, criterion = 'entropy', random_state = 0)
rf_classifier.fit(x_train, y_train)

# Predict
rf_y_pred = rf_classifier.predict(x_test)

# Print if asked
if display_outcome :
    print('Test Results:', np.concatenate((rf_y_pred.reshape(len(rf_y_pred), 1), y_test.reshape(len(y_test), 1)), 1))

# Making the Confusion Matrix
from sklearn.metrics import confusion_matrix, accuracy_score
cm = confusion_matrix(y_test, rf_y_pred)
print('Confusion Matrix:', cm)
print('Accuracy score:', accuracy_score(y_test, rf_y_pred) * 100, '%')

print(color.BOLD + color.YELLOW + 'Random Forest Classifier End ---------------------' + color.END)
##########################################


# Curso Introdução a Linguagem Python - MIT MISTI Brazil–Unicamp
# Ana Luísa Fogarin - 02/02/2021
# a193948@dac.unicamp.br

# SET 0 - Problema 3 - Conta de Energia

# 0 - 500 kWh = 0.45
# 500 - 1500 kWh = 0.74
# 1500 - 2500 kWh = 1.25
# > 2500 = 2.00

# Taxa adicional 20%

kwh_used = 1700

tax1 = 0.45
tax2 = 0.74
tax3 = 1.25
tax4 = 2
addTax = 0.2

kwhValue = kwh_used
energyBill = 0

if kwhValue > 2500:
    kwhCalculated = kwhValue
    kwhCalculated -= 2500
    energyBill += kwhCalculated * tax4
    kwhValue = 2500

if kwhValue > 1500 and kwhValue <= 2500:
    kwhCalculated = kwhValue
    kwhCalculated -= 1500
    energyBill += kwhCalculated * tax3
    kwhValue = 1500

if kwhValue > 500 and kwhValue <= 1500:
    kwhCalculated = kwhValue
    kwhCalculated -= 500
    energyBill += kwhCalculated * tax2
    kwhValue = 500

if kwhValue <= 500:
    energyBill += kwhValue * tax1

energyBill += energyBill * addTax
out = energyBill
print(out)


import deepdish as dd
import lasagne
import numpy as np
import theano
import theano.tensor as T
from gensim.models import Doc2Vec

from ..utils import MovieIdToDocVec, UserIdToDocVec

class BaseSideInfoModel(object):
    def step(self, input, label, lr):
        raise NotImplementedError

class NNSideInfoModel(BaseSideInfoModel):
    """Neural Network"""
    def __init__(self, dim, reg_lambda=None, cosine_lambda=None, feature_vec_dict=None, topn=None):
        self.vector_dict = feature_vec_dict
        self.topn = topn

        input_vec = T.matrix(dtype=theano.config.floatX)
        label_vec = T.vector(dtype=theano.config.floatX)

        network = lasagne.layers.InputLayer(shape=(None, dim[0]), input_var=input_vec)
        for i in range(1, len(dim)-1):
            n_out = dim[i]
            network = lasagne.layers.DenseLayer(network, num_units=n_out)
        network = lasagne.layers.DenseLayer(network, num_units=dim[-1], nonlinearity=lasagne.nonlinearities.identity)
        label_predict = lasagne.layers.get_output(network)

        self.predict = theano.function([input_vec], label_predict, allow_input_downcast=True)
        self.param_values = lasagne.layers.get_all_param_values(network)
        self.network = network

        if reg_lambda is not None and feature_vec_dict is not None:
            loss = ((label_vec - label_predict)**2).sum()
            # cosine similarity
            loss += cosine_lambda * (1.0 - (T.sum(label_vec * label_predict)) / (T.sqrt(T.sum(T.square(label_vec))) * T.sqrt(T.sum(T.square(label_predict)))))
            loss += reg_lambda * lasagne.regularization.regularize_network_params(network, lasagne.regularization.l2)

            d_input_vec = theano.grad(loss, input_vec)
            lr = T.scalar('lr', dtype=theano.config.floatX)

            # create parameter update expressions
            params = lasagne.layers.get_all_params(network, trainable=True)
            updates = lasagne.updates.adagrad(loss, params, lr)

            self._get_loss = theano.function([input_vec, label_vec], loss, allow_input_downcast=True)

            self._gradient_step = theano.function([input_vec, label_vec, lr], outputs=[loss, d_input_vec], updates=updates, allow_input_downcast=True)

    def step(self, input_vec, user_or_item_id, lr):
        label_vec = self.vector_dict[user_or_item_id]
        loss, grad = self._gradient_step(input_vec, label_vec, lr)

        if self.topn:
            imdb_id = self.vector_dict.movie_to_imdb[user_or_item_id]
            key = '{}.txt'.format(imdb_id)
            topn_similar = self.vector_dict.store.most_similar(key, topn=self.topn)

            for k, s in topn_similar:
                l, g = self._gradient_step(input_vec, self.vector_dict.store[k], lr)
                loss += l
                grad += g

            loss /= (self.topn+1)
            grad /= (self.topn+1)
        return loss, grad

    def set_params(self, param_values):
        params_float32 = map(lambda arr: arr.astype(np.float32), param_values)
        lasagne.layers.set_all_param_values(self.network, params_float32)

    def get_loss(self, input_vec, user_or_item_id):
        label_vec = self.vector_dict[user_or_item_id]
        return self._get_loss(input_vec, label_vec)

class ATSideInfoModel(BaseSideInfoModel):
    """Affine Transformation"""
    def __init__(self, dim, reg_lambda=None, cosine_lambda=None, feature_vec_dict=None):
        self.vector_dict = feature_vec_dict

        nb_latent_f = dim[0]
        nb_d2v_features = dim[-1]

        self.weights = np.random.uniform(low=-0.001, high=0.001, size=(nb_d2v_features, nb_latent_f + 1))  # side info weight matrix
        self.ada_cache_weights = np.zeros_like(self.weights)
        self.reg_lambda = reg_lambda
        self.cosine_lambda = cosine_lambda

    def step(self, input_vec, user_or_item_id, lr):
        label_vec = self.vector_dict[user_or_item_id]

        bias = np.array([1]).reshape((-1, 1))
        biased_input_vec = np.hstack([bias, input_vec])

        label_predict = np.dot(self.weights, biased_input_vec.T).reshape((-1,))  # vector
        error = label_vec - label_predict
        loss = np.sum(np.square(error))
        loss += self.reg_lambda * np.sqrt(np.sum(np.square(self.weights)))
        #loss += self.cosine_lambda * (1.0 - (np.sum(label_vec * label_predict) /
        #                                     (np.sqrt(np.sum(np.square(label_vec))) * np.sqrt(np.sum(np.square(label_predict))))))

        delta_weights = np.dot(error.reshape((-1, 1)), biased_input_vec.reshape((1, -1))) - self.reg_lambda * self.weights
        delta_input_vec = np.dot(self.weights[:, 1:].T, error)  # without bias

        self.ada_cache_weights += delta_weights ** 2

        # update parameters
        self.weights += lr * delta_weights / (np.sqrt(self.ada_cache_weights) + 1e-6)

        return loss, delta_input_vec

    def predict(self, input_vec):
        bias = np.array([1]).reshape((-1, 1))
        biased_input_vec = np.hstack([bias, input_vec])

        label_predict = np.dot(self.weights, biased_input_vec.T).reshape((-1,))  # vector
        return label_predict

def create_si_item_model(config, ratings):
    if 'si_item_d2v_model' in config:
        d2v_model = Doc2Vec.load(config['si_item_d2v_model'])
        feature_vec_dict = MovieIdToDocVec(d2v_model.docvecs, ratings)
    elif 'si_item_vector_dict' in config:
        feature_vec_dict = dd.io.load(config['si_item_vector_dict'])

    si_item_nn = list(config['si_item_nn_hidden'])
    si_item_nn.insert(0, config['nb_latent_f'])
    si_item_nn.append(int(feature_vec_dict[config['si_item_valid_id']].shape[0]))
    config['si_item_nn'] = si_item_nn
    #si_item_model = ATSideInfoModel(config['si_item_nn'], config['si_item_reg_lambda'], feature_vec_dict=feature_vec_dict)
    si_item_model = NNSideInfoModel(config['si_item_nn'],
                                    config['si_item_reg_lambda'],
                                    config['si_item_cosine_lambda'],
                                    feature_vec_dict,
                                    config['si_item_topn_similar'])
    return si_item_model, config

def create_si_user_model(config, ratings):
    if 'si_user_d2v_model' in config:
        d2v_model = Doc2Vec.load(config['si_user_d2v_model'])
        feature_vec_dict = UserIdToDocVec(d2v_model.docvecs, ratings)
    elif 'si_user_vector_dict' in config:
        feature_vec_dict = dd.io.load(config['si_user_vector_dict'])

    si_user_nn = list(config['si_user_nn_hidden'])
    si_user_nn.insert(0, config['nb_latent_f'])
    si_user_nn.append(int(feature_vec_dict[config['si_user_valid_id']].shape[0]))
    config['si_user_nn'] = si_user_nn
    si_user_model = NNSideInfoModel(config['si_user_nn'],
                                    config['si_user_reg_lambda'],
                                    config['si_user_cosine_lambda'],
                                    feature_vec_dict)
    return si_user_model, config


"""
Metrics used for evaluating and comparing partial labels models

Heavily based on:
* https://github.com/IndicoDataSolutions/finetune/blob/development/finetune/util/metrics.py
* https://github.com/IndicoDataSolutions/benchner/blob/rd/synthetic/benchner/metrics.py

For all metric calculations below:
- A "label" or "prediction" is a dictionary of the following format:
    {"start": int, "end": int, "text": str, "label": str, "doc_idx": int}
- "true" or "predicted" generally refers to a list of label dictionaries of the format above
"""
import traceback
from collections import defaultdict, OrderedDict
import copy
import functools
import random
from typing import Callable, Dict, List, Set, Tuple, Union

# Need extra packages to use these metrics
# TODO Figure out how to incorporate spacy model download into package install
try:
    import numpy as np
    import spacy
except ImportError:
    traceback.print_exc()
    raise Exception("Using partial labels requires installing additional packages")

@functools.lru_cache()
def get_spacy():
    """
    FIXME return type of spacy.lang.en.English not working for type annotation

    Load English core language model from spacy to use for tokenization. Wrapped
    with lru_cache to ensure that the model is only loaded once.

    Returns:
        nlp: Spacy model
    """
    nlp = spacy.load("en_core_web_sm", disable=["parser", "tagger", "ner", "textcat"])
    nlp.max_length = (
        800000000  # approximately one volume of the encyclopedia britannica.
    )
    return nlp

def calc_recall(num_tps: int, num_fns: int) -> float:
    """
    Calculate recall for a class given the number of true positives and false negatives

    Args:
        num_tps: Number of true positives
        num_fns: Number of false negatives

    Returns:
        recall
    """
    try:
        return num_tps / float(num_fns + num_tps)
    except ZeroDivisionError:
        return 0.0

def calc_precision(num_tps: int, num_fps: int) -> float:
    """
    Calculate precision for a class given the number of true positives and false positives

    Args:
        num_tps: Number of true positives
        num_fps: Number of false positives

    Returns:
        precision
    """
    try:
        return num_tps / float(num_fps + num_tps)
    except ZeroDivisionError:
        return 0.0

def calc_f1(recall: float, precision: float) -> float:
    """
    Calculate F1 for a class given the recall and precision

    Args:
        recall
        precision

    Returns:
        f1
    """
    try:
        return 2 * (recall * precision) / (recall + precision)
    except ZeroDivisionError:
        return 0.0

def _get_unique_classes(
    true: List[List[Dict[str, Union[int, str]]]],
    predicted: List[List[Dict[str, Union[int, str]]]],
) -> List[str]:
    """
    Given ground truth labels and predictions, take the union of label classes
    to determine the set of unique classes

    Args:
        true: Ground truth labels for a list of documents
        predicted: Model predictions for a list of documents

    Returns:
        unique_classes: A list of unique classes
    """
    true_and_pred = list(true) + list(predicted)
    return list(set([seq["label"] for seqs in true_and_pred for seq in seqs]))

def _convert_to_token_list(
    annotations: List[Dict[str, Union[int, str]]], doc_idx: int = None
) -> List[Dict[str, Union[int, str]]]:
    """
    Given annotations (labels or predictions) for a single document, use the spacy
    English tokenizer to split the annotations into individual tokens

    Args:
        annotations: Labels or predictions for a single document
        doc_idx: Document index for current document

    Returns:
        tokens
    """
    nlp = get_spacy()
    tokens = []
    annotations = copy.deepcopy(annotations)

    for annotation in annotations:
        start_idx = annotation.get("start")
        tokens.extend(
            [
                {
                    "start": start_idx + token.idx,
                    "end": start_idx + token.idx + len(token.text),
                    "text": token.text,
                    "label": annotation.get("label"),
                    "doc_idx": doc_idx,
                }
                for token in nlp(annotation.get("text"))
            ]
        )

    return tokens

def sequence_labeling_token_conf_matrix(
    true: List[List[Dict[str, Union[int, str]]]],
    predicted: List[List[Dict[str, Union[int, str]]]],
) -> Dict[str, Dict[str, List[Dict[str, Union[int, str]]]]]:
    """
    Given ground truth labels and predictions, determine the true positives (TPs),
    false positives (FPs), and false negatives (FNs) for each class at a per token level.

    Args:
        true: Ground truth labels for a list of documents
        predicted: Model predictions for a list of documents

    Returns:
        token_conf_matrix: A dictionary containing TP/FP/FN counts for each class:
            {"classA": {"true_positives": List[Dict], "false_positives": List[Dict],
                        "false_negatives": List[Dict]}, ...}
    """
    # Determine set of unique classes, and set up token_count dict for each class
    unique_classes = _get_unique_classes(true, predicted)
    token_conf_matrix = {
        cls: {"true_positives": [], "false_positives": [], "false_negatives": []}
        for cls in unique_classes
    }

    for i, (true_list, pred_list) in enumerate(zip(true, predicted)):
        # Convert span annotations to token annotations
        true_tokens = _convert_to_token_list(true_list, doc_idx=i)
        pred_tokens = _convert_to_token_list(pred_list, doc_idx=i)

        # Calculate true positives and false negatives
        for true_token in true_tokens:
            for pred_token in pred_tokens:
                if (
                    pred_token["start"] == true_token["start"]
                    and pred_token["end"] == true_token["end"]
                ):

                    if pred_token["label"] == true_token["label"]:
                        token_conf_matrix[true_token["label"]]["true_positives"].append(
                            true_token
                        )
                    else:
                        token_conf_matrix[true_token["label"]][
                            "false_negatives"
                        ].append(true_token)
                        token_conf_matrix[pred_token["label"]][
                            "false_positives"
                        ].append(pred_token)

                    break
            else:
                token_conf_matrix[true_token["label"]]["false_negatives"].append(
                    true_token
                )

        # Calculate false positives
        for pred_token in pred_tokens:
            for true_token in true_tokens:
                if (
                    pred_token["start"] == true_token["start"]
                    and pred_token["end"] == true_token["end"]
                ):
                    break
            else:
                token_conf_matrix[pred_token["label"]]["false_positives"].append(
                    pred_token
                )

    return token_conf_matrix

def sequence_labeling_conf_matrix(
    true: List[List[Dict[str, Union[int, str]]]],
    predicted: List[List[Dict[str, Union[int, str]]]],
    equality_fn: Callable[
        [Dict[str, Union[int, str]], Dict[str, Union[int, str]]], bool
    ],
) -> Dict[str, Dict[str, List[Dict[str, Union[int, str]]]]]:
    """
    Given ground truth labels and predictions, determine the true positives (TPs),
    false positives (FPs), and false negatives (FNs) for each class.

    The method to determine TPs/FPs/FNs is based on the equality function,
    and can be exact match, overlap, or superset

    Args:
        true: Ground truth labels for a list of documents
        predicted: Model predictions for a list of documents
        equality_fn: Function to determine equality between two spans

    Returns:
        conf_matrix: A dictionary containing TP/FP/FN counts for each class:
            {"classA": {"true_positives": List[Dict], "false_positives": List[Dict],
                        "false_negatives": List[Dict]}, ...}
    """
    # Determine set of unique classes, and set up count dict for each class
    unique_classes = _get_unique_classes(true, predicted)
    conf_matrix = {
        cls: {"true_positives": [], "false_positives": [], "false_negatives": []}
        for cls in unique_classes
    }

    for i, (true_annotations, predicted_annotations) in enumerate(zip(true, predicted)):
        # Add doc idx to make verification easier
        for annotations in [true_annotations, predicted_annotations]:
            for annotation in annotations:
                annotation["doc_idx"] = i

        # Calculate true positives and false negatives
        for true_annotation in true_annotations:
            for pred_annotation in predicted_annotations:
                # Use equality function to determine match between ground truth and prediction
                if equality_fn(true_annotation, pred_annotation):
                    if pred_annotation["label"] == true_annotation["label"]:
                        conf_matrix[true_annotation["label"]]["true_positives"].append(
                            true_annotation
                        )
                        break
            else:
                conf_matrix[true_annotation["label"]]["false_negatives"].append(
                    true_annotation
                )

        # Calculate false positives
        for pred_annotation in predicted_annotations:
            for true_annotation in true_annotations:
                if (
                    equality_fn(true_annotation, pred_annotation)
                    and true_annotation["label"] == pred_annotation["label"]
                ):
                    break
            else:
                conf_matrix[pred_annotation["label"]]["false_positives"].append(
                    pred_annotation
                )

    return conf_matrix

def sequences_overlap(
    true_seq: Dict[str, Union[int, str]], pred_seq: Dict[str, Union[int, str]]
) -> bool:
    """
    Determine if two annotations overlap

    Args:
        true_seq: Ground truth label
        pred_seq: Prediction

    Both true_seq and pred_seq are label dictionaries

    Returns:
        is_match: True if sequences overlap
    """
    return true_seq["start"] < pred_seq["end"] and pred_seq["start"] < true_seq["end"]

def sequence_exact_match(
    true_seq: Dict[str, Union[int, str]], pred_seq: Dict[str, Union[int, str]]
) -> bool:
    """
    Determine if two annotations are an exact match (with stripped whitespace)

    Args:
        true_seq: Ground truth label
        pred_seq: Prediction

    Both true_seq and pred_seq are label dictionaries

    Returns:
        is_match: True if sequences are an exact match
    """
    true_seq = strip_whitespace(true_seq)
    pred_seq = strip_whitespace(pred_seq)
    return pred_seq["start"] == true_seq["start"] and pred_seq["end"] == true_seq["end"]

def sequence_superset(
    true_seq: Dict[str, Union[int, str]], pred_seq: Dict[str, Union[int, str]]
) -> bool:
    """
    Given a ground truth label and a prediction, determine if the prediction is
    a superset of the ground truth label (with stripped whitespace)

    Args:
        true_seq: Ground truth label
        pred_seq: Prediction

    Both true_seq and pred_seq are label dictionaries

    Returns:
        is_match: True if pred_seq is a superset of true_seq
    """
    true_seq = strip_whitespace(true_seq)
    pred_seq = strip_whitespace(pred_seq)
    return pred_seq["start"] <= true_seq["start"] and pred_seq["end"] >= true_seq["end"]

def get_seq_conf_matrix_fn(
    span_type: str = "token",
) -> Callable[
    [List[List[Dict[str, Union[int, str]]]], List[List[Dict[str, Union[int, str]]]]],
    Dict[str, Dict[str, List[Dict[str, Union[int, str]]]]],
]:
    """
    Determine what function to use for determining TPs/FPs/FNs based on the
    type of equality function to use.

    Args:
        span_type: Type of "span" to use to determine the equality function
            to use. Must be one of {"token", "equality", "partial", "overlap"}

    Returns:
        seq_conf_matrix_fn: Function to determine TPs/FPs/FNs
    """
    valid_span_types = {"token", "equality", "partial", "overlap"}
    if span_type not in valid_span_types:
        raise ValueError(
            f"span_type={span_type} is not valid, must be within {valid_span_types}"
        )

    span_type_fn_mapping = {
        "token": sequence_labeling_token_conf_matrix,
        "overlap": functools.partial(
            sequence_labeling_conf_matrix, equality_fn=sequences_overlap
        ),
        "exact": functools.partial(
            sequence_labeling_conf_matrix, equality_fn=sequence_exact_match
        ),
        "superset": functools.partial(
            sequence_labeling_conf_matrix, equality_fn=sequence_superset
        ),
    }
    return span_type_fn_mapping[span_type]

def strip_whitespace(
    annotation: Dict[str, Union[int, str]]
) -> Dict[str, Union[int, str]]:
    """
    Given a label annotation, strip whitespace from the start and end of the
    string and adjust the start/end indices of the annotation

    Args:
        annotation: Label annotation

    Returns:
        new_annotation: Updated annotation with stripped whitespace
    """
    label_text = annotation["text"]
    lstripped = label_text.lstrip()
    new_start = annotation["start"] + (len(label_text) - len(lstripped))
    stripped = label_text.strip()
    return {
        "text": label_text.strip(),
        "start": new_start,
        "end": new_start + len(stripped),
        "label": annotation["label"],
    }

def seq_recall(
    true: List[List[Dict[str, Union[int, str]]]],
    predicted: List[List[Dict[str, Union[int, str]]]],
    span_type: str = "token",
) -> Dict[str, float]:
    """
    Given ground truth labels and predictions, determine the recall for
    each class

    The method to determine TPs/FPs/FNs is based on the span type, which
    determines the equality function to compare ground truth and preds

    Args:
        true: Ground truth labels for a list of documents
        predicted: Model predictions for a list of documents
        span_type: Type of "span" to use to determine the equality function
            to use. Must be one of {"token", "equality", "partial", "overlap"}

    Returns:
        recalls: Recall for each class
    """
    valid_span_types = {"token", "equality", "partial", "overlap"}
    if span_type not in valid_span_types:
        raise ValueError(
            f"span_type={span_type} is not valid, must be within {valid_span_types}"
        )

    conf_matrix_fn = get_seq_conf_matrix_fn(span_type)
    # Get counts of TP/FP/FN for each class
    class_conf_matrices = conf_matrix_fn(true, predicted)
    recalls = {}
    for cls, conf_matrix in class_conf_matrices.items():
        num_tps = len(conf_matrix["true_positives"])
        num_fns = len(conf_matrix["false_negatives"])
        recalls[cls] = calc_recall(num_tps, num_fns)
    return recalls

def seq_precision(
    true: List[List[Dict[str, Union[int, str]]]],
    predicted: List[List[Dict[str, Union[int, str]]]],
    span_type: str = "token",
) -> Dict[str, float]:
    """
    Given ground truth labels and predictions, determine the precision for
    each class

    The method to determine TPs/FPs/FNs is based on the span type, which
    determines the equality function to compare ground truth and preds

    Args:
        true: Ground truth labels for a list of documents
        predicted: Model predictions for a list of documents
        span_type: Type of "span" to use to determine the equality function
            to use. Must be one of {"token", "equality", "partial", "overlap"}

    Returns:
        precisions: Precision for each class
    """
    valid_span_types = {"token", "equality", "partial", "overlap"}
    if span_type not in valid_span_types:
        raise ValueError(
            f"span_type={span_type} is not valid, must be within {valid_span_types}"
        )

    conf_matrix_fn = get_seq_conf_matrix_fn(span_type)
    # Get counts of TP/FP/FN for each class
    class_conf_matrices = conf_matrix_fn(true, predicted)
    precisions = {}
    for cls, conf_matrix in class_conf_matrices.items():
        num_tps = len(conf_matrix["true_positives"])
        num_fps = len(conf_matrix["false_positives"])
        precisions[cls] = calc_recall(num_tps, num_fps)
    return precisions

def calc_micro_f1(
    true: List[List[Dict[str, Union[int, str]]]],
    predicted: List[List[Dict[str, Union[int, str]]]],
    span_type: str = "token",
    include_recall_and_precision: bool = False,
) -> Union[Tuple[float, float, float], float]:
    """
    Given ground truth labels and predictions, determine the micro F1 score.
    Micro means calculating metrics globally by counting the total true positives,
    false negatives and false positives.

    Args:
        true: Ground truth labels for a list of documents
        predicted: Model predictions for a list of documents
        span_type: Type of "span" to use to determine the equality function
            to use. Must be one of {"token", "equality", "partial", "overlap"}
        include_recall_and_precision: If True, return a Tuple including the micro
            recall, precision, and F1. Else just return F1

    Returns:
        If include_recall_and_precision is True:
            recall
            precision
            f1
        Else:
            f1
    """
    conf_matrix_fn = get_seq_conf_matrix_fn(span_type)
    class_conf_matrices = conf_matrix_fn(true, predicted)
    true_positives, false_positives, false_negatives = 0, 0, 0
    for _, conf_matrix in class_conf_matrices.items():
        false_negatives += len(conf_matrix["false_negatives"])
        true_positives += len(conf_matrix["true_positives"])
        false_positives += len(conf_matrix["false_positives"])
    recall = calc_recall(true_positives, false_negatives)
    precision = calc_precision(true_positives, false_positives)
    f1 = calc_f1(recall, precision)
    if include_recall_and_precision:
        return recall, precision, f1
    return f1

def calc_per_class_f1(
    true: List[List[Dict[str, Union[int, str]]]],
    predicted: List[List[Dict[str, Union[int, str]]]],
    span_type: str = "token",
) -> Dict[str, Dict[str, Union[float, int]]]:
    """
    Given ground truth labels and predictions, determine the support (TPs+FNs)
    and F1 score for each class.

    Args:
        true: Ground truth labels for a list of documents
        predicted: Model predictions for a list of documents
        span_type: Type of "span" to use to determine the equality function
            to use. Must be one of {"token", "equality", "partial", "overlap"}

    Returns:
        results: OrderedDict containing support (TPs + FNs) and F1 score per class
            Note: Using Dict in return val type annotation to prevent TypeError
    """
    conf_matrix_fn = get_seq_conf_matrix_fn(span_type)
    class_conf_matrices = conf_matrix_fn(true, predicted)
    results = OrderedDict()
    for cls, conf_matrix in class_conf_matrices.items():
        results[cls] = {}
        num_tps = len(conf_matrix["true_positives"])
        num_fps = len(conf_matrix["false_positives"])
        num_fns = len(conf_matrix["false_negatives"])
        # Calculate per class precision and recall, and then F1
        recall = calc_recall(num_tps, num_fns)
        precision = calc_precision(num_tps, num_fps)
        results[cls]["support"] = num_tps + num_fns
        results[cls]["f1_score"] = calc_f1(recall, precision)
    return results

def calc_sequence_f1(
    true: List[List[Dict[str, Union[int, str]]]],
    predicted: List[List[Dict[str, Union[int, str]]]],
    span_type: str = "token",
    average: str = None,
) -> Union[float, Dict[str, Dict[str, Union[float, int]]]]:
    """
    Given ground truth labels and predictions, determine F1 score. The F1
    score can be calculated in different ways depending on the average
    parameter, which could be "micro", "macro", or "weighted" to calculate
    a single float value.

    If average is None, instead calculate per class F1 scores

    If average = None, return per-class F1 scores, otherwise
    return the requested model-level score.

    Args:
        true: Ground truth labels for a list of documents
        predicted: Model predictions for a list of documents
        span_type: Type of "span" to use to determine the equality function
            to use. Must be one of {"token", "equality", "partial", "overlap"}
        average: Metric to use to calculate F1. This is explained further in sklearn:
            https://scikit-learn.org/stable/modules/generated/sklearn.metrics.f1_score.html

    Returns:
        f1: F1 score in float form if average is "micro", "macro", or "weighted".
            Otherwise an OrderedDict with per class F1 scores
    """
    if average == "micro":
        return calc_micro_f1(true, predicted, span_type)

    f1s_by_class = calc_per_class_f1(true, predicted, span_type)
    f1s = [value.get("f1_score") for key, value in f1s_by_class.items()]
    supports = [value.get("support") for key, value in f1s_by_class.items()]

    if average == "weighted":
        return np.average(np.array(f1s), weights=np.array(supports))
    if average == "macro":
        return np.average(f1s)
    else:
        return f1s_by_class

################################################################
# All metrics below are not based on existing finetune metrics #
################################################################

def find_overlapping_sequence(
    pred: Dict[str, Union[int, str]], labels: List[Dict[str, Union[int, str]]]
) -> Union[Dict[str, Union[int, str]], None]:
    """
    Given a prediction and a set of ground truth labels, find the label sequence
    that overlaps with the prediction.

    Return None if no sequence is found

    Args:
        pred: Individual prediction (label dictionary)
        labels: List of ground truth label dictionaries for a document

    Returns:
        overlapping_label
    """
    for label in labels:
        if sequences_overlap(pred, label):
            return label

    return None

def identify_tps_fps_fns(
    true: List[List[Dict[str, Union[int, str]]]],
    predicted: List[List[Dict[str, Union[int, str]]]],
    tp_ratio: float = 1,
    fp_ratio: float = 1,
    fn_ratio: float = 1,
    random_seed: int = 42,
    tp_match_type: str = "exact",
    fp_match_type: str = "overlap",
    fn_match_type: str = "exact",
    replace_fps: bool = False,
    missing_label_ratio: float = None,
) -> Tuple[
    List[List[Dict[str, Union[int, str]]]],
    List[List[Dict[str, Union[int, str]]]],
    List[List[Dict[str, Union[int, str]]]],
]:
    """
    Given ground truth labels and predictions for a sequence labeling model,
    we want to identify the true positives, false positives, and false negatives.
    If any of the ratios (tp_ratio/fp_ratio/fn_ratio) are left at the defaults (0),
    we return an empty list. If the ratio is a float between 0 and 1, we sample
    that ratio of TPs, FPs, or FNs.

    The sequence_labeling_counts already identifies true positives, false positives,
    and false negatives, but in a significantly different format from what is expected by finetune.
    The format coming out of sequence_labeling_counts() is as follows:
    {
        <LABEL>: {
            "true_positives": int,
            "false_positives": int,
            "false_negatives": int
        },
        ...
    }

    The format expected by finetune is as follows, where annotations are separated
    for each document, and NOT separated by the different labels:
    [
        # Document 0
        [
            {
                "start": int, "end": int, "text": str, "label": str, "doc_idx": 0,
                "type": OneOf("tp", "fp", "fn")
            },
            ...
        ],
        # Document 1
        [...],
        ...
    ]

    Args:
        true: List of lists, where each sublist contains ground truth annotations
            for a given document
        predicted: List of lists, where each sublist contains predictions for a given document
        tp_ratio: Ratio of true positives to include, sampled randomly across all documents
        fp_ratio: Ratio of false positives to include, sampled randomly across all documents
        fn_ratio: Ratio of false negatives to include, sampled randomly across all documents
        random_seed: Random seed to use when sampling
        tp_match_type: Type of match in finetune metrics to use for identifying true positives.
            Must be one of ("exact", "overlap")
        fp_match_type: Same as above, but for false positives
        fn_match_type: Same as above, but for false negatives
        replace_fps: If we replace false positives with overlapping ground truth
            Very strong supervision, because we're relying on the reviewer or labeler
            to tell us the class and start/end of the overlapping ground truth
        missing_label_ratio: Used for simulating missing label classes for experiments.
            A ratio of X means that we will keep (1 - X) ratio of classes

    Returns:
        tps: List of lists, where each "sublist" includes all true positive
            annotations from a document
        fps: List of lists, where each "sublist" includes all false positive
            annotations from a document
        fns: List of lists, where each "sublist" includes all false negative
            annotations from a document
    """
    # Validate inputs
    if tp_ratio and tp_match_type not in {"exact", "overlap"}:
        raise ValueError(
            f"tp_match_type={tp_match_type} not allowed. Must be one of ('exact', 'overlap')"
        )
    if fp_ratio and fp_match_type not in {"exact", "overlap"}:
        raise ValueError(
            f"fp_match_type={fp_match_type} not allowed. Must be one of ('exact', 'overlap')"
        )
    if fn_ratio and fn_match_type not in {"exact", "overlap"}:
        raise ValueError(
            f"fn_match_type={fn_match_type} not allowed. Must be one of ('exact', 'overlap')"
        )

    equality_slc_map = {
        "exact": sequence_labeling_conf_matrix(true, predicted, sequence_exact_match),
        "overlap": sequence_labeling_conf_matrix(true, predicted, sequences_overlap),
    }

    # Get true positives and false positives in the format returned by sequence_labeling_conf_matrix
    slc_tp = equality_slc_map[tp_match_type] if tp_ratio else {}
    slc_fp = equality_slc_map[fp_match_type] if fp_ratio else {}
    slc_fn = equality_slc_map[fn_match_type] if fn_ratio else {}

    label_classes = set(list(slc_fp.keys()) + list(slc_fp.keys()) + list(slc_fn.keys()))
    # Sample labels to keep only a subset of classes if missing_label_ratio is present
    if missing_label_ratio:
        random.seed(random_seed)
        label_classes = random.sample(
            label_classes, int(len(label_classes) * (1 - missing_label_ratio))
        )

    flat_tps, flat_fps, flat_fns, = (
        [],
        [],
        [],
    )
    for label in label_classes:
        if label in slc_tp:
            flat_tps.extend(slc_tp[label]["true_positives"])
        # Sometimes we see predictions with identical start and end indices
        # It does not make sense to label these sequences since they don't
        # span any text, so skip them
        if label in slc_fp:
            flat_fps.extend(
                fp
                for fp in slc_fp[label]["false_positives"]
                if fp["start"] != fp["end"]
            )
        if label in slc_fn:
            flat_fns.extend(slc_fn[label]["false_negatives"])

    # Sample from flattened list(s)
    # Don't need to sample if ratio is 1
    # If ratio is 0, set flat_XXs back to an empty list
    if not tp_ratio:
        flat_tps = []
    elif tp_ratio != 1:
        num_samples = int(tp_ratio * len(flat_tps))
        random.seed(random_seed)
        flat_tps = random.sample(flat_tps, num_samples)

    if not fp_ratio:
        flat_fps = []
    elif fp_ratio != 1:
        num_samples = int(fp_ratio * len(flat_fps))
        random.seed(random_seed)
        flat_fps = random.sample(flat_fps, num_samples)

    if not fn_ratio:
        flat_fns = []
    elif fn_ratio != 1:
        num_samples = int(fn_ratio * len(flat_fns))
        random.seed(random_seed)
        flat_fns = random.sample(flat_fns, num_samples)

    # tps, fps, and fns should contain len(true) number of lists
    tps = [[] for _ in range(len(true))]
    fps = [[] for _ in range(len(true))]
    fns = [[] for _ in range(len(true))]

    for tp in flat_tps:
        tp["type"] = "tp"
        idx = tp["doc_idx"]
        tps[idx].append(tp)

    for fp in flat_fps:
        fp["orig_label"] = fp["label"]
        if not replace_fps:
            # This is flawed, because treating false positives as <PAD> sequences
            # may cause problems if the sequence actually is label A, but was
            # predicted as label B. In this case, we would be teaching the model the
            # incorrect behavior
            fp["label"] = "<PAD>"
        fp["type"] = "fp"
        idx = fp["doc_idx"]
        fps[idx].append(fp)

    for fn in flat_fns:
        fn["type"] = "fn"
        idx = fn["doc_idx"]
        fns[idx].append(fn)

    return tps, fps, fns

###################
# Complex metrics #
###################

def alphanumeric_lower(label_text: str) -> str:
    """
    Given text for a label, return lowercase alphanumeric characters

    Args:
        label_text

    Returns:
        label_text
    """
    return "".join(char for char in label_text if char.isalnum()).lower()

def find_many_match(
    label: Dict[str, Union[int, str]], preds: List[Dict[str, Union[int, str]]]
) -> Union[Dict[str, Union[int, str]], None]:
    """
    Check if any prediction matches with label based on checking for an
    alphanumeric lowercase string match.

    Each prediction can only match with one label (and vice versa), so
    this requires the prediction match candidate to not already be matched
    to a label.

    Returns None if there is no match

    Args:
        label: Ground truth label
        preds: List of predictions

    Returns:
        pred: Matched prediction
    """
    for pred in preds:
        if (
            alphanumeric_lower(label["text"]) == alphanumeric_lower(pred["text"])
        ) and not pred["matched"]:
            return pred

    return None

def many_conf_matrix(
    labels: List[Dict[str, Union[int, str]]], preds: List[Dict[str, Union[int, str]]]
) -> Tuple[int, int, int]:
    """
    Given ground truth labels and predictions for a given class of a document,
    compute the confusion matrix

    Args:
        labels: Labels for a document
        preds: Predictions for a document

    Returns:
        num_tps: Number of true positives for that class and document
        num_fps: : Number of false positives for that class and document
        num_fns: Number of false negatives for that class and document
    """
    # Initialize match values to False
    for label in labels:
        label["matched"] = False
    for pred in preds:
        pred["matched"] = False

    for label in labels:
        pred_match = find_many_match(label, preds)
        if pred_match:
            label["matched"] = True
            pred_match["matched"] = True

    num_tps = sum(int(label["matched"]) for label in labels)
    num_fns = sum(int(not label["matched"]) for label in labels)
    num_fps = sum(int(not pred["matched"]) for pred in preds)
    return num_tps, num_fns, num_fps

def calc_macro_metric(
    one_metric: Dict[str, float], many_metric: Dict[str, float] = {}
) -> float:
    """
    Given per class metrics for recall, precision, or F1, for both classes
    that appear once or many times per doc, take the mean of the classes to
    return a macro metric.

    Args:
        one_metric: Per class recall, precision, or F1 for labels that should
            appear max once per doc
        many_metric: Same as above, but for labels that appear many times per doc

    Returns:
        macro_metric: Macro average of recall, precision, or F1
    """

    return np.array(list(one_metric.values()) + list(many_metric.values())).mean()

def calc_micro_metric(
    one_metric: Dict[str, float],
    one_counts: Dict[str, Dict[str, int]],
    many_metric: Dict[str, float] = None,
    many_counts: Dict[str, Dict[str, int]] = None,
) -> float:
    """
    Given per class metrics for recall, precision, or F1, for both classes
    that appear once or many times per doc, use the also provided counts
    for each class to compute a weighted average that weights each class
    based on the amount of support it has.

    Args:
        one_metric: Per class recall, precision, or F1 for labels that should
            appear max once per doc
        one_counts: Per class support for one_metric labels for weighting
        many_metric: Same as one_metric, but for labels that appear many times per doc
        many_counts: Per class support for many_metric labels for weighting

    Returns:
        micro_metric: Micro average of recall, precision, or F1
    """
    metric_sum = 0
    metric_counts = 0
    for cls in one_metric:
        cls_count = one_counts[cls]["tp"] + one_counts[cls]["fn"]
        metric_sum += one_metric[cls] * cls_count
        metric_counts += cls_count
    if many_metric:
        for cls in many_metric:
            cls_count = many_counts[cls]["tp"] + many_counts[cls]["fn"]
            metric_sum += many_metric[cls] * cls_count
            metric_counts += cls_count
    return metric_sum / metric_counts

def seq_recall_precision_f1(
    true: List[List[Dict[str, Union[int, str]]]],
    predicted: List[List[Dict[str, Union[int, str]]]],
    span_type: str = "token",
) -> Tuple[
    Dict[str, float], Dict[str, float], Dict[str, float], Dict[str, Dict[str, int]]
]:
    """
    Given true and predicted labels, calculate per class precision,
    recall, F1, and and support

    Args:
        true: Sequence labeling labels for each document
        predicted: Model predictions for labels for each doc
        span_type: How to determine matches between true and prediction.
            Valid options are "token", "overlap", "exact", and "superset"

    Returns:
        recall: Per class recall
        precision: Per class precision
        f1: Per class F1
        support: FP, FN, and TP counts per class
    """
    conf_matrix_fn = get_seq_conf_matrix_fn(span_type)
    class_conf_matrices = conf_matrix_fn(true, predicted)
    recall = {}
    precision = {}
    f1 = {}
    support = {}
    for cls, conf_matrix in class_conf_matrices.items():
        num_fns = len(conf_matrix["false_negatives"])
        num_fps = len(conf_matrix["false_positives"])
        num_tps = len(conf_matrix["true_positives"])
        # Skip classes with no support
        if num_fns + num_fps + num_tps == 0:
            continue
        recall[cls] = calc_recall(num_tps, num_fns)
        precision[cls] = calc_precision(num_tps, num_fps)
        f1[cls] = calc_f1(recall[cls], precision[cls])
        support[cls] = {"fp": num_fps, "fn": num_fns, "tp": num_tps}

    return recall, precision, f1, support

def find_best_label(
    label_cls: str,
    label_candidates: List[Dict[str, Union[int, str]]],
    doc_fns: List[Dict[str, Union[int, str]]],
) -> str:
    """
    Helper function for dedup_test_labels() that is called per label per document
    any time there are multiple labels of a class that should only appear once
    per document.

    Args:
        label_cls: Name of target label class
        label_candidates: Labels of the target label class
        doc_fns: All false negatives for the current document

    Returns:
        text: Text for the chosen best label
    """
    # First try to find false negative
    for fn in doc_fns:
        if fn["label"] == label_cls:
            return fn["text"]
    # If no false negatives, default to first label
    sorted_cands = sorted(label_candidates, key=lambda x: x["start"])
    return sorted_cands[0]["text"]

def dedup_test_labels(
    doc_labels: Dict[str, List[Dict[str, Union[int, str]]]],
    doc_fns: List[Dict[str, Union[int, str]]],
) -> Dict[str, str]:
    """
    For classes that should only appear once per document, we need a way
    to choose the "best" label from review data in the case of duplicates.
    The methodology for doing so is as follows.

    1. If there are any false negatives for this class, choose the first
        one (by start index)
    2. Else, use the first value in the review set (by start index)

    Args:
        doc_labels: Dictionary with a list per label class with
            candidates for that label
        doc_fns: List of all false negatives for a doc

    Returns:
        deduped_labels: Dictionary with the best label for each class
    """
    deduped_labels = {}
    doc_fns = sorted(doc_fns, key=lambda x: x["start"])
    for label_cls, labels in doc_labels.items():
        if len(labels) == 1:
            deduped_labels[label_cls] = labels[0]["text"]
        else:
            deduped_labels[label_cls] = find_best_label(label_cls, labels, doc_fns)
    return deduped_labels

def run_evaluation(
    test_preds: List[List[Dict[str, Union[int, str]]]],
    test_labels: List[List[Dict[str, Union[int, str]]]],
    one_classes: Set[str],
    verbose: bool = False,
) -> Dict[str, float]:
    """
    Complex evaluation methodology to treat labels separately depending on if they
    should appear at most once per document or several times. This methodology
    assumes that we are using review data for our test set.
    \
    1. Separate test preds into one vs many
    2. For one, process down to a single value (no indices) based on the
        highest confidence prediction
    3. Separate test labels into one vs many
    4. Get FNs b/w test preds and labels
    5. For one, process down to max one label per document in this order:
        a) If there are any false negatives, choose the first one (by start index)
        b) Use the first value in the labels (by start index)
    6. Use existing metrics calculations to get per class precision and recall
        on "many" preds vs labels
    7. Use string matching (only alphanumeric chars, all lowercase) to determine
        TPs, FPs, and FNs for "one" preds vs labels
    8. Calculate precision and recall on "one" classes
    9. Calculate macro metrics
    10. Calculate micro metrics

    Args:
        test_preds: Predictions on the test set for each document
        test_labels: Labels on the test set for each document
        one_classes: Set of class names that should appear max once per doc
        verbose: If True, print more detailed recall/precision/F1 statistics
    """
    # 1. Separate test preds into one vs many
    # 2. For one, process down to a single value (no indices) based on the
    # highest confidence prediction
    # Note that test_preds_one is a list of dictionaries (one entry per label
    # per doc) while test_preds_many is a list of lists
    test_preds_one = []
    test_preds_many = []
    for doc_preds in test_preds:
        doc_preds_one = {}
        doc_preds_many = []
        for pred in doc_preds:
            label = pred["label"]
            if label in one_classes:
                if label in doc_preds_one:
                    # Replace existing pred in doc_preds_one if the confidence
                    # of the current pred is higher
                    conf = pred["confidence"][label]
                    if conf > doc_preds_one[label]["conf"]:
                        doc_preds_one[label] = {"text": pred["text"], "conf": conf}
                else:
                    doc_preds_one[label] = {
                        "text": pred["text"],
                        "conf": pred["confidence"][label],
                    }
            else:
                doc_preds_many.append(pred)
        test_preds_one.append(doc_preds_one)
        test_preds_many.append(doc_preds_many)

    # 3. Separate test labels into one vs many
    test_labels_one_w_dups = []
    test_labels_many = []
    for doc_labels in test_labels:
        doc_label_candidates_one = {}
        doc_labels_many = []
        for label in doc_labels:
            if label["label"] in one_classes:
                if label["label"] in doc_label_candidates_one:
                    doc_label_candidates_one[label["label"]].append(label)
                else:
                    doc_label_candidates_one[label["label"]] = [label]
            else:
                doc_labels_many.append(label)
        test_labels_one_w_dups.append(doc_label_candidates_one)
        test_labels_many.append(doc_labels_many)

    # 4. Get FNs between test preds and labels
    _, _, test_fns = identify_tps_fps_fns(
        true=test_labels,
        predicted=test_preds,
        tp_ratio=0,
        fp_ratio=0,
        fn_ratio=1,
    )

    # 5. For "one", process down to max one label per document
    test_labels_one = []
    for doc_labels, doc_fns in zip(test_labels_one_w_dups, test_fns):
        test_labels_one.append(dedup_test_labels(doc_labels, doc_fns))

    # 6. Use existing metrics calculations to get per class precision and recall
    # on "many" preds vs labels
    many_recall, many_precision, many_f1, many_counts = seq_recall_precision_f1(
        test_labels_many, test_preds_many
    )

    # 7. Use string matching (only alphanumeric chars, all lowercase) to determine
    # TPs, FPs, and FNs for "one" preds vs labels
    cm_counts_one = {cls: defaultdict(int) for cls in one_classes}
    for doc_preds, doc_labels in zip(test_preds_one, test_labels_one):
        for cls in one_classes:
            pred_text = doc_preds.get(cls, {}).get("text")
            label_text = doc_labels.get(cls, {}).get("text")
            if pred_text is not None and label_text is None:
                cm_counts_one[cls]["fp"] += 1
            elif label_text is not None and pred_text is None:
                cm_counts_one[cls]["fn"] += 1
            elif pred_text is not None and label_text is not None:
                # Process pred and label to lowercase alphanumeric chars for comparison
                pred = alphanumeric_lower(pred_text)
                label = alphanumeric_lower(label_text)
                if pred == label:
                    cm_counts_one[cls]["tp"] += 1
                else:
                    cm_counts_one[cls]["fp"] += 1
                    cm_counts_one[cls]["fn"] += 1

    # 8. Calculate precision and recall on "one" classes
    one_recall = {}
    one_precision = {}
    one_f1 = {}
    for cls, counts in cm_counts_one.items():
        # Skip classes with no support if that happens for any reason
        if counts["tp"] + counts["fp"] + counts["fn"] > 0:
            one_recall[cls] = calc_recall(counts["tp"], counts["fn"])
            one_precision[cls] = calc_precision(counts["tp"], counts["fp"])
            one_f1[cls] = calc_f1(one_recall[cls], one_precision[cls])

    # 9. Macro metrics
    macro_recall = calc_macro_metric(one_recall, many_recall)
    macro_precision = calc_macro_metric(one_precision, many_precision)
    macro_f1 = calc_macro_metric(one_f1, many_f1)

    # 10. Micro metrics
    micro_recall = calc_micro_metric(
        one_recall, cm_counts_one, many_recall, many_counts
    )
    micro_precision = calc_micro_metric(
        one_precision, cm_counts_one, many_precision, many_counts
    )
    micro_f1 = calc_micro_metric(one_f1, cm_counts_one, many_f1, many_counts)

    if verbose:
        print("One recall", one_recall)
        print("One precision", one_precision)
        print("One f1", one_f1)
        print("One support", cm_counts_one)
        print("Many recall", many_recall)
        print("Many precision", many_precision)
        print("Many f1", many_f1)
        print("Many support", many_counts)

    final_metrics = {
        "macro_recall": macro_recall,
        "macro_precision": macro_precision,
        "macro_f1": macro_f1,
        "micro_recall": micro_recall,
        "micro_precision": micro_precision,
        "micro_f1": micro_f1,
    }
    return final_metrics

def eval_value_matching(
    test_preds: List[List[Dict[str, Union[int, str]]]],
    test_labels: List[List[Dict[str, Union[int, str]]]],
    one_classes: Set[str],
    all_classes: Set[str],
    verbose: bool = False,
) -> Dict[str, float]:
    """
    Complex evaluation methodology to treat labels separately depending on if they
    should appear at most once per document or several times.

    This function is identical to run_evaluation(), except that for this comparison,
    we compare values directly instead of using start/end indices.

    1. Separate test preds into one vs many
    2. For one, process down to a single value (no indices) based on the
        highest confidence prediction
    3. Separate test labels into one vs many
    4. Get FNs between test preds and labels by doing string matches
    5. For "one", process each class down to a set of labels using the following logic:
        a) If there are any false negatives, create a set of them
        b) Else, create a set of all of the labels of that class
    6. For "many" preds vs labels, use string matching (only alphanumeric chars,
        all lowercase) to get TPs, FPs, and FNs
    7. Calculate precision and recall on "many" classes
    8. Apply the same string matching technique to "one" preds vs labels
    9. Calculate precision and recall on "one" classes
    10. Calculate micro metrics
    11. Calculate macro metrics

    Args:
        test_preds: Predictions on the test set for each document
        test_labels: Labels on the test set for each document
        one_classes: Set of class names that should appear max once per doc
        all_classes: Set of all classes that can appear for this workflow
        verbose: If True, print more detailed recall/precision/F1 statistics

    Returns:
        eval_metrics: Micro and macro precision, recall, and F1
    """
    # 1. Separate test preds into one vs many
    # 2. For one, process down to a single value (no indices) for each class
    # based on the highest confidence prediction
    # For many, process down to a list of labels for each class
    test_preds_one = []
    test_preds_many = []
    for doc_preds in test_preds:
        doc_preds_one = {}
        doc_preds_many = defaultdict(list)
        for pred in doc_preds:
            label = pred["label"]
            if label in one_classes:
                if label in doc_preds_one:
                    # Replace existing pred in doc_preds_one if the confidence
                    # of the current pred is higher
                    conf = pred["confidence"][label]
                    if conf > doc_preds_one[label]["conf"]:
                        doc_preds_one[label] = {"text": pred["text"], "conf": conf}
                else:
                    doc_preds_one[label] = {
                        "text": pred["text"],
                        "conf": pred["confidence"][label],
                    }
            else:
                doc_preds_many[label].append(pred)
        test_preds_one.append(doc_preds_one)
        test_preds_many.append(doc_preds_many)

    # 3. Separate test labels into one vs many
    # "one" has a set for each class for each doc. Each element of the set
    #   is just alphanumeric lowercase text
    # "many" has a list of the full label dicts for each class in each doc
    test_labels_one_w_dups = []
    test_labels_many = []
    for doc_labels in test_labels:
        doc_label_candidates_one = defaultdict(set)
        doc_labels_many = defaultdict(list)
        for label in doc_labels:
            if label["label"] in one_classes:
                doc_label_candidates_one[label["label"]].add(
                    alphanumeric_lower(label["text"])
                )
            else:
                doc_labels_many[label["label"]].append(label)
        test_labels_one_w_dups.append(doc_label_candidates_one)
        test_labels_many.append(doc_labels_many)

    # 4. Get FNs between test preds and labels by doing string matches
    # Each element of test_fns is a defaultdict from label class name
    # to a set of the false negatives for the label in the current doc
    test_fns = []
    for doc_labels, doc_preds in zip(test_labels, test_preds):
        doc_fns = defaultdict(set)
        doc_pred_strs = set(alphanumeric_lower(pred["text"]) for pred in doc_preds)
        for label in doc_labels:
            label_str = alphanumeric_lower(label["text"])
            # If this condition is met, label is a false negative
            if label_str not in doc_pred_strs:
                doc_fns[label["label"]].add(label_str)
        test_fns.append(doc_fns)

    # 5. For "one", process each class down to a set of labels using the following logic:
    #   a) If there are any false negatives, create a set of them
    #   b) Else, create a set of all of the labels of that class
    test_labels_one = []
    for doc_labels, doc_fns in zip(test_labels_one_w_dups, test_fns):
        doc_labels_one = {}
        for cls in one_classes:
            if doc_fns.get(cls):
                doc_labels_one[cls] = doc_fns[cls]
            elif doc_labels_one.get(cls):
                doc_labels_one[cls] = doc_labels[cls]
        test_labels_one.append(doc_labels_one)

    # 6. For "many" preds vs labels, use string matching (only alphanumeric chars,
    # all lowercase) to get TPs, FPs, and FNs
    many_classes = all_classes - one_classes
    cm_counts_many = {cls: defaultdict(int) for cls in many_classes}
    for doc_preds, doc_labels in zip(test_preds_many, test_labels_many):
        for cls in many_classes:
            tps, fps, fns = many_conf_matrix(
                doc_labels.get(cls, []), doc_preds.get(cls, [])
            )
            cm_counts_many[cls]["tp"] += tps
            cm_counts_many[cls]["fp"] += fps
            cm_counts_many[cls]["fn"] += fns

    # 7. Calculate precision and recall on "many" classes
    many_recall = {}
    many_precision = {}
    many_f1 = {}
    for cls, counts in cm_counts_many.items():
        # Skip classes with no support if that happens for any reason
        if counts["tp"] + counts["fp"] + counts["fn"] > 0:
            many_recall[cls] = calc_recall(counts["tp"], counts["fn"])
            many_precision[cls] = calc_precision(counts["tp"], counts["fp"])
            many_f1[cls] = calc_f1(many_recall[cls], many_precision[cls])

    # 8. Use string matching (only alphanumeric chars, all lowercase) to determine
    # TPs, FPs, and FNs for "one" preds vs labels
    cm_counts_one = {cls: defaultdict(int) for cls in one_classes}
    for doc_preds, doc_labels in zip(test_preds_one, test_labels_one):
        for cls in one_classes:
            pred = doc_preds.get(cls)
            labels = doc_labels.get(cls, {})
            if pred is not None and labels == {}:
                cm_counts_one[cls]["fp"] += 1
            elif pred is None and labels != {}:
                cm_counts_one[cls]["fn"] += 1
            elif pred is not None and labels != {}:
                # Check if pred is in the set of labels or not
                if alphanumeric_lower(pred["text"]) in labels:
                    cm_counts_one[cls]["tp"] += 1
                # Even if the set of labels has multiple "wrong" labels, we only
                # treat it as a single error, because ideally we would have a
                # constraint in place to only have a single prediction for labels
                # that should appear max once per document
                else:
                    cm_counts_one[cls]["fp"] += 1
                    cm_counts_one[cls]["fn"] += 1

    # 9. Calculate precision and recall on "one" classes
    one_recall = {}
    one_precision = {}
    one_f1 = {}
    for cls, counts in cm_counts_one.items():
        # Skip classes with no support if that happens for any reason
        if counts["tp"] + counts["fp"] + counts["fn"] > 0:
            one_recall[cls] = calc_recall(counts["tp"], counts["fn"])
            one_precision[cls] = calc_precision(counts["tp"], counts["fp"])
            one_f1[cls] = calc_f1(one_recall[cls], one_precision[cls])

    # 10. Macro metrics
    macro_recall = calc_macro_metric(one_recall, many_recall)
    macro_precision = calc_macro_metric(one_precision, many_precision)
    macro_f1 = calc_macro_metric(one_f1, many_f1)

    # 11. Micro metrics
    micro_recall = calc_micro_metric(
        one_recall, cm_counts_one, many_recall, cm_counts_many
    )
    micro_precision = calc_micro_metric(
        one_precision, cm_counts_one, many_precision, cm_counts_many
    )
    micro_f1 = calc_micro_metric(one_f1, cm_counts_one, many_f1, cm_counts_many)

    if verbose:
        print("One recall", one_recall)
        print("One precision", one_precision)
        print("One f1", one_f1)
        print("One support", cm_counts_one)
        print("Many recall", many_recall)
        print("Many precision", many_precision)
        print("Many f1", many_f1)
        print("Many support", cm_counts_many)

    final_metrics = {
        "macro_recall": macro_recall,
        "macro_precision": macro_precision,
        "macro_f1": macro_f1,
        "micro_recall": micro_recall,
        "micro_precision": micro_precision,
        "micro_f1": micro_f1,
    }
    return final_metrics


import csv
import json

# Function to convert a CSV to JSON
# Takes the file paths as arguments
def make_json(csvFilePath, jsonFilePath,out):

    # create a dictionary
    data = {}

    # Open a csv reader called DictReader
    with open(csvFilePath, encoding='utf-8') as csvf:
        csvReader = csv.DictReader(csvf)

        # Convert each row into a dictionary
        # and add it to data
        for rows in csvReader:

            # Assuming a column named 'No' to
            # be the primary key
            key = int(rows["no"])

            rows["inning"]=int(rows["inning"])
            rows["over"]=int(rows["over"])
            rows["ball"]=int(rows["ball"])
            rows["is_super_over"]=int(rows["is_super_over"])
            rows["wide_runs"]=int(rows["wide_runs"])
            rows["legbye_runs"]=int(rows["legbye_runs"])
            rows["noball_runs"]=int(rows["noball_runs"])
            rows["penalty_runs"]=int(rows["penalty_runs"])
            rows["batsman_runs"]=int(rows["batsman_runs"])
            rows["extra_runs"]=int(rows["extra_runs"])
            rows["total_runs"]=int(rows["total_runs"])
            # rows["match_id"]=int(rows["match_id"])
            rows["bye_runs"]=int(rows["bye_runs"])
            out.append(rows)
            data[key] = rows

    # Open a json writer, and use the json.dumps()
    # function to dump data
    with open(jsonFilePath, 'w', encoding='utf-8') as jsonf:
        jsonf.write(json.dumps(data, indent=4))

    with open(jsonFilePath_new, 'w', encoding='utf-8') as jsonf:
        jsonf.write(json.dumps(out))

# Driver Code

# Decide the two file paths according to your
# computer system
csvFilePath = r'deliveries.csv'
jsonFilePath = r'final_deliveries.json'
jsonFilePath_new = r'final_out_deliveries.json'

# Call the make_json function
out=[]
make_json(csvFilePath, jsonFilePath,out)


from mod_exp import mod_exp
def mod_division(a, b, p):
  """
    Compute a / b % p, where p is prime.
    Undefined if p is not prime
  """
  if b % p == 0:
    raise ZeroDivisionError("division by 0")

  # modular inverse, i.e. b * inv = 1 mod p
  inv = mod_exp(b, p - 2, p)
  return ((a % p) * inv) % p


import json
import tokenize
import token
import io

paths = {
    "/app" : "Beacon",
	"/app/view" : "Bn.view",
	"/app/controller" : "Bn.controller",
	"/app/app" : "Bn.app",
    "/app/manager" : "Beacon.manager",
    "/app/model" : "Beacon.model",
    "/app/store" : "Beacon.store",
    "/app/mobile/custom" : "Beacon.custom",
    "/app/desktop/custom" : "Beacon.custom",
    "/app/mobile/view" : "Beacon.view",
    "/app/desktop/view" : "Beacon.view",
    "/app/mobile/controller" : "Beacon.controller",
    "/app/desktop/controller" : "Beacon.controller",
    "/app/mobile/profile" : "Beacon.profile",
    "Ext.custom" : "Ext.custom",
    "utils": "Utils",
    "/app/custom" : 'Bn.custom'
}

def namify(path):
	path = '/' + path
	lowerCasePath = path.replace('\\','/').lower()
	found = ""
	usingProp = ""
	foundDif = 9999999
	pathLen = len(path)
	for prop in paths:
		if lowerCasePath.find(prop.lower()) != -1:
			if (abs(len(prop) - pathLen) < foundDif):
				found = paths[prop]
				usingProp = prop
				foundDif = abs(len(prop) - pathLen)
	start = lowerCasePath.find(usingProp.lower())
    #return path
	return found + path[start+len(usingProp):len(path)].replace('/','.').replace('\\','.').replace('.js','')
    #return found
	#    return path[start:len(path)].replace('/','.').replace('\\','.').replace('.js','')


# -*- coding: utf-8 -*-
"""
Created on Mon Nov  9 17:03:48 2020

@author: Kislay
"""
import numpy as np
import csv

ins = open( "D:/data/your_file.txt", "r" )

array = []
for line in ins:
    line = line.rstrip('\n')
    array.append( line )
ins.close()
data=[[]]

final=[[0]*134]*5000
for item in array :
         data.append(item.split(','))
data.pop(0)
data=np.array(data)
final=np.array(final)
for i in range(4920):
    for j in range(1,18):
        final[i][0]=data[i][0]
        final[i][int(data[i][j])]=1
print(final)

with open("D:/data/new_file.csv","w+") as my_csv:
    csvWriter = csv.writer(my_csv,delimiter=',')
    csvWriter.writerows(final)


import pandas as pd
from numerize import numerize

data = {'TOP10': ['United States','China','Japan','Germany','India','United Kingdom','France','Italy','Brazil','Canada']}
df = pd.DataFrame(data)

print(df)

upperCaseNames = [x.upper() for x in df["TOP10"]]
print(upperCaseNames)

forwardAndReverse = [(x.upper(),x.upper()[::-1]) for x in df["TOP10"]]
print(forwardAndReverse)

data = {'Country Name': ['United States','China','Japan','Germany','India','United Kingdom','France','Italy','Brazil','Canada'],
        'GDP 2021 Est. Trillions':[22675271000000,16642318000000,5378136000000,4319286000000,3124650000000,3049704000000,2938271000000,2106287000000,1883487000000,1806707000000]}
df = pd.DataFrame(data, index = [1,2,3,4,5,6,7,8,9,10])
print(df)

df["Country Name"] = [x.upper() for x in df["Country Name"]]
print(df)

df["GDP 2021 Est. Trillions"] = [ numerize.numerize(x) for x in df["GDP 2021 Est. Trillions"]]
print(df)


import copy
import time

import torch
import torch.nn as nn
import utils

from .impl import iterative_unlearn

def discretize(x):
    return torch.round(x * 255) / 255

def FGSM_perturb(x, y, model=None, bound=None, criterion=None):
    device = model.parameters().__next__().device
    model.zero_grad()
    x_adv = x.detach().clone().requires_grad_(True).to(device)

    pred = model(x_adv)
    loss = criterion(pred, y)
    loss.backward()

    grad_sign = x_adv.grad.data.detach().sign()
    x_adv = x_adv + grad_sign * bound
    x_adv = discretize(torch.clamp(x_adv, 0.0, 1.0))

    return x_adv.detach()

@iterative_unlearn
def boundary_shrink_iter(
    data_loaders, model, criterion, optimizer, epoch, args, mask=None, test_model=None
):
    assert test_model is not None

    bound = 0.1  # hard coding in the paper

    train_loader = data_loaders["forget"]
    print(len(train_loader))
    losses = utils.AverageMeter()
    top1 = utils.AverageMeter()

    # switch to train mode
    model.train()

    start = time.time()

    for i, (image, target) in enumerate(train_loader):
        if epoch < args.warmup:
            utils.warmup_lr(
                epoch, i + 1, optimizer, one_epoch_step=len(train_loader), args=args
            )

        image = image.cuda()
        target = target.cuda()

        test_model.eval()
        image_adv = FGSM_perturb(
            image, target, model=test_model, bound=bound, criterion=criterion
        )

        adv_outputs = test_model(image_adv)
        adv_label = torch.argmax(adv_outputs, dim=1)

        # compute output
        output_clean = model(image)
        loss = criterion(output_clean, adv_label)

        optimizer.zero_grad()
        loss.backward()

        if mask:
            for name, param in model.named_parameters():
                if param.grad is not None:
                    param.grad *= mask[name]
                    # print(mask[name])

        optimizer.step()

        output = output_clean.float()
        loss = loss.float()
        # measure accuracy and record loss
        prec1 = utils.accuracy(output.data, target)[0]

        losses.update(loss.item(), image.size(0))
        top1.update(prec1.item(), image.size(0))

        if (i + 1) % args.print_freq == 0:
            end = time.time()
            print(
                "Epoch: [{0}][{1}/{2}]\t"
                "Loss {loss.val:.4f} ({loss.avg:.4f})\t"
                "Accuracy {top1.val:.3f} ({top1.avg:.3f})\t"
                "Time {3:.2f}".format(
                    epoch, i, len(train_loader), end - start, loss=losses, top1=top1
                )
            )
            start = time.time()

    print("train_accuracy {top1.avg:.3f}".format(top1=top1))

    return top1.avg

def boundary_shrink(data_loaders, model: nn.Module, criterion, args, mask=None):
    device = model.parameters().__next__().device
    test_model = copy.deepcopy(model).to(device)
    return boundary_shrink_iter(
        data_loaders, model, criterion, args, mask, test_model=test_model
    )


from collections import Counter

from data.bible.training_data import read_bible, get_corresponding_sentences_in_bible
from embedding.embedder import word2vec
from constants import MAX_GENERATION_SIZE

nirv = list(read_bible(translations=['NIRV']).values())
sents = []
s = []
for sent in nirv:
    for w in sent.split():
        s.append(w)
        if w in '.?!':
            sents.append(s)
            s = []

VOCAB_THRESH = 500 # over all translations
SENT_LEN = 20

common = list(filter(lambda x: all(map(lambda y:y in word2vec and word2vec.vocab[y].count >= VOCAB_THRESH, x)), sents))

# print(sorted(common, key=len)[-1])

short = list(" ".join(sent) for sent in filter(lambda x: len(x) <= SENT_LEN, common))

# print('Vocab size:', len(set([w for s in short for w in s])))
# print('Sentence lengths:', Counter(map(len, short)))
# print(len(sents), len(common), len(short))
pairs = get_corresponding_sentences_in_bible('NIV', 'NIRV')
pairs = list(filter(lambda x:all(map(lambda y: len(y.split()) <= MAX_GENERATION_SIZE, x)), pairs))
common_pairs = list(filter(lambda z: all(map(lambda x: all(map(lambda y:y in word2vec and word2vec.vocab[y].count >= VOCAB_THRESH, x.split())), z)), pairs))
sentences = list(sum(pairs, ()))

"""
niv = list(read_bible(translations=['NIV']).values())
all_trans = read_bible(flatten_translations=True, filter_none=True)
sentence_lengths_niv = Counter(map(lambda x: len(x.split()), niv))
sentence_lengths_all = Counter(map(lambda x: len(x.split()), all_trans))
"""


'''
1 2 3  >    4 1
4 5 6       5 2
            6 3

current_column = max_column - row
current_row = col
'''
def get_dd_matrix(rows, columns):
    return [[0 for x in range(columns)] for y in range(rows)]

def rotate_matrix(matrix):
    row = len(matrix)
    col = len(matrix[0])

    new_matrix = get_dd_matrix(col,row)

    for each_row in range(row):
        for each_col in range(col):
            new_matrix[each_col][row-each_row-1] = matrix[each_row][each_col]
    return new_matrix

matrix = [[1,2,3,4],
          [5,6,7,8],
          [9,10,11,12]]

print (rotate_matrix(matrix))


class DeploymentConfiguration(object):
    def __init__(self, context_environment_vars):
        '''
        From
        https://cloud.google.com/deployment-manager/docs/configuration/templates/use-environment-variables

        Keys allowed for built-in environment variables related to deployment using deployment manager

        deployment:	The name of the deployment.
        name:	The name declared in the configuration that is using the template. This can be useful if you want the
                name you declare in the configuration to be the name of the resource in the underlying templates.
        project: The project ID for this deployment.
        project_number: The project number for this deployment.
        current_time:   The UTC timestamp when expansion started for the deployment.
        type:	The resource type declared in the top-level configuration.
        username:	The current Deployment Manager user.

        For example:
        >>gcloud config list
        [core]
        account = first.last@domain.com
        disable_usage_reporting = False
        project = some-project
        [gcloudignore]
        enabled = true
        Your active configuration is: [default]

        >>gcloud deployment-manager deployments create deploy-cmdline-name --config dm.yaml

        would result in:

        "deployment": "deploy-cmdline-name"
        "project": "some-project"
        "current_time": 1592015636
        "project_number": "423452468050"
        "username": "first.last@domain.com"
        NOTE:  Dependent on which template calls generate_config()
        "name": "function"
        "type": "dmt_cloud_function.py"

        '''
        self.deployment = context_environment_vars['deployment']
        self.name = context_environment_vars['name']
        self.deployment_gcp_project = context_environment_vars['project']
        self.deployment_gcp_project_number = context_environment_vars['project_number']
        self.deployment_utc_time = context_environment_vars['current_time']
        self.type = context_environment_vars['type']
        self.username = context_environment_vars['username']

class GaFlattenerDeploymentConfiguration(DeploymentConfiguration):
    def __init__(self, context_environment_vars):
        super(GaFlattenerDeploymentConfiguration, self).__init__(context_environment_vars)
        self.FILTER = '''
        resource.type="bigquery_dataset"
        protoPayload.methodName="google.cloud.bigquery.v2.JobService.InsertJob"
        protoPayload.authenticationInfo.principalEmail="firebase-measurement@system.gserviceaccount.com"
        severity: "NOTICE"
        NOT "events_intraday_"
        '''
        self.FILTER_INTRADAY = '''
        resource.type="bigquery_resource"
        protoPayload.methodName="tableservice.insert" OR protoPayload.methodName="tableservice.delete"
        "events_intraday_"
        protoPayload.authenticationInfo.principalEmail="firebase-measurement@system.gserviceaccount.com"
        NOT severity="error"
        '''
        # we exclude severity="error" in intraday log filter, because we only want successful creations of intraday table
        # there are logs about unsuccessful creation of this table: "table already exists" -
        # they cause the intraday CF to fail:
        # File "/workspace/main.py", line 160, in manage_intraday_schedule
        #     input_event = InputValidatorIntraday(event)
        #   File "/workspace/main.py", line 22, in __init__
        #     self.gcp_project = bq_destination_table['projectId']
        # KeyError: 'projectId'
        # this is because the log about unsuccessful creation of the intraday nested table is not the right log for us
        # it doesn't have table information in the API response part

        self.gcp_resource_name_limit = 63
        """As of 2021-11-09, there are the following chars limits:
            Deployment name         63
            CF name                 63
            GCS bucket              63
            Pub / Sub topic id      255
            Log sink name           100

           Deployment name will determine the names/ids of GCP resources.
           This constant will help truncate deployment name, so the length of GCP resources is valid,
           to prevent installation errors
            """

        self.user_environment_variables = {
            "CONFIG_BUCKET_NAME": self.get_bucket_name(),
            "CONFIG_FILENAME": "config_datasets.json",
            "EVENTS": "events",
            "EVENT_PARAMS": "event_params",
            "USER_PROPERTIES": "user_properties",
            "ITEMS": "items",
            "LOCATION_ID": "us-central1",
            "TOPIC_NAME": self.get_topic_id()
        }

    def get_project(self):
        return self.deployment_gcp_project

    def get_project_number(self):
        return self.deployment_gcp_project_number

    def get_filter(self, intraday=False):
        if intraday:
            return self.FILTER_INTRADAY
        else:
            return self.FILTER

    def get_sink_name(self, intraday=False):
        '''
        As long as deployment name is valid, sink name will also be valid
        As of 2021-11-09:
            Deployment name can be up to 63 chars long.
            Sink name can be up to 100 chars long.
        '''
        if intraday:
            return f"{self._create_valid_gcp_resource_name(self.deployment)}-sink-intraday"
        else:
            return f"{self._create_valid_gcp_resource_name(self.deployment)}-sink"

    def get_topic_id(self, intraday=False):
        '''
        Returns topic id
        As long as deployment name is valid, topic id will also be valid
        As of 2021-11-09:
            Deployment name can be up to 63 chars long.
            Pub/Sub topic id can be up to 255 chars long.
        '''
        if intraday:
            return f"{self._create_valid_gcp_resource_name(self.deployment)}-topic-intraday"
        else:
            return f"{self._create_valid_gcp_resource_name(self.deployment)}-topic"

    def get_cf_name(self, code_location):
        """
        As of 2021-11-09:
            Deployment name can be up to 63 chars long.
            CF name can also be up to 63 chars long.

        We will be appending a string to deployment name to get a bucket name.
        We need to truncate deployment name, so installation doesn't fail.
        """
        fn = code_location[:-1]

        # -1 because of delimiter hyphen
        trunc_deployment_name_length = self.gcp_resource_name_limit - len(fn) - 1

        return f"{self.deployment[:trunc_deployment_name_length]}-{fn}"

    def get_bucket_name(self):
        """
        As of 2021-11-09:
            Deployment name can be up to 63 chars long.
            GCS bucket name can also be up to 63 chars long.

        We will be appending a string to deployment name to get a bucket name.
        We need to truncate deployment name, so installation doesn't fail.
        """
        suffix = "adswerve-ga-flat-config"
        number = self._create_valid_gcp_resource_name(self.get_project_number())

        # -2 because we have 2 additional hyphens which are delimiters between deployment, name and
        trunc_deployment_name_length = self.gcp_resource_name_limit - len(suffix) - len(number) - 2

        deployment = self._create_valid_gcp_resource_name(self.deployment)[:trunc_deployment_name_length]

        return f"{deployment}-{number}-{suffix}"

    def _create_valid_gcp_resource_name(self, p_field):
        '''
        GCP resources must only contain letters, numbers, underscores, dots or dashes
        and be between 3 and 63 chars long.  Resources must start with a letter and may not end with a dash
        :param p_field: starting point of the field
        :return: cleaned big query field name
        '''
        r = ""
        for char in p_field.lower():
            if char.isalnum():
                r += char
            else:
                r += "-"
        return r


# class Solution:
#     def removeElement(self, nums, val) -> int:
#         i = 0
#         while i < len(nums) :
#             if nums[i] == val :
#                 nums.remove(nums[i])
#             else :
#                 i = i+1
#         return len(nums)

# s = Solution()
nums = [3,2,2,3]
val=3
# print(s.removeElement(nums,val))
nums2=[]

a = len(nums)
for i in range(a):
    if nums[i] == val :
        continue
    else:
        nums2.append(nums[i])
print(nums2)
print(len(nums2))


from rest_framework import permissions
from rest_framework.authtoken.models import Token

from api.models import User

class AdminAuthentication(permissions.BasePermission):

    def has_permission(self, request, view):
        try:
            token = request.headers['token']
        except:
            token = ''

        token_query = Token.objects.filter(key=request.headers['token']).first()
        user = token_query.user
        return user.is_admin


import pandas as pd
from import_pack.import_data import *
from stats_pack.stats_features import stats_features

def get_averages():
    df = grab_epl_data()
    df['margin'] = df['FTHG'] - df['FTAG']
    df['Date'] = pd.to_datetime(df['Date'])
    df = df.loc[df['Date'] > '2022/07/01']

    df_home = df.groupby(df['HomeTeam']).mean()
    df_home['THC'] = df_home['HY'] + df_home['HR']
    df_home = df_home[['HST', 'THC', 'HC']]

    df_away = df.groupby(df['AwayTeam']).mean()
    df_away['TAC'] = df_away['AY'] + df_away['AR']
    df_away = df_away[['AST', 'TAC', 'AC']]

    return df_home, df_away


def replace_spaces_and_commas(string):
    # Split the string into words
    words = string.split()

    # If there are exactly two words
    if len(words) == 2:
        # Reverse the order and join with a slash
        new_string = "/".join(words[::-1])
    elif len(words) > 2:
        # If there are more than two words, last one should go to first with '/' like 'word/' and rest having spaces should replaced with '-' so final string can be 'word/word-word'
        new_string = words[-1] + "/"
        for i in range(len(words) - 1):
            new_string += words[i] + "-"
        new_string = new_string[:-1]

    else:
        # If there are more than two words, replace the comma with a dash
        new_string = string.replace(",", "-")

    if new_string[0] == "/":
        print(f"new stirng = {new_string}")
        # remove it
        new_string = new_string[1] + new_string[2:]

    print(f"new stirng = {new_string}")

    return new_string

print(replace_spaces_and_commas("East Orange NJ"))


import csv
import numpy as np
import random
import Levenshtein as ls
from py_stringmatching import GeneralizedJaccard, Soundex

from data import Dataset

class EquivalenceClasses:
    def __init__(self, path):
        self.dataset = Dataset('train.csv')
        self.contents = []
        self.gj = GeneralizedJaccard(sim_func=Soundex().get_raw_score)

        with open(path, 'r', encoding='utf-8') as f:
            reader = csv.reader(f)
            contents = []
            for line in reader:
                elements = []
                for element in line[1:]:
                    elements.append(self.dataset.rows[int(element)])
                contents.append(elements)
            self.contents = [content for content in contents if len(content) != 0]

    def calcEquivalentSsn(self):
        print('ssn-eq')
        ssn_info = []
        for content in self.contents:
            for i in range(len(content) - 2):
                ssn1 = content[i].__getitem__('ssn')
                if ssn1 != '000000000':
                    for j in range(i + 1, len(content) - 1):
                        ssn2 = content[j].__getitem__('ssn')
                        if ssn2 != '000000000':
                            ssn_info.append(ls.ratio(ssn1, ssn2))
        narray = np.array(ssn_info)
        print('mean: ' + str(np.mean(narray)))
        print('max: ' + str(np.max(narray)))
        print('min: ' + str(np.min(narray)))

    def calcInequivalentSsn(self):
        print('ssn-neq')
        ssn_info = []
        random_array = self.generateRandomArray(len(self.contents) - 2, 100)
        for i in random_array:
            ssn1 = self.contents[i][0].__getitem__('ssn')
            if ssn1 != '000000000':
                for j in range(i + 1, len(self.contents) - 1):
                    ssn2 = self.contents[j][0].__getitem__('ssn')
                    if ssn2 != '000000000':
                        ssn_info.append(ls.ratio(ssn1, ssn2))
        narray = np.array(ssn_info)
        print('mean: ' + str(np.mean(narray)))
        print('max: ' + str(np.max(narray)))
        print('min: ' + str(np.min(narray)))

    def calcEquivalentName(self):
        print('name-eq')
        name_info = []

        for content in self.contents:
            for i in range(len(content) - 2):
                name1 = [content[i].__getitem__('fname'), content[i].__getitem__('lname')]
                if len(content[i].__getitem__('minit')) != 0:
                    name1.append(content[i].__getitem__('minit'))
                for j in range(i + 1, len(content) - 1):
                    name2 = [content[j].__getitem__('fname'), content[j].__getitem__('lname')]
                    if len(content[j].__getitem__('minit')) != 0:
                        name2.append(content[j].__getitem__('minit'))
                    name_info.append(self.gj.get_sim_score(name1, name2))
        narray = np.array(name_info)
        print('mean: ' + str(np.mean(narray)))
        print('max: ' + str(np.max(narray)))
        print('min: ' + str(np.min(narray)))

    def calcInequivalentName(self):
        print('name-neq')
        name_info = []
        random_array = self.generateRandomArray(len(self.contents) - 2, 10)
        print(random_array)
        for i in random_array:
            name1 = [self.contents[i][0].__getitem__('fname'), self.contents[i][0].__getitem__('lname')]
            if len(self.contents[i][0].__getitem__('minit')) != 0:
                name1.append(self.contents[i][0].__getitem__('minit'))
            print(name1)
            for j in range(i + 1, len(self.contents) - 1):
                name2 = [self.contents[j][0].__getitem__('fname'), self.contents[j][0].__getitem__('lname')]
                if len(self.contents[j][0].__getitem__('minit')) != 0:
                    name2.append(self.contents[j][0].__getitem__('minit'))
                name_info.append(self.gj.get_sim_score(name1, name2))
        narray = np.array(name_info)
        print('mean: ' + str(np.mean(narray)))
        print('max: ' + str(np.max(narray)))
        print('min: ' + str(np.min(narray)))

    def calcEquivalentStadd(self):
        print('stadd-eq')
        stadd_info = []
        for content in self.contents:
            for i in range(len(content) - 2):
                stadd1 = content[i].__getitem__('stadd').split(' ')[0]
                for j in range(i + 1, len(content) - 1):
                    stadd2 = content[j].__getitem__('stadd').split(' ')[0]
                    stadd_info.append(ls.ratio(stadd1, stadd2))
        narray = np.array(stadd_info)
        print('mean: ' + str(np.mean(narray)))
        print('max: ' + str(np.max(narray)))
        print('min: ' + str(np.min(narray)))

    def calcInequivalentStadd(self):
        print('stadd-neq')
        stadd_info = []
        random_array = self.generateRandomArray(len(self.contents) - 2, 100)
        for i in random_array:
            stadd1 = self.contents[i][0].__getitem__('stadd').split(' ')[0]
            for j in range(i + 1, len(self.contents) - 1):
                stadd2 = self.contents[j][0].__getitem__('stadd').split(' ')[0]
                stadd_info.append(ls.ratio(stadd1, stadd2))
        narray = np.array(stadd_info)
        print('mean: ' + str(np.mean(narray)))
        print('max: ' + str(np.max(narray)))
        print('min: ' + str(np.min(narray)))

    def calcEquivalentOtherAttributes(self, attr_name):
        print(attr_name + '-eq')
        attr_info = []
        for content in self.contents:
            for i in range(len(content) - 2):
                attr1 = content[i].__getitem__(attr_name)
                for j in range(i + 1, len(content) - 1):
                    attr2 = content[j].__getitem__(attr_name)
                    attr_info.append(ls.ratio(attr1, attr2))
        narray = np.array(attr_info)
        print('mean: ' + str(np.mean(narray)))
        print('max: ' + str(np.max(narray)))
        print('min: ' + str(np.min(narray)))

    def calcInequivalentOtherAttributes(self, attr_name):
        print(attr_name + '-neq')
        attr_info = []
        random_array = self.generateRandomArray(len(self.contents) - 2, 100)
        for i in random_array:
            attr1 = self.contents[i][0].__getitem__(attr_name)
            for j in range(i + 1, len(self.contents) - 1):
                attr2 = self.contents[j][0].__getitem__(attr_name)
                attr_info.append(ls.ratio(attr1, attr2))
        narray = np.array(attr_info)
        print('mean: ' + str(np.mean(narray)))
        print('max: ' + str(np.max(narray)))
        print('min: ' + str(np.min(narray)))

    def generateRandomArray(self, max_value, number):
        random_array = []
        for i in range(0, number):
            random_array.append(random.randint(0, max_value))
        return set(random_array)

def main():
    equivalenceClasses = EquivalenceClasses('bm_real.csv')

    equivalenceClasses.calcEquivalentSsn()
    equivalenceClasses.calcInequivalentSsn()

    equivalenceClasses.calcEquivalentName()
    equivalenceClasses.calcInequivalentName()

    equivalenceClasses.calcEquivalentOtherAttributes('stnum')
    equivalenceClasses.calcInequivalentOtherAttributes('stnum')

    equivalenceClasses.calcEquivalentStadd()
    equivalenceClasses.calcInequivalentStadd()

    equivalenceClasses.calcEquivalentOtherAttributes('apmt')
    equivalenceClasses.calcInequivalentOtherAttributes('apmt')

    equivalenceClasses.calcEquivalentOtherAttributes('city')
    equivalenceClasses.calcInequivalentOtherAttributes('city')

    equivalenceClasses.calcEquivalentOtherAttributes('state')
    equivalenceClasses.calcInequivalentOtherAttributes('state')

    equivalenceClasses.calcEquivalentOtherAttributes('zip')
    equivalenceClasses.calcInequivalentOtherAttributes('zip')

if __name__ == '__main__':
    main()