#  Data Science
#  Term project - Cricket World Cup Matches Prediction
#  Name            :  Jo Byeong Geun, Choi Ji Won, Jeong Da Hee
#  Student Number  :  201835528, 201835538, 201931889
#  Class           :  Mon 5678
#  Date            :  2021.05.26
#  Update          :  2021.06.11

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import MinMaxScaler, StandardScaler, RobustScaler
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression, LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import confusion_matrix, plot_confusion_matrix, classification_report
from sklearn.model_selection import KFold
from sklearn.ensemble import RandomForestClassifier
from sklearn import metrics
from sklearn import svm, datasets
from warnings import simplefilter
import warnings
with warnings.catch_warnings():
    warnings.filterwarnings("ignore")

simplefilter(action='ignore', category=FutureWarning)

# Make Confusion Matirx
def make_cf_matrix(y_test, y_pred):
    # Compare y testing sets with predicted y values
    cf_matrix = confusion_matrix(y_test, y_pred)
    # Show the confusion Matrix
    print("\n***************Confusion Matrix******************")
    print(cf_matrix)

# Make Classification Report
def make_cf_report(y_test, y_pred):
    # Evaluate the precision and reproducibility
    cf_report = classification_report(y_test, y_pred)
    # Show the Classificatin Report
    print("\n***************Classification Report******************")
    print(cf_report)

# Scaling data from wicketts, runs
def scaler_team(df_team, df, scaler):
    value = []
    for i in df_team.index:
        # Importing data in the 'Margin' column
        value.append(df.loc[i]['Margin'])
    # Change dimension and shape
    value = np.array(value).reshape(-1, 1)
    scaled = scaler.transform(value)
    # Scaling to mean values
    return np.mean(scaled)

# Scaling using scaler
def scaling(scaler):
    wickets_np = df_wickets['Margin'].to_numpy().reshape(-1, 1)
    scaler.fit(wickets_np)

    # Declaring a list to insert values for scaling
    wicket_scaler = []
    wicket_scaler.append(scaler_team(df_England_wicket, df_wickets, scaler))
    wicket_scaler.append(scaler_team(df_SouthAfrica_wicket, df_wickets, scaler))
    wicket_scaler.append(scaler_team(df_WestIndies_wicket, df_wickets, scaler))
    wicket_scaler.append(scaler_team(df_Pakistan_wicket, df_wickets, scaler))
    wicket_scaler.append(scaler_team(df_NewZealand_wicket, df_wickets, scaler))
    wicket_scaler.append(scaler_team(df_SriLanka_wicket, df_wickets, scaler))
    wicket_scaler.append(scaler_team(df_Afghanistan_wicket, df_wickets, scaler))
    wicket_scaler.append(scaler_team(df_Australia_wicket, df_wickets, scaler))
    wicket_scaler.append(scaler_team(df_Bangladesh_wicket, df_wickets, scaler))
    wicket_scaler.append(scaler_team(df_India_wicket, df_wickets, scaler))

    runs_np = df_runs['Margin'].to_numpy().reshape(-1, 1)
    scaler.fit(runs_np)
    # Declaring a list to insert values for scaling
    runs_scaler = []
    runs_scaler.append(scaler_team(df_England_runs, df_runs, scaler))
    runs_scaler.append(scaler_team(df_SouthAfrica_runs, df_runs, scaler))
    runs_scaler.append(scaler_team(df_WestIndies_runs, df_runs, scaler))
    runs_scaler.append(scaler_team(df_Pakistan_runs, df_runs, scaler))
    runs_scaler.append(scaler_team(df_NewZealand_runs, df_runs, scaler))
    runs_scaler.append(scaler_team(df_SriLanka_runs, df_runs, scaler))
    runs_scaler.append(scaler_team(df_Afghanistan_runs, df_runs, scaler))
    runs_scaler.append(scaler_team(df_Australia_runs, df_runs, scaler))
    runs_scaler.append(scaler_team(df_Bangladesh_runs, df_runs, scaler))
    runs_scaler.append(scaler_team(df_India_runs, df_runs, scaler))

    # Outputs scaling results for worldcup_team in bar form
    plt.bar(worldcup_teams, wicket_scaler)
    # X axis name
    plt.xlabel('Teams')
    # Y axis name
    plt.ylabel('Scaled Wicket')
    # Show plotting result
    plt.show()
    # Outputs scaling results for worldcup_team in bar form
    plt.bar(worldcup_teams, runs_scaler)
    # X axis name
    plt.xlabel('Teams')
    # Y axis name
    plt.ylabel('Scaled Runs')
    # Show plotting result
    plt.show()

    return wicket_scaler, runs_scaler

# Kfold- Linear Regression
def kfold_lin(kfold, df_teams):
    sum_training = 0
    sum_testing = 0
    count = 0
    for train_index, test_index in kfold.split(df_teams):
        # Data for training
        df_train = df_teams.iloc[train_index]
        # Data for testing
        df_test = df_teams.iloc[test_index]
        model = LinearRegression()
        # Change the training set and testing set to the correct structure and store them.
        train = pd.get_dummies(df_train, prefix=['Team_1', 'Team_2'], columns=['Team_1', 'Team_2'])
        test = pd.get_dummies(df_test, prefix=['Team_1', 'Team_2'], columns=['Team_1', 'Team_2'])
        # Drop 'Winner' column of training set and put it in the X training set.
        X_train = train.drop(['Winner'], axis=1)
        # Insert the 'Winner' column of training set into the y training set.
        y_train = train["Winner"]
        # Drop 'Winner' column of testing set and put it in the X testing set.
        X_test = test.drop(['Winner'], axis=1)
        # Insert the 'Winner' column of testing set into the y testing set.
        y_test = test["Winner"]
        model.fit(X_train, y_train)
        # Derives the value of learning the training set to Linear Regression
        score = model.score(X_train, y_train)
        # Derives the value of learning the testing set to Linear Regression
        score2 = model.score(X_test, y_test)
        count = count + 1
        sum_training = sum_training + score
        sum_testing = sum_testing + score2
        # Accuracy output each K-fold data
        print("Training set accuracy: ", '%.3f' % (score))
        print("Test set accuracy: ", '%.3f' % (score2))
    print("Training set accuracy average: ", '%.3f' % (sum_training / count))
    print("Test set accuracy average: ", '%.3f' % (sum_testing / count))
    return sum_testing/count

# Kfold - Logistic Regression
def kfold_log(kfold,df_teams):
    sum_training = 0
    sum_testing = 0
    count = 0
    for train_index, test_index in kfold.split(df_teams):
        # Data for training
        df_train = df_teams.iloc[train_index]
        # Data for testing
        df_test = df_teams.iloc[test_index]
        model = LogisticRegression()
        # Change the training set and testing set to the correct structure and store them.
        train = pd.get_dummies(df_train, prefix=['Team_1', 'Team_2'], columns=['Team_1', 'Team_2'])
        test = pd.get_dummies(df_test, prefix=['Team_1', 'Team_2'], columns=['Team_1', 'Team_2'])
        # Drop 'Winner' column of training set and put it in the X training set.
        X_train = train.drop(['Winner'], axis=1)
        # Insert the 'Winner' column of training set into the y training set.
        y_train = train["Winner"]
        # Drop 'Winner' column of testing set and put it in the X testing set.
        X_test = test.drop(['Winner'], axis=1)
        # Insert the 'Winner' column of testing set into the y testing set.
        y_test = test["Winner"]
        model.fit(X_train, y_train)
        # Derives the value of learning the training set to Logistic Regression
        score = model.score(X_train, y_train)
        # Derives the value of learning the testing set to Logistic Regression
        score2 = model.score(X_test, y_test)
        count = count + 1
        sum_training = sum_training + score
        sum_testing = sum_testing + score2
        # Accuracy output each K-fold data
        print("Training set accuracy: ", '%.3f' % (score))
        print("Test set accuracy: ", '%.3f' % (score2))
    print("Training set accuracy average: ", '%.3f' % (sum_training / count))
    print("Test set accuracy average: ", '%.3f' % (sum_testing / count))
    return sum_testing/count

# Kfold - Random Forest
def kfold_ran(kfold, best_score, df_teams):
    sum_training=0
    sum_testing=0
    count=0
    for train_index, test_index in kfold.split(df_teams):
        # Data for training
        df_train = df_teams.iloc[train_index]
        # Data for testing
        df_test = df_teams.iloc[test_index]
        # Change the training set and testing set to the correct structure and store them.
        train = pd.get_dummies(df_train, prefix=['Team_1', 'Team_2'], columns=['Team_1', 'Team_2'])
        test = pd.get_dummies(df_test, prefix=['Team_1', 'Team_2'], columns=['Team_1', 'Team_2'])

        # Drop 'Winner' column of training set and put it in the X training set.
        X_train = train.drop(['Winner'], axis=1)
        # Insert the 'Winner' column of training set into the y training set.
        y_train = train["Winner"]
        # Drop 'Winner' column of testing set and put it in the X testing set.
        X_test = test.drop(['Winner'], axis=1)
        # Insert the 'Winner' column of testing set into the y testing set.
        y_test = test["Winner"]
        # Random Forest
        rf = RandomForestClassifier(n_estimators=100, max_depth=20, random_state=0)
        # Learning X training set by y training
        rf.fit(X_train, y_train)
        # Derives the value of learning the training set to Random Forest
        score = rf.score(X_train, y_train)
        # Derives the value of learning the testing set to Random Forest
        score2 = rf.score(X_test, y_test)
        count=count+1
        sum_training=sum_training+score
        sum_testing=sum_testing+score2

        # Store the Random Forest of the best test value that have learned
        if best_score < score2:
            best_score = score2
            best_rf = rf

        # Accuracy output each K-fold data
        print("Training set accuracy: ", '%.3f' % (score))
        print("Test set accuracy: ", '%.3f' % (score2))

    print("Training set accuracy average: ", '%.3f' % (sum_training/count))
    print("Test set accuracy average: ", '%.3f' % (sum_testing/count))
    return sum_testing/count, best_rf

# Label Encoding
def labelEncoding(df_teams):
        df_teams = df_teams.apply(LabelEncoder().fit_transform)
        return df_teams

# Best accuracy of K-Fold & algorithm (Linear, Logistic and RandomForest)
def best_kfold_reg(df_teams):
    #K-fold Linear Regression
    print("\n*************Kfold of Linear Regression***************")
    scores = []
    kfold = KFold(n_splits=3, shuffle=True)
    scaled_kfold_lin = kfold_lin(kfold, df_teams)

    #K-fold Logistic Regression
    print("\n*************Kfold of Logistic Regression***************")
    scores = []
    kfold = KFold(n_splits=3, shuffle=True)
    df_teams['Winner']=df_teams['Winner'].astype(int)
    scaled_kfold_log = kfold_log(kfold,df_teams)

    # Kfold random forest
    # Split data into three
    print("\n*************Kfold of Random Forest ***************")
    kfold = KFold(n_splits=3, shuffle=True)
    # Declaring a variable to put the best value in
    best_score = 0
    scaled_kfold_ran, best_rf = kfold_ran(kfold, best_score, df_teams)

    best_acc = bestAcc(scaled_kfold_lin, scaled_kfold_log, scaled_kfold_ran)
    #print("\nthe best acc = ", '%0.3f' % best_acc)

    return best_acc

# Calculate best accuracy
def bestAcc(lin, log, ran):
    return max(lin, max(log, ran))

# Best combination
def best_combi(df_teams_minmax, df_teams_standard, df_teams_robust):
    return max(best_kfold_reg(df_teams_minmax), max(best_kfold_reg(df_teams_standard), best_kfold_reg(df_teams_robust)))

# Read the csv file
results = pd.read_csv('dataset/results.csv')

# dataset statistical data
print("\n***********statistical data***************")
print(results.describe())
# Feature names
print("\n***********feature names***************")
print(results.columns.values)
# data types
print("\n***********feature data types***************")
print(results.dtypes)
# data shape
print("\n***********data shape***************")
print(results.shape)
# data index
print("\n***********data index***************")
print(results.index)
# dataset columns
print("\n***********dataset columns***************")
print(results.columns)
# Top 5 data in dataset
print("\n***********dataset***************")
print(results.head())

# data preprocessing
# Drop missing null or ground information that is not needed
results.dropna(axis=0, inplace=True)
results.drop(['Ground'], axis=1, inplace=True)
# Limited to 10 teams participating in the World Cup
worldcup_teams = ['England', 'South Africa', 'West Indies',
                  'Pakistan', 'New Zealand', 'Sri Lanka', 'Afghanistan',
                  'Australia', 'Bangladesh', 'India']

# Include only countries corresponding to worldcup_teams in results dataset of column 'Team_1'
df_teams_1 = results[results['Team_1'].isin(worldcup_teams)]
# Include only countries corresponding to worldcup_teams in results dataset of column 'Team_2'
df_teams = df_teams_1[df_teams_1['Team_2'].isin(worldcup_teams)]
# deduplication
df_teams = df_teams.drop_duplicates()
# Clean up index
df_teams = df_teams.sort_index(ascending=True)

# Show df_teams dataset
print("\n***********df_teams dataset***************")
print(df_teams.head())
# data shape
print("\n***********df_teams data shape***************")
print(df_teams.shape)

# If the defense team wins
# data with a value of 'Wicket' in the 'Margin' column will be separately saved.
df_wickets = df_teams.loc[df_teams['Margin'].str.contains('wickets')].copy()
# To omit the word 'wickets' from the value and only insert numbers (int types)
df_wickets['Margin'] = df_wickets.Margin.str.extract('(\d+)')
# Create time series data using the 'data' value
df_wickets['year'] = pd.DatetimeIndex(df_wickets['date']).year

df_England_wicket = df_wickets[df_wickets['Winner'] == worldcup_teams[0]]
df_SouthAfrica_wicket = df_wickets[df_wickets['Winner'] == worldcup_teams[1]]
df_WestIndies_wicket = df_wickets[df_wickets['Winner'] == worldcup_teams[2]]
df_Pakistan_wicket = df_wickets[df_wickets['Winner'] == worldcup_teams[3]]
df_NewZealand_wicket = df_wickets[df_wickets['Winner'] == worldcup_teams[4]]
df_SriLanka_wicket = df_wickets[df_wickets['Winner'] == worldcup_teams[5]]
df_Afghanistan_wicket = df_wickets[df_wickets['Winner'] == worldcup_teams[6]]
df_Australia_wicket = df_wickets[df_wickets['Winner'] == worldcup_teams[7]]
df_Bangladesh_wicket = df_wickets[df_wickets['Winner'] == worldcup_teams[8]]
df_India_wicket = df_wickets[df_wickets['Winner'] == worldcup_teams[9]]

# If the offense team wins
# data with a value of 'runs' in the 'Margin' column will be separately saved.
df_runs = df_teams.loc[df_teams['Margin'].str.contains('runs')].copy()
# To omit the word 'runs' from the value and only insert numbers (int types)
df_runs['Margin'] = df_runs.Margin.str.extract('(\d+)')
# Create time series data using the 'data' value
df_runs['year'] = pd.DatetimeIndex(df_runs['date']).year

df_England_runs = df_runs[df_runs['Winner'] == worldcup_teams[0]]
df_SouthAfrica_runs = df_runs[df_runs['Winner'] == worldcup_teams[1]]
df_WestIndies_runs = df_runs[df_runs['Winner'] == worldcup_teams[2]]
df_Pakistan_runs = df_runs[df_runs['Winner'] == worldcup_teams[3]]
df_NewZealand_runs = df_runs[df_runs['Winner'] == worldcup_teams[4]]
df_SriLanka_runs = df_runs[df_runs['Winner'] == worldcup_teams[5]]
df_Afghanistan_runs = df_runs[df_runs['Winner'] == worldcup_teams[6]]
df_Australia_runs = df_runs[df_runs['Winner'] == worldcup_teams[7]]
df_Bangladesh_runs = df_runs[df_runs['Winner'] == worldcup_teams[8]]
df_India_runs = df_runs[df_runs['Winner'] == worldcup_teams[9]]

########### Scaling ###########
# MinMax Scaling
minmax_scaler = MinMaxScaler()
minmax_wicket_scaler, minmax_runs_scaler = scaling(minmax_scaler)

# Standard Scaling
standard_scaler = StandardScaler()
standard_wicket_scaler, standard_runs_scaler = scaling(standard_scaler)

# Robust Scaling
robust_scaler = RobustScaler()
robust_wicket_scaler, robust_runs_scaler = scaling(robust_scaler)

# Drop columns of 'Margin' and 'date' that has nothing to do with winning or losing.
df_teams = df_teams.drop(['date', 'Margin'], axis=1)
# reorder index
df_teams = df_teams.reset_index(drop=True)

########## Encoding ###########
# LabelEncoding to change team name to number
df_teams = labelEncoding(df_teams)

# Divide the categorical properties for Team_1 and Team_2.
final = pd.get_dummies(df_teams, prefix=['Team_1', 'Team_2'], columns=['Team_1', 'Team_2'])
# Drop 'Winner' column from final dataset and put it in X
X = final.drop(['Winner'], axis=1)
# Store only 'Winner' column in y in final dataset
y = final["Winner"]
# Divide training and testing set by 7:3 each.
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.30, random_state=42)

# Random Forest
rf = RandomForestClassifier(n_estimators=100, max_depth=20, random_state=0)
# Learning X training set by y training
rf.fit(X_train, y_train)
# Predicting an X testing set with learning results
y_pred = rf.predict(X_test)
# Derives the value of learning the training set to Random Forest
score = rf.score(X_train, y_train)
# Derives the value of learning the testing set to Random Forest
score2 = rf.score(X_test, y_test)
# Show Random Forest
print("\n***************Random Forest******************")
print('Mean Absolute Error:', metrics.mean_absolute_error(y_test, y_pred))
print('Mean Squared Error:', metrics.mean_squared_error(y_test, y_pred))
print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(y_test, y_pred)))
# Accuracy output
print("Training set accuracy: ", '%.3f' % (score))
print("Test set accuracy: ", '%.3f' % (score2))

# K neareast neighbor
k_list = range(1,10)
training_accuracy = []
test_accuracy = []
for k in k_list:
  classifier = KNeighborsClassifier(n_neighbors = k)
  # Learning X training set by y training
  classifier.fit(X_train, y_train)
  # Store accuracy of training set
  training_accuracy.append(classifier.score(X_train, y_train))
  # Store accuracy of testing set
  test_accuracy.append(classifier.score(X_test, y_test))

# Outputs K neareast neighbor result
plt.plot(k_list, training_accuracy)
# X axis name
plt.xlabel("n_neighbors")
# Y axis name
plt.ylabel("Accuracy")
# Show plotting result
plt.show()

# Prediction of testing set
clf = KNeighborsClassifier(n_neighbors = 7)
# Learning X training set by y training
clf.fit(X_train, y_train)
# Show the prediction and accuracy of testing set
print("\n*********************KNN***********************")
print("Test Prediction: {}".format(clf.predict(X_test)))
print("Test Accuracy: {:.2f}".format(clf.score(X_test, y_test)))

# Linear Regression
linreg = LinearRegression()
# Learning X training set by y training
linreg.fit(X_train, y_train)
# Predicting an X testing set with learning results
y_pred = linreg.predict(X_test)
# Derives the value of learning the training set to Linear regression
score = linreg.score(X_train, y_train)
# Derives the value of learning the testing set to Linear regression
score2 = linreg.score(X_test, y_test)
# Show Linear Regression
print("\n***************Linear Regression******************")
print('Mean Absolute Error:', metrics.mean_absolute_error(y_test, y_pred))
print('Mean Squared Error:', metrics.mean_squared_error(y_test, y_pred))
print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(y_test, y_pred)))
# Accuracy output
print("Training set accuracy: ", '%.3f' % (score))
print("Test set accuracy: ", '%.3f' % (score2))

# Logistic Regression
logreg = LogisticRegression()
# Learning X training set by y training
logreg.fit(X_train, y_train)
# Predicting an X testing set with learning results
y_pred = logreg.predict(X_test)
# Derives the value of learning the training set to Logistic regression
score = logreg.score(X_train, y_train)
# Derives the value of learning the testing set to Logistic regression
score2 = logreg.score(X_test, y_test)
# Show Logistic Regression
print("\n******************Logistic Regression*********************")
print('Mean Absolute Error:', metrics.mean_absolute_error(y_test, y_pred))
print('Mean Squared Error:', metrics.mean_squared_error(y_test, y_pred))
print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(y_test, y_pred)))
# Accuracy output
print("Training set accuracy: ", '%.3f' % (score))
print("Test set accuracy: ", '%.3f' % (score2))
# Show Confusion Matrix and Classification Report
make_cf_matrix(y_test, y_pred)
make_cf_report( y_test, y_pred)

# Confusion matrix Visualization
cf_matrix = confusion_matrix(y_test, y_pred)
# Support Vector classifier
clf = svm.SVC(random_state=0)
# Learning X training set by y training
clf.fit(X_train, y_train)
svm.SVC(random_state=0)
# Decide how to show the matrix
matrix = plot_confusion_matrix(clf, X_test, y_test, cmap=plt.cm.Blues, normalize='true')
# Title of matrix table
plt.title('Confusion matrix for our classifier')
# Show the matrix table
plt.show()

Scaler = MinMaxScaler()
df_teams_minmax = pd.DataFrame(Scaler.fit_transform(df_teams), columns=['Team_1', 'Team_2', 'Winner'])
print(df_teams_minmax)

Scaler = StandardScaler()
df_teams_standard = pd.DataFrame(Scaler.fit_transform(df_teams), columns=['Team_1', 'Team_2', 'Winner'])
print(df_teams_standard)

Scaler = RobustScaler()
df_teams_robust = pd.DataFrame(Scaler.fit_transform(df_teams), columns=['Team_1', 'Team_2', 'Winner'])
print(df_teams_robust)

best_combi_acc = best_combi(df_teams_minmax, df_teams_standard, df_teams_robust)
print("\nbest combination accuracy : ", best_combi_acc)

# Expected qualifier rank
# Read the csv file
ranking = pd.read_csv('dataset/icc_rankings.csv')
fixtures = pd.read_csv('dataset/fixtures.csv')
# Show Top 5 data of fixtures dataset
print("\n***************icc_rankings dataset******************")
print(ranking.head())
# Show last 5 data of fixtures dataset
print("\n***************Fixtures dataset******************")
print(fixtures.tail())

# Ranked using icc_ranking dataset
fixtures.insert(1, 'first_position', fixtures['Team_1'].map(ranking.set_index('Team')['Position']))
fixtures.insert(2, 'second_position', fixtures['Team_2'].map(ranking.set_index('Team')['Position']))
# Find the value through the integral position
fixtures = fixtures.iloc[:45, :]
# Show last 5 data of fixtures dataset with ranking
print("\n***************fixtures dataset with ranking******************")
print(fixtures.tail())

# Declaring a variable to put the predicted value
pred_ranking_set = []
# The first variable is indexed, and the second variable is accessed one by one in the row in the column.
for idx, row in fixtures.iterrows():
    # Change the value of team_1 or team_2 according to rank
    if row['first_position'] < row['second_position']:
        pred_ranking_set.append({'Team_1': row['Team_1'], 'Team_2': row['Team_2'], 'Winner': None})
    else:
        pred_ranking_set.append({'Team_1': row['Team_2'], 'Team_2': row['Team_1'], 'Winner': None})
# Convert DataFrame type
pred_ranking_set = pd.DataFrame(pred_ranking_set)
# Top 5 data in dataset
print("\n***************Predicted dataset******************")
print(pred_ranking_set.head())

# Declares variables for backing up data in 'df_teams'
backup_df_teams = df_teams
# Divide the categorical properties for Team_1 and Team_2.
backup_final = pd.get_dummies(backup_df_teams, prefix=['Team_1', 'Team_2'], columns=['Team_1', 'Team_2'])
# Declares variables for backing up data in 'pred_set'
backup_pred_set = pred_ranking_set

# Divide the categorical properties for Team_1 and Team_2.
pred_ranking_set = pd.get_dummies(pred_ranking_set, prefix=['Team_1', 'Team_2'], columns=['Team_1', 'Team_2'])
# Store the differences between existing dataset and predicted dataset
missing_cols = set(backup_final.columns) - set(pred_ranking_set.columns)
# Store missing value as 0 in predicted dataset
for c in missing_cols:
    pred_ranking_set[c] = 0
# Insert data of the column in 'backup_final' into the predicted dataset
pred_ranking_set = pred_ranking_set[backup_final.columns]
# Drop the 'Winner' column
pred_ranking_set = pred_ranking_set.drop(['Winner'], axis=1)

# Put the dataset to experiment into the random forest to derive the result value
best_rf = RandomForestClassifier(n_estimators=100, max_depth=20, random_state=0)
best_rf.fit(X_train, y_train)
predict_result = best_rf.predict(pred_ranking_set)
# Show expected qualifier rank
print("\n*************Qualifier***************")
for i in range(fixtures.shape[0]):
    # Show the country to compete with
    print(backup_pred_set.iloc[i, 1] + " and " + backup_pred_set.iloc[i, 0])
    if predict_result[i] == 1:
        # The second written country wins
        print("Winner: " + backup_pred_set.iloc[i, 1])
    else:
        # The first written country wins
        print("Winner: " + backup_pred_set.iloc[i, 0])
    print("")


# EDITED: 8-29-20
import serial
import os
import time
#import context  # Ensures paho is in PYTHONPATH
import paho.mqtt.client as MQTT # For MQTT publish/subscribe
import RPi.GPIO as GPIO # For input/output
#from tkinter import *

import pygame.mixer
from sys import exit

import pygame, sys
from pygame.locals import *

#slideWinAFlag = False

#pygame.init()

#import signal

#ser = serial.Serial('/dev/ttyUSB-wire',9600,timeout=1)
#ser = serial.Serial('/dev/ttyUSB-toggle',9600,timeout=1)
#ser = serial.Serial('/dev/ttyUSB-finale',9600,timeout=1)
#ser1 = serial.Serial('/dev/ttyACM0',9600,timeout=.1)
#ser1 = serial.Serial('/dev/ttyUSB-mega',9600,timeout=1)
#ser1 = serial.Serial('/dev/ttyUSB-finale',9600,timeout=1)
##ser1.flushInput()

def printString( str ):
   #"This prints a passed string into this function"
    length = len(str)
    for x in range (length):
        print(x , " is " , (str[x]))
    return;

def convertString(str ):
   #"This take in a string of ASCII chars, converts to Chars and strips last 2 elements"
    stringLen = len(str)
    realLen = stringLen - 2

    tempStr = ""

    for x in range (realLen):
        temp = chr(str[x])
        tempStr = tempStr + temp
    str = tempStr
    return str;
#read from Arduino
# Callback after connecting to MQTT server
def on_connect(self, client, userdata, rc):
  print("Connected!")
  print("Subscribing to ToDevice/All")
  self.subscribe("ToDevice/All")
  print("Subscribing to ToDevice/1")
  self.subscribe("phone/#")
  print("Subscribing to phone")

# Callback when message is published to MQTT subscribed topic
def on_message(client, userdata, msg):
  # Print the message and topic in which it was published
  print(msg.topic + " " + str(msg.payload))
  # Change the global puzzle state based on message received
  global state
  #temp = int(msg.payload)
  temp = int(msg.payload)

# Callback when client is disconnected from MQTT broker
def on_disconnect(client, userdata, rc):
    notConnected = True
    try:
        client.connect("10.1.10.177", 1883, 60)
        notConnected = False
    except:
        time.sleep(1)
        notConnected = True
  #client.loop_stop()

# Setup MQTT
client = MQTT.Client()
# Assign callbacks
client.on_connect = on_connect
client.on_message = on_message
client.on_disconnect = on_disconnect
# Connect to MQTT server
#client.connect("10.0.0.142", 1883, 60)
notConnected = True
try:
    client.connect("10.1.10.177", 1883, 60)
    notConnected = False
except:
    time.sleep(1)
    notConnected = True

#client.connect("10.1.10.177", 1883, 60)

# Start the MQTT update loop
client.loop_start()

print("Main Loop Start")

# Main program loop ==============================================================================================================================

def newMain():
    while True:
        time.sleep(500)

if __name__ == '__main__':
    newMain()


from collections import namedtuple

Point = namedtuple("Point", ["x", "y"])
RectSize = namedtuple("RectSize", ["width", "height"])
Rect = namedtuple("Rect", ["left", "top", "width", "height"])

def rect_to_xxyy(rect):
    return [
        rect.left,
        rect.left + rect.width,
        rect.top,
        rect.top + rect.height,
    ]

def xxyy_to_rect(x1, x2, y1, y2):
    return Rect(
        left=min(x1, x2),
        top=min(y1, y2),
        width=abs(x1 - x2),
        height=abs(y1 - y2),
    )

def rect_from_2points(point1, point2):
    x1, y1 = point1
    x2, y2 = point2
    return xxyy_to_rect(x1, x2, y1, y2)


import base64
import json
import subprocess
import logging
import os
from datetime import datetime, timezone
import time
import requests
import schedule
import toml

class ConsumerMonitor():
    def __init__(self,
                 config_file: str="config.toml"):
        """
        Read in the config file settings.
        """
        with open(config_file, "r") as config_toml:
            config = toml.load(config_toml)
        try:
            self.MONITOR_PERIOD = int(config['monitor_period'])
            self.CHAIN_BINARY = config['gaia_path']
            self.HERMES_BINARY = config['hermes_path']
            self.HERMES_CONFIG = config['hermes_config_path']
            self.CHAIN_ID = config['chain_id']
            self.API_NODE = config['api']
            self.RPC_NODE = config['rpc']
            self.GRPC_NODE = config['grpc']
            self.ALARM_DELAY = int(config['matured_packet_grace_period'])
            self.MATRIX_HOMESERVER_URL = config['homeserver']
            self.MATRIX_ROOM = config['room']
            self.MATRIX_TOKEN = config['token']
            self.RECORD_FILE = config['record_file']
        except:
            logging.error("Configuration file could not be processed.")

        self.send_blocks = []
        self.recv_txs = []
        self.chain_vsc = {}

    def get_unbonding_periods(self):
        """
        Read the unbonding periods for each chain using the Gaia binary.
        """
        logging.info("Collecting consumer chain unbonding periods")
        for chain in self.chain_vsc.keys():
            result = subprocess.run([f"{self.CHAIN_BINARY}",
                    "q", "provider", "consumer-genesis", chain,
                    "--output",
                    "json",
                    "--node",
                    self.RPC_NODE],
                    stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                    text=True)
            try:
                result.check_returncode()
                ccv = json.loads(result.stdout)
                unbonding_period = int(ccv['params']['unbonding_period'][:-1])
                alarm_period = unbonding_period + self.ALARM_DELAY
                self.chain_vsc[chain]['unbonding_period'] = unbonding_period
                self.chain_vsc[chain]['alarm_period'] = alarm_period

            except subprocess.CalledProcessError as cpe:
                logging.error("Exception:", result.stderr)

    def get_connection_id(self, client_id):
        """
        Get the connection id for the consumer chain using Hermes.
        """
        result = subprocess.run([f"{ self.HERMES_BINARY }",
                "--json",
                "--config", self.HERMES_CONFIG,
                "query", "client", "connections",
                "--chain", self.CHAIN_ID, "--client",
                client_id],
                stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                text=True)
        lines = result.stdout
        connection_json = json.loads(lines.split('\n')[-2])
        connection_id = connection_json['result'][0]
        return connection_id

    def get_channel_id(self, connection_id):
        """
        Get the connection id for the consumer chain using Hermes.
        """
        result = subprocess.run([f"{self.HERMES_BINARY}",
                        "--json",
                        "--config", self.HERMES_CONFIG,
                        "query", "connection", "channels",
                        "--chain", self.CHAIN_ID, "--connection",
                        connection_id],
                        stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                        text=True)
        lines = result.stdout
        channel_json = json.loads(lines.split('\n')[-2])
        for channel in channel_json['result']:
            if channel['port_id'] == 'provider':
                channel_id = channel['channel_id']
                return channel_id

    def detect_consumer_chains(self):
        """
        Query the API endpoint to list all available consumer chains.
        """
        logging.info("Updating consumer chains list")
        response = requests.get(self.API_NODE + '/interchain_security/ccv/provider/consumer_chains').json()
        chains = [chain for chain in response['chains']]
        if chains:
            for chain in chains:
                if chain['chain_id'] not in self.chain_vsc.keys():
                    self.chain_vsc[chain['chain_id']] = {
                        'unbonding_period': 0,
                        'alarm_period': 0,
                        'client': chain['client_id'],
                        'connection': '',
                        'channel': '',
                        'vsc_ids': {}
                    }

            remove_chains = []
            for chain in self.chain_vsc.keys():
                chain_names = [chain_name['chain_id'] for chain_name in chains]
                if chain not in chain_names:
                    remove_chains.append(chain)

            for chain in remove_chains:
                self.chain_vsc.pop(chain)

            # Populate connection and channel id fields
            for chain in self.chain_vsc.keys():
                self.chain_vsc[chain]['connection'] = self.get_connection_id(self.chain_vsc[chain]['client'])
                self.chain_vsc[chain]['channel'] = self.get_channel_id(self.chain_vsc[chain]['connection'])

    def search_send_blocks(self, start_height: int=1):
        """
        Query the RPC endpoint to search all blocks that include
        transactions with “provider” as the source port.
        """
        logging.info("Searching blocks that include packets sent from the provider port")
        send_query=f'query="send_packet.packet_src_port=\'provider\'"'
        page = 1
        try:
            block_search = requests.get(self.RPC_NODE + '/block_search?' + send_query + \
            f'&per_page=100&page={page}', timeout = (3, 20)).json()
        except Exception as exc:
            logging.error("Exception:", exc)
        collected = len(block_search['result']['blocks'])
        total_blocks = int(block_search['result']['total_count'])
        block_list = block_search['result']['blocks']

        while collected<total_blocks:
            page = page + 1
            block_search = requests.get(self.RPC_NODE + '/block_search?' + send_query + \
            f'&per_page=100&page={page}', timeout = (3, 20)).json()
            collected = collected + len(block_search['result']['blocks'])
            block_list.extend(block_search['result']['blocks'])

        self.send_blocks = [
            int(block['block']['header']['height'])
            for block in block_list
            if int(block['block']['header']['height']) > start_height
            ]

    def decode_attributes(self, attributes: list):
        """
        Base64 decode the list of attributes and return a decoded list
        """
        decoded_attributes = []
        for encoded_attr in attributes:
            key = base64.b64decode(encoded_attr['key']).decode('utf-8')
            value = base64.b64decode(encoded_attr['value']).decode('utf-8')
            decoded_attributes.append(
                {
                    'key': key,
                    'value': value
                }
            )
        return decoded_attributes

    def add_valset_ids(self, height: int):
        """
        Query the RPC endpoint to get all the validator set ids
        sent out and add them to the relevant consumer chain.
        """
        try:
            events = requests.get(self.RPC_NODE + '/block_results?height=' + str(height)).json()
            block_info = requests.get(self.RPC_NODE + '/block?height=' + str(height)).json()
            block_time_str = block_info['result']['block']['header']['time']
            result = events['result']
            end_block_events = result['end_block_events']
            if end_block_events:
                for ev in end_block_events:
                    if ev['type'] == 'send_packet':
                        decoded = self.decode_attributes(ev['attributes'])
                        vsc_id = 0
                        channel = ''
                        for attribute in decoded:
                            if attribute['key'] == 'packet_data':
                                packet_data = json.loads(attribute['value'])
                                if 'valset_update_id' in packet_data.keys():
                                    vsc_id = int(packet_data['valset_update_id'])
                            if attribute['key'] == 'packet_src_channel':
                                channel = attribute['value']
                        if vsc_id > 0 and channel:
                            for chain in self.chain_vsc.keys():
                                if channel == self.chain_vsc[chain]['channel']:
                                    if vsc_id not in self.chain_vsc[chain]['vsc_ids']:
                                        self.chain_vsc[chain]['vsc_ids'][vsc_id] = {
                                           'state': 'unbonding_period',
                                           'start': block_time_str
                                    }

        except Exception as exc:
            print("Error!: ", exc)
            return

    def record_valset_updates(self):
        """
        Record the validator set updates by querying all the blocks in self.send_blocks
        """
        logging.info("Recording validator set updates")
        for block_no, height in enumerate(self.send_blocks):
            self.add_valset_ids(height)

    def search_recv_txs(self):
        """
        Query the RPC endpoint to Search all transactions that include
        packets with “provider” as the destination port.
        """
        logging.info("Searching transactions that include packets received by the provider port")
        recv_query=f'query="recv_packet.packet_dst_port=\'provider\'"'
        matured_vsc_ids = {}
        page = 1
        channels = [chain_data['channel'] for _, chain_data in self.chain_vsc.items()]

        try:
            tx_search = requests.get(self.RPC_NODE + '/tx_search?' + recv_query + f'&per_page=100&page={page}').json()
        except Exception as exc:
            print(exc)
        collected = len(tx_search['result']['txs'])
        total_txs = int(tx_search['result']['total_count'])
        tx_list = tx_search['result']['txs']

        while collected<total_txs:
            page = page + 1
            tx_search = requests.get(self.RPC_NODE + '/tx_search?' + recv_query + f'&per_page=100&page={page}').json()
            collected = collected + len(tx_search['result']['txs'])
            tx_list.extend(tx_search['result']['txs'])

        if tx_list:
            for tx in tx_list:
                events = tx['tx_result']['events']
                if events:
                    for event in events:
                        if event['type'] == 'recv_packet':
                            vsc_id = 0
                            channel = ''
                            decoded_attr = self.decode_attributes(event['attributes'])
                            for attr in decoded_attr:
                                key = attr['key']
                                value = attr['value']
                                if key == 'packet_data':
                                    packet_data = json.loads(value)
                                    if packet_data['type'] == 'CONSUMER_PACKET_TYPE_VSCM':
                                        vsc_id = int(packet_data['vscMaturedPacketData']['valset_update_id'])
                                if key == 'packet_dst_channel' and value in channels:
                                    channel = value
                            if vsc_id and channel:
                                if vsc_id not in matured_vsc_ids.keys():
                                    matured_vsc_ids[vsc_id] = [channel]
                                else:
                                    matured_vsc_ids[vsc_id].append(channel)

        for chain in self.chain_vsc.keys():
            for vsc_id in matured_vsc_ids.keys():
                if vsc_id in list(self.chain_vsc[chain]['vsc_ids'].keys()):
                    for channel in matured_vsc_ids[vsc_id]:
                        if channel == self.chain_vsc[chain]['channel']:
                            self.chain_vsc[chain]['vsc_ids'][vsc_id]['state'] = 'pass'

    def evaluate_age(self):
        """
        Check for packets that have:
        - matured
        - reached an alarm state
        - reached the VSC timeout
        """
        logging.info("Marking VSC ids that have reached the matured or alarm state")
        current_time = datetime.now(timezone.utc)
        for chain, chain_data in self.chain_vsc.items():
            for vsc_id, vsc_data in chain_data['vsc_ids'].items():
                start_time_str = vsc_data['start']
                time_sections = start_time_str.split('.')
                start_time_str = time_sections[0]+'Z'
                start_time = datetime.strptime(start_time_str, '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone.utc)

                if vsc_data['state'] != 'pass':
                    time_difference_seconds = (current_time - start_time).total_seconds()
                    if time_difference_seconds > chain_data['alarm_period']:
                        vsc_data['state'] = 'alarm'
                    elif time_difference_seconds > chain_data['unbonding_period']:
                        vsc_data['state'] = 'matured'

    def alert_matrix(self, message: str):
        """
        Send alert message to Matrix room
        """
        logging.info(f'Sending alert: "{message}"')
        message_endpoint = f'/_matrix/client/r0/rooms/{self.MATRIX_ROOM}/send/m.room.message'
        payload = { 'msgtype': 'm.text', 'body': message }
        full_url = self.MATRIX_HOMESERVER_URL + message_endpoint + f'?access_token={self.MATRIX_TOKEN}'
        print(full_url)
        r = requests.post(full_url, data=json.dumps(payload))
        print(r.text)

    def update(self):
        """
        Update the available consumer chains and their unbonding periods,
        collect new VSC ids, detect matured packets, and
        issue any relevant alarm messages.
        """
        self.detect_consumer_chains()
        self.get_unbonding_periods()
        # Adjust starting height if record file is available
        # Look for the earliest height of a vsc among all chains
        # that has not reached the 'pass' state
        starting_height = 1
        if os.path.isfile(self.RECORD_FILE):
            with open(self.RECORD_FILE, 'r') as record_file:
                self.chain_vsc = json.load(record_file)
                for _, chain_data in self.chain_vsc.items():
                    for vsc_id in chain_data['vsc_ids'].keys():
                        if starting_height == 1:
                            starting_height = int(vsc_id)
                        else:
                            if int(vsc_id) < starting_height and chain_data['vsc_ids'][vsc_id]['state'] != 'pass':
                                starting_height = int(vsc_id)

        self.search_send_blocks(starting_height)
        self.record_valset_updates()
        self.search_recv_txs()
        self.evaluate_age()
        with open(self.RECORD_FILE, 'w', encoding='utf-8') as output:
            json.dump(self.chain_vsc, output, indent=4)

        for chain, chain_data in self.chain_vsc.items():
            for vsc_id, vsc_data in chain_data['vsc_ids'].items():
                if vsc_data['state'] == 'matured' or vsc_data['state'] == 'alarm':
                    message = f'Consumer monitor> VSC ID {vsc_id} has reached the {vsc_data["state"]} state for chain {chain}.'
                    self.alert_matrix(message)
        logging.info("Update complete")

    def check_hermes_config(self):
        """
        Creates a hermes config file is one is not found.
        """
        if not os.path.isfile(self.HERMES_CONFIG):
            trimmed_rpc = self.RPC_NODE.split('://')[1]
            lines = [
                "[[chains]]\n",
                f"id = '{self.CHAIN_ID}'\n",
                f"rpc_addr = '{self.RPC_NODE}'\n",
                f"grpc_addr = '{self.GRPC_NODE}'\n",
                "event_source = { mode = 'push', url = 'ws://" + trimmed_rpc + "/websocket', batch_delay = '500ms' }\n", # Hermes >= v1.6.0
                # f"websocket_addr = 'ws://{trimmed_rpc}/websocket'\n", # Hermes < v1.6.0
                "account_prefix = 'cosmos'\n",
                "key_name = 'wallet'\n",
                "store_prefix = 'ibc'\n",
                "gas_price = { price = 0.0025, denom = 'uatom' }\n"
            ]
            with open(self.HERMES_CONFIG, 'w') as config_file:
                config_file.writelines(lines)

    def start(self):
        """
        Call this function to begin monitoring.
        """
        self.check_hermes_config()
        self.update()
        schedule.every(self.MONITOR_PERIOD).seconds.do(self.update)
        while True:
            schedule.run_pending()
            time.sleep(10)

logging.basicConfig(
    filename=None,
    level=logging.INFO,
    format="%(asctime)s %(levelname)s %(message)s",
    datefmt="%Y-%m-%d %H:%M:%S",
)

monitor = ConsumerMonitor()
monitor.start()


import sys
from xlrd import open_workbook
import xlrd as xd
import pandas as pd
from pylab import *
from time import time

print('Network starts running')

wb = open_workbook('excel1.xlsx')
sheet = wb.sheet_by_name('Sheet1')
initdata = []
for r in range(sheet.nrows):
    data1 = []
    for c in range(sheet.ncols):
        data1.append(sheet.cell_value(r,c))
    initdata.append(list(data1))

D = []
V = []
wb = open_workbook('phase_detector.xlsx')
for s in wb.sheets():
    print('Sheet:', s.name)
    for row in range(s.nrows):
        print('the row is:', row)
        values = []
        for col in range(s.ncols):
            values.append(s.cell(row, col).value)
        print(values)
        V.append(values)
        print(V)

        dis = math.sqrt((values[0] - firstPoint[0]) ** 2 + (values[1] - firstPoint[1]) ** 2)
        dis1 = round(dis, 1)
        D.append(dis1)
    D.sort()
    print("The order of the distance between the node and the BS:")
    print(D)
for i in range(len(D) - 1):
    for j in range(len(D) - i - 1):
        if (V[j][0] - V[0][0]) ** 2 + (V[j][1] - V[0][1]) ** 2 > (V[j + 1][0] - V[0][0]) ** 2 + (
                V[j + 1][1] - V[0][1]) ** 2:
            temp = V[j]
            V[j] = V[j + 1]
            V[j + 1] = temp
print("The sorted dataset:")
print(V)

# The number of the inner-ring nodes
total = 0
dis3 = 0
TOTAL=[]
for i in range(len(D) - 1):
    dis2 = math.sqrt((V[i][0] - V[i + 1][0]) ** 2 + (V[i][1] - V[i + 1][1]) ** 2)
    dis3 += dis2
    total = dis3 + math.sqrt((V[i + 1][0] - firstPoint[0]) ** 2 + (V[i + 1][1] - firstPoint[1]) ** 2)
    TOTAL.append(total)
    if i * Emax + total * Um > Q:
        break
print("The number of the inner-ring nodes:")
print(i)
print("The total moving distance:")
print(TOTAL)

# The neighbor of the inner-ring nodes
neiset = []
for index in range(i):
    index = index + 1
    b = 0
    for j in range(i):
        j = j + 1
        if j != index and math.sqrt((V[j][0] - V[index][0]) ** 2 + (V[j][1] - V[index][1]) ** 2) < comdis:
            b = b + 1
    neiset.append(b)
print("The neighbor set of inner-ring nodes:")
print(neiset)

# The weight of the neighbor
WSI = []
for index in range(i):
    wsi = (i - neiset[index]) / i
    WSI.append(wsi)
print("The weight of the neighbor:")
print(WSI)


import torch
import torch.optim as optim
import torch.nn.functional as F
# Define loss function
from torch import nn

class MarginLoss(nn.Module):
    def __init__(self, m_pos=0.9, m_neg=0.1, lambda_=0.5):
        super(MarginLoss, self).__init__()
        self.m_pos = m_pos
        self.m_neg = m_neg
        self.lambda_ = lambda_

    def forward(self, output, target):
        # Convert target to one-hot encoding
        target_onehot = torch.eye(2)[target]

        # Compute lengths of output vectors
        lengths = torch.sqrt((output ** 2).sum(dim=2))

        # Compute margin loss
        loss_pos = target_onehot * F.relu(self.m_pos - lengths).pow(2)
        loss_neg = (1 - target_onehot) * F.relu(lengths - self.m_neg).pow(2)
        margin_loss = loss_pos + self.lambda_ * loss_neg
        margin_loss = margin_loss.sum(dim=1).mean()

        return margin_loss


#!/usr/bin/env python3

from argparse import ArgumentParser
from itertools import cycle
from os import path
from sys import exit, stderr

import matplotlib
from matplotlib import pyplot as plt
from matplotlib.ticker import AutoMinorLocator
from pandas import read_csv

THRESH = 'Threshold'
MARKERS = [None]
# List of matplotlib.artist.Artist properties which we should copy between
# subplots
COMMON_PROPS = {
    'linestyle', 'marker', 'visible', 'drawstyle', 'linewidth',
    'markeredgewidth', 'markeredgecolor', 'markerfacecoloralt',
    'dash_joinstyle', 'zorder', 'markersize', 'solid_capstyle',
    'dash_capstyle', 'markevery', 'fillstyle', 'markerfacecolor', 'label',
    'alpha', 'path_effects', 'color', 'solid_joinstyle'
}
PRIORITIES = {
    'Shoulder': 0,
    'Elbow': 1,
    'Wrist': 2
}

parser = ArgumentParser(
    description="Take accuracy at different thresholds and plot it nicely"
)
parser.add_argument(
    '--save', metavar='PATH', type=str, default=None,
    help="Destination file for graph"
)
parser.add_argument(
    '--input', nargs=2, metavar=('NAME', 'PATH'), action='append', default=[],
    help='Name (title) and path of CSV to plot; can be specified repeatedly'
)
parser.add_argument(
    '--poster', action='store_true', dest='is_poster', default=False,
    help='Produce a plot for the poster rather than the report.'
)
parser.add_argument(
    '--dims', nargs=2, type=float, metavar=('WIDTH', 'HEIGHT'),
    default=[6, 3], help="Dimensions (in inches) for saved plot"
)

def load_data(inputs):
    labels = []
    thresholds = None
    parts = None

    for name, path in inputs:
        labels.append(name)

        csv = read_csv(path)

        if thresholds is None:
            thresholds = csv[THRESH]
        if parts is None:
            parts = {part: [] for part in csv.columns - [THRESH]}

        assert len(parts) == len(csv.columns - [THRESH])
        assert (csv[THRESH] == thresholds).all()

        for part in parts:
            part_vals = csv[part]
            assert len(part_vals) == len(thresholds)
            parts[part].append(part_vals)

    return labels, thresholds, parts

if __name__ == '__main__':
    args = parser.parse_args()
    if not args.input:
        parser.print_usage(stderr)
        print('error: must specify at least one --input', file=stderr)
        exit(1)

    if args.is_poster:
        matplotlib.rcParams.update({
            'font.family': 'sans',
            'pgf.rcfonts': False,
            'xtick.labelsize': '12',
            'ytick.labelsize': '12',
            'legend.fontsize': '14',
            'axes.labelsize': '16',
            'axes.titlesize': '18',
        })
    else:
        matplotlib.rcParams.update({
            'font.family': 'serif',
            'pgf.rcfonts': False,
            'pgf.texsystem': 'pdflatex',
            'xtick.labelsize': 'xx-small',
            'ytick.labelsize': 'xx-small',
            'legend.fontsize': 'xx-small',
            'axes.labelsize': 'x-small',
            'axes.titlesize': 'small',
        })

    labels, thresholds, parts = load_data(args.input)

    _, subplots = plt.subplots(1, len(parts), sharey=True)
    common_handles = None
    part_keys = sorted(parts.keys(), key=lambda s: PRIORITIES.get(s, -1))
    for part_name, subplot in zip(part_keys, subplots):
        pcks = parts[part_name]
        if common_handles is None:
            # Record first lot of handles for reuse
            common_handles = []
            for pck, label, marker in zip(pcks, labels, cycle(MARKERS)):
                handle, = subplot.plot(
                    thresholds, 100 * pck, label=label, marker=marker
                )
                common_handles.append(handle)
        else:
            for pck, handle in zip(pcks, common_handles):
                props = handle.properties()
                kwargs = {k: v for k, v in props.items() if k in COMMON_PROPS}
                subplot.plot(thresholds, 100 * pck, **kwargs)

        # Labels, titles
        subplot.set_title(part_name)
        subplot.set_xlabel('Threshold (px)')
        subplot.grid(which='both')

    subplots[0].set_ylabel('Accuracy (%)')
    subplots[0].set_ylim(ymin=0, ymax=100)
    minor_locator = AutoMinorLocator(2)
    subplots[0].yaxis.set_minor_locator(minor_locator)
    subplots[0].set_yticks(range(0, 101, 20))
    ax = plt.gca()
    box = ax.get_position()
    ax.set_position([box.x0, box.y0, box.width * 0.8, box.height])
    legend = plt.figlegend(
        common_handles, labels, 'center left', bbox_to_anchor=(0.965, 0.5),
        borderaxespad=0.
    )

    if args.save is None:
        plt.show()
    else:
        print('Saving figure to', args.save)
        plt.gcf().set_size_inches(args.dims)
        plt.tight_layout()
        plt.savefig(
            args.save, bbox_inches='tight', bbox_extra_artists=[legend]
        )


# keywork - lambda
# uses of lambda - https://towardsdatascience.com/lambda-functions-with-practical-examples-in-python-45934f3653a8

def add(a, b):
    c = a + b
    return c

l = add(5, 136)
print(l)

x = lambda a, b: a + b

s = x(3, 133)
print(s)

f = lambda x: 3 * x + 1
eq = f(3)
print(eq)

full_name = lambda fn, ln: fn.strip().title() + " " + ln.strip().title()
print(full_name('    amal    ', ' pushp          '))

# Quadratic function f(x) = ax**2 + bx + c

def quad(a, b, c):
    return lambda x: a * x ** 2 + b * x + c

q = quad(1, 2, 3)

print(q)
print(q(3))
print("Type of q: ", type(q))
print("Type of q(3): ", type(q(3)))

1. Python Part 1 - Basics/7. Functions/4. Anonymous Functions.py


import torch
from LightningFunc.utils import writeCSV, write_Best_model_path
from LightningFunc.optimizer import get_lr

def training_step(self, batch, batch_idx):
    # training_step defined the train loop.
    # It is independent of forward
    x, y = batch
    self.reference_image = x
    out = self.forward(x)
    loss = self.criterion(out, y)

    # acc
    _, predicted = torch.max(out, dim=1)
    train_acc = self.accuracy_score(predicted.cpu(), y.cpu()).clone().detach().requires_grad_(True)

    self.logger.experiment.add_scalars("Loss/Step", {"Train":loss}, self.global_step)

    return {'loss':loss, 'acc':train_acc}

def training_epoch_end(self, outputs): # 在Validation的一個Epoch結束後，計算平均的Loss及Acc.
    avg_loss = torch.stack([x['loss'] for x in outputs]).mean()
    avg_train_acc = torch.stack([x['acc'] for x in outputs]).mean()

    self.logger.experiment.add_scalars("Loss/Epoch", {"Train":avg_loss}, self.current_epoch)

    self.logger.experiment.add_scalars("Accuracy/Epoch", {"Train":avg_train_acc}, self.current_epoch)

    if(self.current_epoch==1):
        self.sampleImg=torch.rand((1,3, 512, 512)).cuda()
        self.logger.experiment.add_graph(self.model, self.sampleImg)

    # iterating through all parameters
    for name,params in self.named_parameters():
        self.logger.experiment.add_histogram(name,params,self.current_epoch)

    # logging reference image
    # self.logger.experiment.add_image("input",torch.Tensor.cpu(self.reference_image[0][0]),self.current_epoch,dataformats="HW")

def validation_step(self, batch, batch_idx):
    x, y = batch
    out = self.forward(x)
    loss = self.criterion(out, y)

    # loss
    values = {'val_loss':loss}
    self.log_dict(values, logger=True, on_epoch=True)

    # acc
    _, predicted = torch.max(out, dim=1)
    val_acc = self.accuracy_score(predicted.cpu(), y.cpu()).clone().detach().requires_grad_(False)

    return {'val_loss': loss, 'val_acc': val_acc}

def validation_epoch_end(self, outputs): # 在Validation的一個Epoch結束後，計算平均的Loss及Acc.
    avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()
    avg_val_acc = torch.stack([x['val_acc'] for x in outputs]).mean()

    self.logger.experiment.add_scalars("Loss/Epoch", {"Val":avg_loss}, self.current_epoch)
    self.logger.experiment.add_scalars("Loss/Step", {"Val":avg_loss}, self.global_step)

    self.logger.experiment.add_scalars("Accuracy/Epoch", {"Val":avg_val_acc}, self.current_epoch)

    self.write_Best_model_path()

def test_step(self, batch, batch_idx, only_pred = True): #定義 Test 階段
    x, y = batch
    out = self.forward(x)

    # acc
    _, predicted = torch.max(out, dim=1)

    if not only_pred:
        test_acc = self.accuracy_score(predicted.cpu(), y.cpu()).clone().detach().requires_grad_(True)
        return {'test_acc': test_acc, 'predicted': predicted.data[0]}
    else:
        return {'predicted': predicted.data[0]}

def test_epoch_end(self, outputs, only_pred = True): # 在test的一個Epoch結束後，計算平均的Loss及Acc.
    if not only_pred:
        avg_test_acc = torch.stack([x['test_acc'] for x in outputs]).mean()
        self.log('avg_val_acc', avg_test_acc)

    total_predict_list = torch.stack([x['predicted'] for x in outputs]).tolist()

    if len(self.target) != 0:
        self.target['Label'] = total_predict_list
        tensorboard_logs = self.writeCSV()
        return {'progress_bar': tensorboard_logs}


#Yes or No Machine

#Takes N factors and how confident you are.

#Your factor is either affirm or neg and it is rated from 1 to 10

Question = input("Should *action*?:")

Factors = []

NumberOfFactors = int(input("Number of factors are you considering:"))

for i in range(1,NumberOfFactors + 1):
    FactorSpec = [input("Aff or Neg:"), int(input("Weight:"))]
    Factors.append(FactorSpec)

print(Factors)

ValuePool = 0
ExtractionPosition = 0
for i in range(1,NumberOfFactors + 1):
    ExtractionType = Factors[ExtractionPosition][0]
    if ExtractionType == 'yes':
        ExtractionWeight = int(Factors[ExtractionPosition][1])
        ValuePool = ValuePool + ExtractionWeight
    if ExtractionType == 'no':
        ExtractionWeight = int(Factors[ExtractionPosition][1]) * -1
        ValuePool = ValuePool + ExtractionWeight
    ExtractionPosition += 1

if ValuePool >= 1 and ValuePool <= 5:
    Answer = 'Maybe'
if ValuePool == 0:
    Answer = 'possibly'
if ValuePool >= 5 and ValuePool <= 10:
    Answer = 'Yes'
if ValuePool <= -1 and ValuePool >= -5:
    Answer = 'Maybe'
if ValuePool <= -5 and ValuePool >= -10:
    Answer = 'No'

print(ValuePool)
print('*' * 33)
print(Answer + ", " + Question + ".")


import tkinter
import matplotlib

matplotlib.use('TkAgg')

import csv
import random

def Euler_dis(node1, node2):
    distance=0.0
    for i in range(len(node1)):
        distance += (node1[i]-node2[i])**2
    return distance

class node:
    def __init__(self, data):
        self.input = data[1:]
        self.true_label = data[0]
        self.pre_label = 0

    def predict(self, k_classify):
        single_loss = 0.0
        distance_buffer=0
        index_buf = 0
        for i in range(len(k_classify)):
            if i ==0 :
                distance_buffer = Euler_dis(self.input, k_classify[i][1:])
                self.pre_label = k_classify[i][0]
            else:
                distance = Euler_dis(self.input, k_classify[i][1:])
                if distance<distance_buffer:
                    distance_buffer = distance
                    self.pre_label = k_classify[i][0]
                    index_buf = i
        single_loss = Euler_dis(self.input, k_classify[index_buf][1:])
        return single_loss

def sum1(all_node, k_classify_node):
    all_loss = 0.0
    for i in range(len(all_node)):
        all_loss+=all_node[i].predict(k_classify_node)
    return all_loss

def list_add(list1, list2):
    list_buf = [0 for i in range(len(list1))]
    for i in range(len(list1)):
        list_buf[i] = list1[i]+list2[i]
    return list_buf

def classify_node_generate(all_node, k):
    classify_node = [[0 for i in range(14)] for j in range(k)]
    classify_num = [0, 0, 0]
    for i in range(len(classify_node)):
        classify_node[i][0] = i
    for i in range(len(all_node)):
        index = all_node[i].pre_label
        #print(index)
        classify_node[index][1:]=list_add(classify_node[index][1:], all_node[i].input)
        classify_num[index] += 1
    for i in range(k):
        for j in range(13):
            classify_node[i][1+j] /= classify_num[i]
    return classify_node

def random_init_3(data, k):
    init_node = random.sample(data, 1)
    init_data = init_node.copy()
    init_data[0][0] = 0
    #print(init_data)
    distance_max = 0
    #data_buf = 0
    node_index = 0
    for i in range(len(data)):
        #print(len(data[i][1:]))
        #print(len(init_data[0][1:]))
        distance_buf = Euler_dis(data[i][1:], init_data[0][1:])

        if distance_buf>distance_max:
            distance_max = distance_buf
            node_index = i
    data_buf = data[node_index].copy()
    data_buf[0] = 1
    init_data.append(data_buf)
    distance_max = 0
    node_index = 0
    for i in range(len(data)):
        distance1 = Euler_dis(data[i][1:], init_data[0][1:])
        distance2 = Euler_dis(data[i][1:], init_data[1][1:])
        distance_buf = distance1+distance2
        if distance1!=0 and distance2!=0:
            if distance_buf>distance_max:
                distance_max = distance_buf
                node_index = i
    data_buf = data[node_index].copy()
    data_buf[0] = 2
    init_data.append(data_buf)
    return init_data

def true_classify_node_generate(all_node, k):
    classify_node = [[0 for i in range(14)] for j in range(k)]
    classify_num = [0, 0, 0]
    for i in range(len(classify_node)):
        classify_node[i][0] = i
    for i in range(len(all_node)):
        index = all_node[i].true_label-1
        # print(index)
        classify_node[index][1:] = list_add(classify_node[index][1:], all_node[i].input)
        classify_num[index] += 1
    for i in range(k):
        for j in range(13):
            classify_node[i][1 + j] /= classify_num[i]
    return classify_node

def get_label_classify_node(pre_classify_node, true_nodes):
    distance_reocrd = 0
    type_record = 0
    for x in true_nodes:
        distance_buf = Euler_dis(pre_classify_node[1:], x[1:])
        if distance_reocrd==0:
            distance_reocrd = distance_buf
            pre_classify_node[0] = x[0]
        elif distance_buf<distance_reocrd:
            distance_reocrd = distance_buf
            pre_classify_node[0] = x[0]
    return pre_classify_node

import matplotlib.pyplot as plt
from sklearn.decomposition import PCA
import numpy as np

def druw(all_node, accur, loss):
    type_node = [[] for i in range(3)]
    pcy_array = np.zeros([len(all_node), 13])
    pca = PCA(n_components=2)
    for i in range(len(all_node)):
        pcy_array[i] = all_node[i].input
        #print(all_node[i].pre_label)
        #type_node[all_node[i].pre_label].append(all_node[i].input)
    new_pcy_array = pca.fit_transform(pcy_array)
    for i in range(len(all_node)):
        type_node[all_node[i].pre_label].append(pcy_array[i])
    x_scas = [[] for i in range(3)]
    y_scas = [[] for i in range(3)]
    for i in range(len(type_node)):
        for j in range(len(type_node[i])):
            x_scas[i].append(type_node[i][j][0])
            y_scas[i].append(type_node[i][j][1])
    plt.scatter(x_scas[0], y_scas[0], color='r')
    plt.scatter(x_scas[1], y_scas[1], color='b')
    plt.scatter(x_scas[2], y_scas[2], color='g')

    name = 'SSE={:.2f}, Acc={:.2f}'.format(loss, accur)
    plt.title(name)

    plt.show()

def print_k(alist, k):
    for i in range(min(len(alist), k)):
        print(alist[i])

data = []

with open('归一化数据.csv') as csvfile:
    csv_reader = csv.reader(csvfile)
    for row in csv_reader:
        for i in range(len(row)):
            row[i] = eval(row[i])
        data.append(row)

#print(data[0])
print_k(data, 3)
print(len(data[0]))

node_num = len(data)
print(node_num)
all_node = []
for i in range(node_num):
    all_node.append(node(data[i]))

k_classify_node = random_init_3(data, 3)
#print(k_classify_node)
print_k(k_classify_node, 3)

count = 0
delta = 100.0
old_loss=0
loss = 0.0

while(delta>1e-9):

    print('old_loss', old_loss)
    loss=sum1(all_node, k_classify_node)
    print('new_loss', loss)
    delta = abs(loss-old_loss)
    k_classify_node=classify_node_generate(all_node, 3)
    old_loss = loss
    count+=1
    if(count%100)==0:
        print('count:', count)

print('delta:%f'%delta)
print(count)

true_classify_node = true_classify_node_generate(all_node, 3)
for i in range(len(k_classify_node)):
    k_classify_node[i] = get_label_classify_node(k_classify_node[i], true_classify_node)

for i in range(len(all_node)):
    all_node[i].pre_label = k_classify_node[all_node[i].pre_label][0]

valid_count = 0

for i in range(len(all_node)):
    if all_node[i].pre_label==all_node[i].true_label-1:
        valid_count+=1

print(k_classify_node)
accur = valid_count/len(all_node)
print('accuracy:{}%'.format(100*accur))

druw(all_node, accur, loss)


import sys
import os
f=open(sys.argv[1])
h=open(sys.argv[2],'w')
f.readline()
annot=f.readline().split()
for i in range(6,len(annot)): annot[i]=annot[i].split("/")[-1].strip(".Aligned.sortedByCoord.out.bam")
hline=[annot[0]]+annot[6:]
h.write("\t".join(hline)+"\n")
for line in f:
	line=line.split()
	hline=[line[0]]+line[6:]
	h.write("\t".join(hline)+"\n")

f.close()
h.close()


import os

import pytest
from sticker import Config, Sentence

def test_config(tests_root):
    Config(os.path.join(tests_root, "sticker.conf"))

def test_bogus_config(tests_root):
    with pytest.raises(ValueError):
        Config(os.path.join(tests_root, "bogus.conf"))

def test_missing_config(tests_root):
    with pytest.raises(IOError):
        Config(os.path.join(tests_root, "nonexistant.conf"))

def test_labeler(tests_root):
    config = Config(os.path.join(tests_root, "sticker.conf"))

    assert config.labeler.labels.endswith("sticker.labels")

    config.labeler.labels = "another.labels"
    assert config.labeler.labels == "another.labels"

def test_model(tests_root):
    config = Config(os.path.join(tests_root, "sticker.conf"))

    assert config.model.graph.endswith("sticker.graph")
    assert config.model.intra_op_parallelism_threads == 2
    assert config.model.inter_op_parallelism_threads == 4
    assert config.model.parameters.endswith("epoch-37")

    config.model.graph = "another.graph"
    config.model.intra_op_parallelism_threads = 1
    config.model.inter_op_parallelism_threads = 3
    config.model.parameters = "epoch-42"

    assert config.model.graph == "another.graph"
    assert config.model.intra_op_parallelism_threads == 1
    assert config.model.inter_op_parallelism_threads == 3
    assert config.model.parameters == "epoch-42"


import numpy as np
import torch
from torch import nn
from torch.autograd import Variable
import torch.optim as optim
from torch.utils.data import DataLoader

from types import SimpleNamespace
import inspect

from quantumdraw.solver.torch_utils import DataSet, Loss, ZeroOneClipper
from quantumdraw.solver.solver_base import Solver
from tqdm import tqdm
import time

class NeuralSolver(Solver):

    def __init__(self,wf=None, sampler=None,
                      optimizer=None,scheduler=None):
        super(NeuralSolver,self).__init__(wf,sampler)
        self.opt = optimizer
        self.scheduler = scheduler

        #esampling
        self.resampling(ntherm=-1, resample=100,
                        resample_from_last=True,
                        resample_every=1)

        # observalbe
        self.observable(['local_energy'])

    def resampling(self,ntherm=-1, resample=100,
                        resample_from_last=True,
                        resample_every=1):
        '''Configure the resampling options.'''
        self.resample = SimpleNamespace()
        self.resample.ntherm = ntherm
        self.resample.resample = resample
        self.resample.resample_from_last = resample_from_last
        self.resample.resample_every = resample_every

    def save_checkpoint(self,epoch,loss,filename):
        torch.save({
            'epoch' : epoch,
            'model_state_dict' : self.wf.state_dict(),
            'optimzier_state_dict' : self.opt.state_dict(),
            'loss' : loss
            }, filename)
        return loss

    def run(self, nepoch, pos = None, batchsize=None,
            save='model.pth',  loss='variance', plot = None,
            with_tqdm=True):

        '''Train the model.

        Arg:
            nepoch : number of epoch
            batchsize : size of the minibatch, if None take all points at once
            pos : presampled electronic poition
            obs_dict (dict, {name: []} ) : quantities to be computed during the training
                                           'name' must refer to a method of the Solver instance
            ntherm : thermalization of the MC sampling. If negative (-N) takes the last N entries
            resample : number of MC step during the resampling
            resample_from_last (bool) : if true use the previous position as starting for the resampling
            resample_every (int) : number of epch between resampling
            loss : loss used ('energy','variance' or callable (for supervised)
            plot : None or plotter instance from plot_utils.py to interactively monitor the training
        '''

        # checkpoint file
        self.save_model = save

        # sample the wave function
        pos = self.sample(pos=pos, ntherm=self.resample.ntherm, with_tqdm=with_tqdm)

        # determine the batching mode
        if batchsize is None:
            batchsize = len(pos)

        # change the number of steps
        _nstep_save = self.sampler.nstep
        self.sampler.nstep = self.resample.resample

        # create the data loader
        self.dataset = DataSet(pos)
        self.dataloader = DataLoader(self.dataset,batch_size=batchsize)

        # get the loss
        self.loss = Loss(self.wf,method=loss)

        # clipper for the fc weights
        clipper = ZeroOneClipper()

        cumulative_loss = []

        for n in range(nepoch):
            #print('----------------------------------------')
            #print('epoch %d' %n)

            cumulative_loss = 0
            for data in self.dataloader:

                lpos = Variable(data).float()
                lpos.requires_grad = True

                loss = self.loss(lpos)
                cumulative_loss += loss

                if torch.isnan(loss):
                    print('ooops ran into an issue')
                    continue

                self.opt.zero_grad()
                loss.backward()
                self.opt.step()

                self.check_parameters()

                if self.wf.fc.clip:
                    self.wf.fc.apply(clipper)

            if plot is not None:
                plot.drawNow()

            # if cumulative_loss < min_loss:
            #     min_loss = self.save_checkpoint(n,cumulative_loss,self.save_model)

            # get the observalbes
            # self.get_observable(self.obs_dict,pos)
            # print('loss %f' %(cumulative_loss))
            # print('variance : %f' %np.var(self.obs_dict['local_energy'][-1]))
            # print('energy : %f' %np.mean(self.obs_dict['local_energy'][-1]) )
            print('score : %f' %self.get_score() )
            # print('----------------------------------------')

            # resample the data
            if (n%self.resample.resample_every == 0): # or (n == nepoch-1):
                if self.resample.resample_from_last:
                    pos = pos.clone().detach()
                else:
                    pos = None
                pos = self.sample(pos=pos,ntherm=self.resample.ntherm,with_tqdm=False)
                self.dataloader.dataset.data = pos

            if self.scheduler is not None:
                self.scheduler.step()

        #restore the sampler number of step
        self.sampler.nstep = _nstep_save

        return pos

    def check_parameters(self):

        check_nan =  torch.isnan(self.wf.rbf.centers.data)
        self.wf.rbf.centers.data[check_nan] = 0.

        check_nan =  torch.isnan(self.wf.rbf.sigma.data)
        self.wf.rbf.sigma.data[check_nan] = 1.0

        check_nan =  torch.isnan(self.wf.fc.weight.data)
        self.wf.fc.weight.data[check_nan] = 0.


# import sys
#!/usr/bin/python
import string

def text_analyzer(str, __doc__):
    if (not str):
        print("What is the text to analyze?")
        str = input()
    print("The text contains ", len(str) , " character(s):")
    sum = 0
    for i in str:
        if (i.isupper()):
            sum += 1
    print("- ",  sum , " upper letter(s)")
    sum = 0
    for i in str:
        if (i.islower()):
            sum += 1
    print("- ",  sum , " lower letter(s)")
    sum = 0
    for i in str:
        if i in string.punctuation:
            sum += 1
    print("- ",  sum , " punctuation mark(s)")
    sum = str.count(" ")
    print("- ",  sum , " space(s)")
    return

# text_analyzer("Python 2.0, released 2000, introduced features like List comprehensions and a garbage collection system capable of collecting reference cycles.")

# text_analyzer("Python is an interpreted, high-level, general-purpose programming language. Created by Guido van Rossum and first released in 1991, Python's design philosophy emphasizes code readability with its notable use of significant whitespace.")


"""
Python script to read compute and plot Vorticity (zeta) output from MITgcm
"""

import numpy as np
import MITgcmutils as mit
from MITgcmutils import rdmds
import matplotlib.pyplot as plt
from matplotlib import cm
import matplotlib.colors as colors
#import cv2 as cv

plt.ion()

class MidpointNormalize(colors.Normalize):
    def __init__(self, vmin=None, vmax=None, midpoint=None, clip=False):
        self.midpoint = midpoint
        colors.Normalize.__init__(self, vmin, vmax, clip)
    def __call__(self, value, clip=None):
        # I'm ignoring masked values and all kinds of edge cases to make a
        # simple example...
        x, y = [self.vmin, self.midpoint, self.vmax], [0, 0.5, 1]
        return np.ma.masked_array(np.interp(value, x, y))
def center_of_mass(X):
    # calculate center of mass of a closed polygon
    x = X[:,0]
    y = X[:,1]
    g = (x[:-1]*y[1:] - x[1:]*y[:-1])
    A = 0.5*g.sum()
    cx = ((x[:-1] + x[1:])*g).sum()
    cy = ((y[:-1] + y[1:])*g).sum()
    return 1./(6*A)*np.array([cx,cy])

#nit = 60 # no of time step
#ts = 60  # time step = ts*dt (in second);
timestep = 120 #timestep input in second
dumpfreq = 7200 # file every 3 hours for dumpfreq 10800
file_dit = dumpfreq/timestep

#ts = file_dit
#file_dit = 3600
day_s = 0
day_e = 15

startfile = day_s*file_dit #or any integer*file_dit
endfile = day_e*86400/timestep + 50 #1 day
itrs = np.arange(startfile,endfile, file_dit)

ipdepth = 60
idepth = 25 # depth for subsurface drifts 36 for approx -99 in 75 grid

#itrs = np.arange(0, 21610, file_dit)
#itrs = [0, 2400,4800, 7200] #, 2160, 2880, 3600, 4320, 5040, 5760, 6480, 7200]

nit = itrs.size
cx = np.zeros((nit))
cy = np.zeros((nit))
c2x = np.zeros((nit))
c2y = np.zeros((nit))

rho0 = 999.8
f0 = 1e-4
XC = mit.rdmds('XC')
dXC = mit.rdmds('DXC')
YC = mit.rdmds('YC')
dYC = mit.rdmds('DYC')
RC = mit.rdmds('RC')
XG = mit.rdmds('XG')
YG = mit.rdmds('YG')

nr = RC.size
nx = XC[0,:].size
ny = YC[:,0].size

U_c = np.zeros((nr,ny,nx))
V_c = np.zeros((nr,ny,nx))
KE_tot = np.zeros((nit))

# values for plots
depth_plot_ratio= 0.5#0.7 or 0.5
plta = 40
pltb = 290
xc_d = XC[plta:pltb,plta:pltb]*1e-3
yc_d = YC[plta:pltb,plta:pltb]*1e-3
sec_y = int(nx/2)
x_c=XC[sec_y,sec_y]*1e-3
y_c=YC[sec_y,sec_y]*1e-3
#pltf = plta#110
pltc = 120#100
pltd = 210#210
plte = pltb#220
xc_dc = XC[pltc:pltd,pltc:pltd]*1e-3
yc_dc = YC[pltc:pltd,pltc:pltd]*1e-3

# Set default fontsizes for plots
SMALL_SIZE = 10
MEDIUM_SIZE = 12
BIGGER_SIZE = 14

plt.rc('font', size=SMALL_SIZE)          # controls default text sizes
plt.rc('axes', titlesize=SMALL_SIZE)     # fontsize of the axes title
plt.rc('axes', labelsize=SMALL_SIZE)    # fontsize of the x and y labels
plt.rc('xtick', labelsize=SMALL_SIZE)    # fontsize of the tick labels
plt.rc('ytick', labelsize=SMALL_SIZE)    # fontsize of the tick labels
plt.rc('legend', fontsize=SMALL_SIZE)    # legend fontsize
plt.rc('figure', titlesize=BIGGER_SIZE)  # fontsize of the figure title

ke_range = np.linspace(0,0.1,100, endpoint=True)
ke_ticks = np.linspace(0,0.1,11, endpoint=True)
#ke_range = 99

for it in itrs:
    U = mit.rdmds('U',it)
    V = mit.rdmds('V',it)
#
# Must interpolate to cell center before computing the tangential and radial velocity
    for z in np.arange(0,nr):
        for y in np.arange(0,ny):
            U_c[z,y,:] = np.interp(XC[y,:],XG[y,:],U[z,y,:])
        for x in np.arange(0,nx):
            V_c[z,:,x] = np.interp(YC[:,x],YG[:,x],V[z,:,x])

    KE = (U_c**2 + V_c**2)#/2
    KE_tot[int((it-itrs[0])/file_dit)] = np.sum(KE)
#

#############Plot figures ##############
############# KE ####################################
    fig = plt.figure(figsize=(11,4))
#
    ax1 = fig.add_subplot(1, 2, 1)
    ax1.set_aspect(1)
    plt.contourf(xc_d,yc_d,KE[0,plta:pltb,plta:pltb],ke_range,cmap=cm.ocean_r)
    plt.colorbar(label=r'$KE  m^2s^{-2}$', format='%.2f', ticks=ke_ticks)
    plt.xlabel("x (km)")
    plt.ylabel("y (km)")
    plt.title(r'surface KE $m^2s^{-2}$, time step %d hr' % (int(it*timestep/3600)))
#
    ax2 = fig.add_subplot(1, 2, 2)
    ax2.set_aspect(depth_plot_ratio)
    #plt.subplot(121, aspect='equal', adjustable='box-forced')
    plt.contourf (XC[sec_y,plta:pltb]*1e-3, RC[:ipdepth].squeeze(), KE[:ipdepth, sec_y,plta:pltb], ke_range,cmap=cm.ocean_r)
    plt.colorbar(label=r'$KE \ [m^2s^{-2}]$', format='%.2f', ticks=ke_ticks)
    plt.xlabel("x (km)")
    plt.ylabel("z (m)")
    plt.title(r'KE $m^2s^{-2}$, time step %d hr' % (int(it*timestep/3600)))
#
    plt.tight_layout(pad=1)
    if it ==0:
        plt.savefig("./figures/KE_0000"+ str(int(it)) + ".png")
    elif it < 100:
        plt.savefig("./figures/KE_000"+ str(int(it)) + ".png")
    elif it < 1000:
        plt.savefig("./figures/KE_00"+ str(int(it)) + ".png")
    elif it < 10000:
        plt.savefig("./figures/KE_0"+ str(int(it)) + ".png")
    else:
        plt.savefig("./figures/KE_"+ str(int(it)) + ".png")
    plt.close()

    fig = plt.figure(figsize=(5,4))
#
    ax1 = fig.add_subplot(1, 1, 1)
    ax1.set_aspect(1)
    plt.contourf(xc_d,yc_d,KE[idepth,plta:pltb,plta:pltb],ke_range,cmap=cm.ocean_r)
    plt.colorbar(label=r'$KE  m^2s^{-2}$', format='%.2f', ticks=ke_ticks)
    plt.xlabel("x (km)")
    plt.ylabel("y (km)")
    plt.title(r'KE (at z=%d m) $m^2s^{-2}$, time step %d hr' % (RC[idepth], int(it*timestep/3600)))
#
    plt.tight_layout(pad=1)
    if it ==0:
        plt.savefig("./figures/KE50_0000"+ str(int(it)) + ".png")
    elif it < 100:
        plt.savefig("./figures/KE50_000"+ str(int(it)) + ".png")
    elif it < 1000:
        plt.savefig("./figures/KE50_00"+ str(int(it)) + ".png")
    elif it < 10000:
        plt.savefig("./figures/KE50_0"+ str(int(it)) + ".png")
    else:
        plt.savefig("./figures/KE50_"+ str(int(it)) + ".png")
    plt.close()

time = itrs*timestep/3600

# KE_evolution
#fig = plt.figure(figsize=(10,5))
fig, ax1 = plt.subplots(figsize=(10,5))
ax1.plot(time,KE_tot)
#ax1.plot(time,cy-cy[0], label='meridional drift')
plt.xlabel("time (hour)")
plt.ylabel('KE')
plt.xlim(0,itrs[-1]*timestep/3600)
#plt.legend(loc='lower left')
plt.title(r'Total kinetic energy evolution')
plt.tight_layout(pad=1)
#ax1.set_aspect(1)
plt.savefig('./figures/time_evol_KE%d.png' % (day_e))

"""
fig, ax1 = plt.subplots(figsize=(5,6))
#plt.plot(time,cx, label='zonal drift')
ax1.plot(cx-cx[0],cy-cy[0], label='surface')
ax1.plot(c2x-c2x[0],c2y-c2y[0], label='%d m' % (RC[idepth]))
plt.xlabel("x (km)")
plt.ylabel('y (km)')
plt.legend(loc='best')
plt.xlim(-4,4)
plt.ylim(-11,0)
ax1.set_aspect(1)
plt.text(cx[-1]-cx[0],cy[-1]-cy[0],'%.2f , %.2f' % (cx[-1]-cx[0],cy[-1]-cy[0]))
plt.text(c2x[-1]-c2x[0],c2y[-1]-c2y[0],'%.2f , %.2f' % (c2x[-1]-c2x[0],c2y[-1]-c2y[0]))
plt.title(r'Eddy drift from center due to wind forcing in %d days' % (day_e-day_s))
plt.tight_layout(pad=1)
plt.savefig('./figures/coretraject%d.png' % (day_e))

# surface vort core
plt.tight_layout(pad=1)
fig = plt.figure(figsize=(5,3))
plt.plot(itrs*timestep/3600,svortc/f0)
plt.xlabel("time (hour)")
plt.ylabel(r'$\frac{\zeta}{|f|}$')
plt.xlim(0,itrs[-1]*timestep/3600)
plt.title(r'Minimum Surface vorticity, $\frac{\zeta_{min}}{|f|}$')
plt.tight_layout(pad=1)
plt.savefig('./figures/Surfvortmin%d.png' % (day_e))

"""


from ckeditor.fields import RichTextField
from django.contrib.auth.models import User
from django.db import models
from django.db.models import Sum

from overflow.models import QuestionModel

class AnswerModel(models.Model):
    answer = RichTextField(max_length=120)
    views = models.PositiveIntegerField(default=0)
    votes = models.IntegerField(default=0)
    question = models.ForeignKey(QuestionModel, null=True, on_delete=models.SET_NULL, related_name='am_question')
    answer_at = models.DateTimeField(auto_now_add=True)
    modified_on = models.DateTimeField(auto_now=True)
    answer_by = models.ForeignKey(User, null=True, on_delete=models.SET_NULL)

    def __repr__(self):
        return f'{self.answer}'

    def get_vote_count(self):
        result = self.avm_answer.aggregate(Sum('vote'))['vote__sum']
        if result:
            return result
        else:
            return 0

class AnswerVoteModel(models.Model):
    answer = models.ForeignKey(AnswerModel, null=True, on_delete=models.CASCADE, related_name='avm_answer')
    user = models.ForeignKey(User, null=True, on_delete=models.SET_NULL, related_name='avm_user')
    vote = models.IntegerField(default=0)
    created_at = models.DateTimeField(auto_now_add=True)
    modified_at = models.DateTimeField(auto_now=True)


import logging
from aiogram import types, Router, F
from aiogram.filters import Command, CommandObject
from Config import ADMIN_ID
from keyboards import ReplyKeyboard, InlineKeyboard
from .CallbackHandlers import userCallbackRout
from .CallbackData import ChangeSheduleCallbackData
from lexicon.Lexicon import weekday_names, short_weekday_names, list_settings, help_msg
from logic.Parser import load_json
from logic.Setting import get_settings, register
from datetime import date

userRout = Router()
userRout.include_router(userCallbackRout)

@userRout.message(Command(commands=['start', 'help', 'day', 'week', 'bagreport', 'setting', 'change']))
async def event_command(message: types.Message, command: CommandObject):
    logging.info(f'Username: {message.from_user.username}, id: {message.from_user.id}, Имя: {message.from_user.full_name}: {message.text}')
    # command = message.text.split()[0]
    if command.command == 'start':
        await welcome(message)
        register(message.from_user.id, message.from_user.full_name, message.from_user.username)
    elif command.command == 'help':
        await help(message)
    elif command.command == 'day':
        await day(message)
    elif command.command == 'week':
        await week(message)
    elif command.command == 'bagreport':
        await bagreport(message)
    elif command.command == 'setting':
        await setting(message)
    elif command.command == 'change':
        await change_shedule(message)

# Обрабатываем команду /start
async def welcome(message: types.Message):
    await message.answer_photo('AgACAgIAAxkBAAIPzWT1q-8bqxh0trAk87BvCHtch0rxAAINzDEbD-CxSwsdKruDaAABKAEAAwIAA3MAAzAE', 'Йоу')
    # await message.answer_photo(types.FSInputFile(r'img\statham.jpg'), 'Йоу')
    await help(message)
    await replace_keyboard(message, True)

# Обрабатываем команду /help
async def help(message: types.Message):
    await message.answer(text=help_msg)

# Обрабатываем команду /bagreport
async def bagreport(message: types.Message):
    try:
        report = message.text.split(' ', 1)[1]
        await message.bot.send_message(chat_id = ADMIN_ID, text = f'Баг репорт от пользователя Username: {message.from_user.username}, id: {message.from_user.id}, Имя: {message.from_user.full_name}:\n{report}')
        await message.answer('Информация передана!')
    except IndexError:
        await message.answer('Вы не ввели информацию об ошибке!\nСинтаксис данной команды:\n/bagreport [information]\n[information] - ваше сообщение об ошибке')

# Обрабатываем команду /setting
async def setting(message: types.Message):
    kb = InlineKeyboard.create_inline_kb(1, **list_settings, exit='🔚')
    await message.answer('Настройки:', reply_markup=kb)

# Обрабатываем команду /change
async def change_shedule(message: types.Message):
    kb = InlineKeyboard.create_inline_kb(3, **{ChangeSheduleCallbackData(change='add', week='', day='').pack():'Добавить', ChangeSheduleCallbackData(change='edit', week='', day='').pack():'Изменить', ChangeSheduleCallbackData(change='delete', week='', day='').pack():'Удалить'}, exit='🔚')
    await message.answer('Редактирование расписания\nIn developing...', reply_markup=kb)

# Смена клавиатуры
@userRout.message(F.text.in_({'<- Текущая неделя', 'Следующая неделя ->'}))
async def replace_keyboard(message: types.Message, cur_week: bool = True):
    if message.text == 'Следующая неделя ->' or cur_week == False:
        kb = ReplyKeyboard.kb_for_next_week()
        text = 'Выбрана следующая неделя'
    else:
        kb = ReplyKeyboard.kb_for_cur_week()
        text = 'Выбрана текущая неделя'

    await message.answer(text, reply_markup=kb)

# userRout.message.register(week, F.text.regexp('(?i)^неделя$'))
# userRout.message.register(week, F.text.regexp('(?i)^⠀?неделя⠀?$'))
# userRout.message.register(day, F.text.regexp('(?i)^⠀?(?:пн|вт|ср|чт|пт|сб|день)⠀?$'))
# userRout.message.register(day, F.text.regexp('(?i)^⠀(?:пн|вт|ср|чт|пт|сб)⠀$'))

# Обрабатываем команду /day
@userRout.message(F.text.regexp('(?i)^⠀?(?:пн|вт|ср|чт|пт|сб|день)⠀?$'))
async def day(message: types.Message):
    if '⠀' in message.text:
        week = 'next_week'
        msg = message.text.replace('⠀', '')
    else:
        week = 'current_week'
        msg = message.text

    if msg.lower() in ['пн', 'вт', 'ср', 'чт', 'пт', 'сб']:
        day = load_json(week, short_weekday_names[msg.lower()])
    else:
        day = load_json(week, weekday_names[date.today().weekday()])

    kol_msg = -1
    if day:
        if get_settings(message.from_user.id, 'day_block_messages'):
            block = '-'*20+'\n'
            for d in day[0]:
                if 'серверная' in d.lower() and message.from_user.id != 493465069:
                    continue
                block += d+'\n'+'-'*20+'\n'
                kol_msg +=1
            if kol_msg > 0:
                await message.answer(block)
            del block
        else:
            for d in day[0]:
                if 'серверная' in d.lower() and message.from_user.id != 493465069:
                    continue
                await message.answer('-'*20+'\n'+d+'\n'+'-'*20)
                kol_msg +=1

    if kol_msg <= 0:
        await message.answer_photo('AgACAgIAAxkBAAIPz2T1rBQOuJlA4QTrS93IPvP0NRK7AAIPzDEbD-CxS8up9voh4FXUAQADAgADcwADMAQ', 'Староста разрешает сегодня не ходить😎')
        # await message.answer_photo(types.FSInputFile(r'img\today.jpg'), 'Староста разрешает сегодня не ходить😎')

# Обрабатываем команду /week
@userRout.message(F.text.regexp('(?i)^⠀?неделя⠀?$'))
async def week(message: types.Message):
    if '⠀' in message.text:
        week = 'next_week'
    else:
        week = 'current_week'

    full_week = load_json(week)

    if get_settings(message.from_user.id, 'week_block_messages'):
        block = ''
        for row in full_week:
            block += '-'*20+'\n'
            for item in row:
                block += item+'\n'+'-'*20+'\n'
            block +='\n'
        await message.answer(block)
    else:
        for row in full_week:
            block = '-'*20+'\n'
            for item in row:
                block += item+'\n'+'-'*20+'\n'
            await message.answer(block)

# Обрабатываем все входящие сообщения
@userRout.message()
async def echo(message: types.Message):
    await message.answer_photo('AgACAgIAAxkBAAIP0WT1rDaARKe1yDQbb83bLVtH6H2EAAIQzDEbD-CxS-8Fu-K6d-g8AQADAgADcwADMAQ', 'Зачем ты сюда пишешь? Держи обои на рабочий стол :)')
    # photo = await message.answer_photo(types.FSInputFile(r'img\desktop.png'), 'Зачем ты сюда пишешь? Держи обои на рабочий стол :)')
    logging.info(f'Username: {message.from_user.username}, id: {message.from_user.id}, Имя: {message.from_user.full_name}: {message.text}')


import json
import logging
from pathlib import Path

from geopandas import GeoSeries
from pystac import Catalog, Extent, SpatialExtent, TemporalExtent, Asset, MediaType, STAC_IO
from pystac.extensions.eo import Band

from sac_stac.adapters.repository import S3Repository, NoObjectError
from sac_stac.domain.model import SacCollection, SacItem
from sac_stac.domain.operations import obtain_date_from_filename, get_geometry_from_cog, \
    get_projection_from_cog
from sac_stac.load_config import config, LOG_LEVEL, LOG_FORMAT, get_s3_configuration

logging.basicConfig(level=LOG_LEVEL, format=LOG_FORMAT)
logger = logging.getLogger(__name__)

S3_ENDPOINT = get_s3_configuration()["endpoint"]
S3_BUCKET = get_s3_configuration()["bucket"]
S3_STAC_KEY = get_s3_configuration()["stac_key"]
S3_CATALOG_KEY = f"{S3_STAC_KEY}/catalog.json"
S3_HREF = f"{S3_ENDPOINT}/{S3_BUCKET}"
GENERIC_EPSG = 4326

def create_collection(sensor_conf: dict, repo: S3Repository) -> SacCollection:
    logger.info(f"Creating {sensor_conf.get('id')} collection...")
    collection = SacCollection(
        id=sensor_conf.get('id'),
        title=sensor_conf.get('title'),
        description=sensor_conf.get('description'),
        extent=Extent(SpatialExtent([[0, 0, 0, 0]]), TemporalExtent([["", ""]])),
        properties={}
    )

    collection.add_providers(sensor_conf)
    collection.add_product_definition_extension(
        product_definition=sensor_conf.get('extensions').get('product_definition'),
        bands_metadata=sensor_conf.get('extensions').get('eo').get('bands')
    )

    catalog = get_create_catalog(repo)

    catalog.add_child(collection)
    catalog.normalize_hrefs(f"{S3_HREF}/{S3_STAC_KEY}")

    # TODO: Replace STAC_IO.write_text_method
    repo.add_json_from_dict(
        bucket=S3_BUCKET,
        key=S3_CATALOG_KEY,
        stac_dict=catalog.to_dict()
    )

    repo.add_json_from_dict(
        bucket=S3_BUCKET,
        key=f"{S3_STAC_KEY}/{sensor_conf.get('id')}/collection.json",
        stac_dict=collection.to_dict()
    )
    logger.info(f"{sensor_conf.get('id')} collection added to {S3_CATALOG_KEY}")

    return collection

def get_sensor_config(sensor_name: str) -> dict:
    sensor_configs = [s for s in config.get('sensors')]
    try:
        sensor_conf = [s for s in sensor_configs if s.get('id') == sensor_name][0]
    except IndexError:
        logger.warning(f"No config found for {sensor_name} sensor")
        return 'collection', None
    return sensor_conf

def get_create_catalog(repo: S3Repository) -> Catalog:
    try:
        catalog_dict = repo.get_dict(bucket=S3_BUCKET, key=S3_CATALOG_KEY)
        catalog = Catalog.from_dict(catalog_dict)
    except NoObjectError:
        logger.info(f"No catalog found in {S3_CATALOG_KEY}")
        logger.info("Creating new catalog...")
        catalog = Catalog(
            id=config.get('id'),
            title=config.get('title'),
            description=config.get('description'),
            stac_extensions=config.get('stac_extensions')
        )
    return catalog

def add_stac_collection(repo: S3Repository, sensor_key: str):
    STAC_IO.read_text_method = repo.stac_read_method

    sensor_name = sensor_key.split('/')[-2]
    sensor_conf = get_sensor_config(sensor_name)

    collection_key = f"{S3_STAC_KEY}/{sensor_name}/collection.json"
    try:
        repo.get_dict(bucket=S3_BUCKET, key=collection_key)
        logger.info(f"Collection {sensor_name} already exists in {collection_key}")
    except NoObjectError:
        create_collection(sensor_conf=sensor_conf, repo=repo)

    acquisition_keys = repo.get_acquisition_keys(bucket=S3_BUCKET,
                                                 acquisition_prefix=sensor_key)
    for acquisition_key in acquisition_keys:
        add_stac_item(repo=repo, acquisition_key=acquisition_key)

    return 'collection', collection_key

def add_stac_item(repo: S3Repository, acquisition_key: str):
    STAC_IO.read_text_method = repo.stac_read_method

    sensor_name = acquisition_key.split('/')[-3]
    collection_key = f"{S3_STAC_KEY}/{sensor_name}/collection.json"
    logger.debug(f"[Item] Adding {acquisition_key} item to {sensor_name}...")

    try:
        try:
            collection_dict = repo.get_dict(bucket=S3_BUCKET, key=collection_key)
            collection = SacCollection.from_dict(collection_dict)
        except NoObjectError:
            logger.info(f"No collection found in {S3_ENDPOINT}/{S3_BUCKET}/{collection_key}")
            sensor_conf = get_sensor_config(sensor_name)
            collection = create_collection(sensor_conf=sensor_conf, repo=repo)

        item_id = acquisition_key.split('/')[-2]
        item_key = f"{S3_STAC_KEY}/{collection.id}/{item_id}/{item_id}.json"
        try:
            repo.get_dict(bucket=S3_BUCKET, key=item_key)
            logger.info(f"Item {item_id} already exists in {item_key}")
        except NoObjectError:
            sensor_conf = [s for s in config.get('sensors') if s.get('id') == collection.id][0]
            logger.debug(f"[Item] Creating {item_id} item...")
            # Get date from acquisition name
            date = obtain_date_from_filename(
                file=acquisition_key,
                regex=sensor_conf.get('formatting').get('date').get('regex'),
                date_format=sensor_conf.get('formatting').get('date').get('format')
            )

            # Get sample product and extract geometry
            try:
                product_sample_key = repo.get_smallest_product_key(
                    bucket=S3_BUCKET,
                    products_prefix=acquisition_key
                )
                product_sample_href = f"{S3_HREF}/{product_sample_key}"
                geometry, crs = get_geometry_from_cog(product_sample_href)
            except NoObjectError:
                logger.error(f"No bands found on {acquisition_key} acquisition.")
                raise

            item = SacItem(
                id=item_id,
                datetime=date,
                geometry=json.loads(GeoSeries([geometry], crs=crs).to_crs(GENERIC_EPSG).to_json()).get('features')[0].get(
                    'geometry'),
                bbox=list(geometry.bounds),
                properties={}
            )

            item.ext.enable('projection')
            item.ext.projection.epsg = GENERIC_EPSG

            item.add_extensions(sensor_conf.get('extensions'))
            item.add_common_metadata(sensor_conf.get('common_metadata'))

            bands_metadata = sensor_conf.get('extensions').get('eo').get('bands')
            product_keys = repo.get_product_keys(bucket=S3_BUCKET, products_prefix=acquisition_key)

            for band_name, band_common_name in [(b.get('name'), b.get('common_name')) for b in bands_metadata]:
                asset_href = f"{S3_HREF}/{product_keys[0]}"
                proj_shp = [0, 0]
                proj_tran = [0, 0, 0, 0, 0, 0]

                band_name_in_product_keys = [p for p in product_keys if band_name in p]

                if band_name_in_product_keys:
                    product_key = band_name_in_product_keys[0]
                    asset_href = f"{S3_HREF}/{product_key}"
                    proj_shp, proj_tran = get_projection_from_cog(asset_href)
                else:
                    logger.warning(f"{band_name} band not found on {collection.id}/{item.id} acquisition.")

                asset = Asset(
                    href=asset_href,
                    media_type=MediaType.COG
                )

                # Set Projection
                item.ext.projection.set_transform(proj_tran, asset)
                item.ext.projection.set_shape(proj_shp, asset)

                # Set bands
                item.ext.eo.set_bands([Band.create(
                    name=band_common_name, common_name=band_common_name)],
                    asset
                )
                logger.debug(f"[Asset] Adding {asset_href} asset to {acquisition_key}...")
                item.add_asset(key=band_common_name, asset=asset)

            collection.add_item(item)
            collection.update_extent_from_items()
            collection.normalize_hrefs(f"{S3_HREF}/{S3_STAC_KEY}/{collection.id}")

            # TODO: Replace STAC_IO.write_text_method
            repo.add_json_from_dict(
                bucket=S3_BUCKET,
                key=collection_key,
                stac_dict=collection.to_dict()
            )

            repo.add_json_from_dict(
                bucket=S3_BUCKET,
                key=item_key,
                stac_dict=item.to_dict()
            )
            logger.info(f"{item.id} item added to {collection.id}")

        return 'item', item_key

    except TypeError:
        logger.error(f"Invalid collection in {collection_key}, "
                     f"could not add {acquisition_key}.")
        return 'item', None
    except KeyError:
        logger.error(f"No collection found in {collection_key},"
                     f"could not add {acquisition_key}.")
        return 'item', None
    except NoObjectError as e:
        logger.error(f"Could not find object in S3: {e}")
        return 'item', None