#!/usr/bin/env python
# coding: utf-8

# In[1]:

from sklearn import tree
import numpy as np
import pydot
import pandas as pd
import graphviz
import seaborn as sns
from tensorflow import keras
import matplotlib.pyplot as plt
import tensorflow as tf
from pylab import rcParams
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from sklearn import preprocessing
from sklearn.metrics import r2_score
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import mean_squared_error
from sklearn.metrics import accuracy_score
from sklearn.metrics import classification_report, confusion_matrix
from sklearn.metrics import plot_confusion_matrix

# In[4]:
# Initializing dataset

heart =pd.read_csv('D:/University Modules/Machine Learning/heart.csv')
# In[5]:
#Checking the dataset
heart

# In[6]:
#Exploring the data

#describing the data

heart.describe()
# In[7]:

#Checking the column types and selecting particular columns
heart_exp = heart.select_dtypes(include=["float64","int64"])
heart_exp.head()

# In[8]:
#Checking the scale of the dataset

heart.shape
# In[9]:
# EXPLORATORY DATA ANALYSIS------------------------------------------------------

#created a box plot for the dataset with respect to each column values

f, axes = plt.subplots(len(heart_exp.columns), 1, constrained_layout = True, figsize=([13,9]))

for i in range(len(heart_exp.columns)):

    sns.boxplot(x=heart_exp[heart_exp.columns[i]], ax=axes[i])

# In[10]:
heart.columns
# In[11]:

#Plotted a bar graph for patient with heart disease and no heart disease using the heartdisease as a target column
f = sns.countplot(x='HeartDisease', data=heart)
f.set_title("Heart disease presence distribution")
f.set_xticklabels(['No Heart disease', 'Heart Disease'])
plt.xlabel("");

# In[12]:
#plotting a bar graph for patient having heart diease with respect to their sex( M or F)

f = sns.countplot(x='HeartDisease', data=heart, hue='Sex')
plt.legend(['Female', 'Male'])
f.set_title("Heart disease presence by gender")
f.set_xticklabels(['No Heart disease', 'Heart Disease'])
plt.xlabel("");

# In[13]:
# Created a correlation matrix with respect to all the columns in the heart dataset.

heat_map = sns.heatmap(heart.corr(method='pearson'), annot=True, fmt='.2f', linewidths=2)
heat_map.set_xticklabels(heat_map.get_xticklabels());

sns.set() #

corr = heart.corr()
print(corr)

# In[14]:

corr['Age'].sort_values(ascending=False)

# In[15]:

plt.scatter(x=heart.Age[heart.HeartDisease==1], y=heart.RestingBP[(heart.HeartDisease==1)], c="red", s=60)
plt.scatter(x=heart.Age[heart.HeartDisease==0], y=heart.RestingBP[(heart.HeartDisease==0)], s=60)
plt.legend(["Disease", "No Disease"])
plt.xlabel("Age")
plt.ylabel("Maximum Heart Rate");

# In[16]:
#Created a new column to compare the data of people having heart disease and no heart disease with respect to Age
heart['Age_Category'] = pd.cut(heart['Age'],bins=list(np.arange(25, 85, 5)))

# In[17]:
#Created a bar plot and compared the both with repect to age group of people having heart disease and no heart disease.

plt.subplot(121)
heart[heart['HeartDisease']==1].groupby('Age_Category')['Age'].count().plot(kind='bar')
plt.title('Age Distribution of Patients with +ve Heart Diagonsis')

plt.subplot(122)
heart[heart['HeartDisease']==0].groupby('Age_Category')['Age'].count().plot(kind='bar')
plt.title('Age Distribution of Patients with -ve Heart Diagonsis')

# In[18]:
#Deleting the column created after the comparison as a part of EDA.

del heart['Age_Category']

# In[25]:
#Created a bar plot for the type chest pain people are having comparing the category of chestpaintype
#Comparing with respect to heart disease
f = sns.countplot(x='ChestPainType', data=heart, hue='HeartDisease')
f.set_xticklabels(['ATA', 'NAP', 'TSY','NA']);
f.set_title('Disease presence by chest pain type')
plt.ylabel('Chest Pain Type')
plt.xlabel('')
plt.legend(['No Disease', 'Disease']);

# In[20]:
#Comparing the Exersice Angina to the Sex Column for both male and females.

heart['Sex'] = np.where(heart['Sex'] == "F", 0, 1)
heart['ExerciseAngina'] = np.where(heart['ExerciseAngina'] == "N", 0, 1)

# In[31]:

#Plotted a comparison barplot for patients with different category of chestpain and checking the total amount of people having the type of pain.
sns.countplot(heart['ChestPainType'])
plt.xticks([0,1,2,3],["atypical angina", "non-anginal pain", "asymptomatic", "typical angina" ])
plt.xticks()
plt.show()

# In[32]:

# Showing Fasting Blood Sugar Distribution According To HeartDisease as Variable
sns.countplot(x="FastingBS", hue="HeartDisease", data=heart)
plt.legend(labels = ['No-Disease','Disease'])
plt.show()

# In[34]:

heart['RestingBP'].hist() #Check Resting Blood Pressure Distribution

# In[36]:

#Checking the blood pressure range for the patients with respect to their sex.
g = sns.FacetGrid(heart,hue="Sex",aspect=4)
g.map(sns.kdeplot, 'RestingBP', shade=True)
plt.legend(labels=['Male', 'Female'])

# EXPLORATORY DATA ANALYSIS------------------------------------------------------END
# In[32]:

#MODEL CLASSIFICATION------------------------------------------------------------------------START

# Manipulating data and creating dummy variables for Model processing.

heart = pd.get_dummies(heart)
heart.head().T

# In[81]:

#creating dataset to pass the value for test_train_split

y=heart["HeartDisease"]
x=heart.drop(["HeartDisease"], axis=1)

x_train, x_test , y_train, y_test = train_test_split(x,y,test_size = 0.25, random_state=42)

# In[35]:
# 1st MODEL DECISION TREE CLASSIFIER

decision=DecisionTreeClassifier().fit(x_train,y_train)

y_pred = decision.predict(x_test)
np.sqrt(mean_squared_error(y_test,y_pred))

# In[36]:
#Calcuating the model accuracy for Test Set and Training Set

print('Training set score: {:.4f}'.format(decision.score(x_train, y_train)*100))
print('Test set score: {:.4f}'.format(decision.score(x_test, y_test)*100))

# In[37]:
#Printed the classification report using the classification_report module from skelearn

print(classification_report(y_pred,y_test))

# In[38]:
#plotting the confusion matrix for decsion tree model

plot_confusion_matrix(decision, x_test, y_test)

#Getting Accuracy Score
decision.score(x_test, y_test)

accuracy_score_dt = round(decision.score(x_test, y_test)*100,2)

accuracy_score_dt

print("The accuracy score achieved using Neural Network is: "+str(accuracy_score_dt)+" %")

# In[85]: #Reference [3] from kaggle mentioned in Report
# HPYER PARAMETER TUNING FOR DECISION TREE CLASSIFIER

#creating a grid search CV classifier using decision tree

def evaluate_model(decision):
    print("Train Accuracy :", accuracy_score(y_train, decision.predict(x_train)))
    print("Train Confusion Matrix:")
    print(confusion_matrix(y_train, decision.predict(x_train)))
    print("-"*50)
    print("Test Accuracy :", accuracy_score(y_test, decision.predict(x_test)))
    print("Test Confusion Matrix:")
    print(confusion_matrix(y_test, decision.predict(x_test)))

# In[86]:
#evaluating the model accuracy
evaluate_model(decision)

# In[88]:

# Create the parameter grid based on the results of random search
params = {
    'max_depth': [2, 3, 5, 10, 20],
    'min_samples_leaf': [5, 10, 20, 50, 100],
    'criterion': ["gini", "entropy"]
}

# In[90]:

# Instantiate the grid search model
grid_search = GridSearchCV(estimator=decision,
                           param_grid=params,
                           cv=4, n_jobs=-1, verbose=1, scoring = "accuracy")

# In[91]:

#Running the Grid Search CV (Cross Validation for 4 folds on decision tree classifier under hyper parameters grid)
get_ipython().run_cell_magic('time', '', 'grid_search.fit(x_train, y_train)')

# In[92]:

#Calculating the accracy for the gridsearchcv model run on decision tree

score_df = pd.DataFrame(grid_search.cv_results_)
score_df.head()

# In[93]:
#Calculating the best score out of the all the results

heart_best=grid_search.best_estimator_

evaluate_model(heart_best)

# In[95]:

#created the classification report for the GridsearchCV classifier for hyper parameter tuning done on decision tree classifer.
print(classification_report(y_test, heart_best.predict(x_test)))
#Created a confusion matrix for gridsearchcv
plot_confusion_matrix(heart_best, x_train, y_train)
# In[46]:
## SECOND MODEL RANDOM FOREST CLASSIFIER

#Building up the random forest model.

rf_model = RandomForestClassifier().fit(x_train, y_train)

# In[47]:
#Creating a function and calculating the mean squared error for random forest

y_pred_rf = rf_model.predict(x_test)
np.sqrt(mean_squared_error(y_test, y_pred_rf))

# In[48]:
#Calculating the accuracy for test set and train for random forest classifier

print('Training set score: {:.4f}'.format(rf_model.score(x_train, y_train)*100))
print('Test set score: {:.4f}'.format(rf_model.score(x_test, y_test)*100))
# In[49]:

#Created a classification report for random forest model accuracy
print(classification_report(y_pred_rf,y_test))

#Getting the accuracy score for random forest classifier

rf_accuracy=accuracy_score(y_pred_rf,y_test)*100
rf_accuracy

print("The accuracy score achieved using Neural Network is: "+str(rf_accuracy)+" %")

# In[50]:
#plotting the confusion matrix for random forest

plot_confusion_matrix(rf_model, x_test, y_test)

# In[3]: # Reference [4] from Towards Data Science mentioned in the report
## Third MODEL USING KERAS BACKED BY TENSORFLOW

#for creating graph on white plots

sns.set(style='whitegrid', palette='muted', font_scale=1.5)

rcParams['figure.figsize'] = 14, 8

#For classifier in tensorflow

RANDOM_SEED = 42

np.random.seed(RANDOM_SEED)

# In[54]:
#Data PreProcessing
# Tensorflow feature columns are used here as the data contains a mixture of categorial and numerical data.
#so i have created functions for Numeric columns, bucketed other columns having float values and created embedded columns with the string and float values which are amended in the feature column
feature_columns = []

# numeric cols
for header in ['Age', 'RestingECG_Normal' ,'Cholesterol', 'RestingBP', 'Oldpeak']:
      feature_columns.append(tf.feature_column.numeric_column(header))

# bucketized cols
age = tf.feature_column.numeric_column("Age")
age_buckets = tf.feature_column.bucketized_column(age, boundaries=[25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 85])
feature_columns.append(age_buckets)

# indicator cols
heart["FastingBS"] = heart["FastingBS"].apply(str)
FBS = tf.feature_column.categorical_column_with_vocabulary_list(
      'FastingBS', ['0', '1', '2'])
FBS_one_hot = tf.feature_column.indicator_column(FBS)
feature_columns.append(FBS_one_hot)

heart["Sex_F"] = heart["Sex_F"].apply(str)
sex_F = tf.feature_column.categorical_column_with_vocabulary_list(
      'Sex_F', ['0', '1'])
sex_F_one_hot = tf.feature_column.indicator_column(sex_F)
feature_columns.append(sex_F_one_hot)

heart["Sex_M"] = heart["Sex_M"].apply(str)
sex_M = tf.feature_column.categorical_column_with_vocabulary_list(
      'Sex_M', ['0', '1'])
sex_M_one_hot = tf.feature_column.indicator_column(sex_M)
feature_columns.append(sex_M_one_hot)

heart["ST_Slope_Up"] = heart["ST_Slope_Up"].apply(str)
slope_Up = tf.feature_column.categorical_column_with_vocabulary_list(
      'ST_Slope_Up', ['0', '1', '2'])
slope_Up_one_hot = tf.feature_column.indicator_column(slope_Up)
feature_columns.append(slope_Up_one_hot)

heart["ST_Slope_Down"] = heart["ST_Slope_Down"].apply(str)
slope_Down = tf.feature_column.categorical_column_with_vocabulary_list(
      'ST_Slope_Down', ['0', '1', '2'])
slope_down_one_hot = tf.feature_column.indicator_column(slope_Down)
feature_columns.append(slope_down_one_hot)

heart["ChestPainType_TA"] = heart["ChestPainType_TA"].apply(str)
cp = tf.feature_column.categorical_column_with_vocabulary_list(
      'ChestPainType_TA', ['0', '1', '2'])
cp_one_hot = tf.feature_column.indicator_column(cp)
feature_columns.append(cp_one_hot)

# embedding cols
thal_embedding = tf.feature_column.embedding_column(FBS, dimension=8)
feature_columns.append(thal_embedding)

# crossed cols
age_FBS_crossed = tf.feature_column.crossed_column([age_buckets, FBS], hash_bucket_size=1000)
age_FBS_crossed = tf.feature_column.indicator_column(age_FBS_crossed)
feature_columns.append(age_FBS_crossed)

cp_slope_crossed = tf.feature_column.crossed_column([cp, slope_Up], hash_bucket_size=1000)
cp_slope_crossed = tf.feature_column.indicator_column(cp_slope_crossed)
feature_columns.append(cp_slope_crossed)

# In[55]:

# Created a pandas dataframe and converted it into a Tensorflow dataset

def create_dataset(dataframe, batch_size=32):
    dataframe = dataframe.copy()
    labels = dataframe.pop('HeartDisease')
    return tf.data.Dataset.from_tensor_slices((dict(dataframe), labels))     .shuffle(buffer_size=len(dataframe))     .batch(batch_size)

# In[56]:

#created a test trian split for the model
train, test = train_test_split(heart, test_size=0.2, random_state=RANDOM_SEED)

# In[57]:
#Created train and test dataset to fit in the model

train_ds = create_dataset(train)
test_ds = create_dataset(test)

# In[58]:
#Creating a binary classifier using tensorflow

model = tf.keras.models.Sequential([
        tf.keras.layers.DenseFeatures(feature_columns=feature_columns),
        tf.keras.layers.Dense(units=128, activation='relu'),
        tf.keras.layers.Dropout(rate=0.2),
        tf.keras.layers.Dense(units=128, activation='relu'),
        tf.keras.layers.Dense(units=1, activation='sigmoid')
])

# In[59]:

#This model will run the classifier build above for the batch size created and will show the accuracy and loss values for each iteration.

model.compile(optimizer='adam',
              loss='binary_crossentropy',
              metrics=['accuracy'])

history = model.fit(train_ds, validation_data=test_ds, epochs=100, use_multiprocessing=True)

# In[60]:

#finded the model accuracy for the test set
model.evaluate(test_ds)

# In[61]:
#finded the model accuracy for train set

model.evaluate(train_ds)

# In[62]:

#plotted a graph for the model accuracy
plt.plot(history.history['accuracy'])
plt.plot(history.history['val_accuracy'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.ylim((0, 1))
plt.legend(['train','test'], loc='upper left');

# In[63]:
#plotted a graph for the model loss.

plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()

# In[66]:
#Predicting the heart disease
#here the model is making a binary decsion, so we are taking the maximum binary probability of the output layer

predictions = model.predict(test_ds)
bin_predictions = tf.round(predictions).numpy().flatten()
y_test_keras = tf.round(model.predict(test_ds)).numpy().flatten()

# In[69]:

#Build up a classification report stating the accuracy and other scores.
print(classification_report(y_test_keras,bin_predictions))

#Getting the accuracy score for Keras Model:

keras_accuracy=accuracy_score(y_test_keras, bin_predictions)*100
keras_accuracy

print("The accuracy score achieved using Neural Network is: "+str(keras_accuracy)+" %")

# In[70]:

#Created a function for plotting the confusion matrix using the predicted values and test_labels
cnf_matrix = confusion_matrix(y_test_keras, bin_predictions)
cnf_matrix

# In[71]:
#Plotted the confusion matrix for the model.

class_names = [0,1]
fig,ax = plt.subplots()
tick_marks = np.arange(len(class_names))
plt.xticks(tick_marks,class_names)
plt.yticks(tick_marks,class_names)

sns.heatmap(pd.DataFrame(cnf_matrix),annot=True,cmap="Blues",fmt="d",cbar=False)
ax.xaxis.set_label_position('top')
plt.tight_layout()
plt.ylabel('Actual label')
plt.xlabel('Predicted label');

##MODEL CLASSIFICATION -----------------------------------------------------------------------------------END

#MODEL ACCURACY COMPARISON # Reference [1] from Github Mentioned in Report

# Created a list of scores and Models

scores=[accuracy_score_dt,rf_accuracy,keras_accuracy]
models= ['DecisionTreeClassifier','Random Forest Classifier', 'Sequnetial Model-Keras']

#Printing the accuracy of each models:

for i in range(len(models)):
    print("The accuracy score achieved using "+models[i]+" is: "+str(scores[i])+" %")

# Comapring the accuracy of each model

sns.set(rc={'figure.figsize':(15,8)})
plt.xlabel("Models")
plt.ylabel("Accuracy score")

sns.barplot(models,scores)


from scapy.layers.inet import *

from MNetwork import MHost

class Trace:
    def __init__(self, ip, *, port=80):
        self.ip = MHost.Host(str(ip)).GetHost()
        self.port = port

    def ICMP(self):
        ip = str(self.ip)

        ans, unans = sr(IP(dst=ip, ttl=(1, 30)) / ICMP(), timeout=3)
        res = {}
        for i in ans[IP]:
            if i.answer.src != ip:
                # print(i)
                res[i.answer.src] = i.query.ttl

        # res = sorted(set(res), key=res.values)
        res = sorted(res.items(), key=lambda kv: (kv[1], kv[0]))
        res = [x[0] for x in res]

        if len(res) != 0:
            return __getJson__("Trace-ICMP", True, "OK", res)
        else:
            return __getJson__("Trace-ICMP", False, "Timeout", res)

    def TCP(self):
        ip = str(self.ip)
        port = int(self.port)

        ans, unans = traceroute(ip, port, maxttl=30, timeout=3)
        res = {}
        for i in ans.get_trace()[ip]:
            if ans.get_trace()[ip][i][0] != ip:
                # print(str(i) + ":" + ans.get_trace()[ip][i][0])
                res[ans.get_trace()[ip][i][0]] = i

        res = sorted(res.items(), key=lambda kv: (kv[1], kv[0]))
        res = [x[0] for x in res]

        if len(res) != 0:
            return __getJson__("Trace-TCP", True, "OK", res)
        else:
            return __getJson__("Trace-TCP", False, "Timeout", res)

def __getJson__(protocol, state, msg, res):
    return {
        "protocol": protocol,
        "state": state,
        "msg": msg,
        "ip": res
    }


#!/usr/bin/env python3
# coding: utf-8
from __future__ import absolute_import, division, print_function

from builtins import filter, map, range, zip
from minitn.heom.corr import Drude

from minitn.heom.network import tensor_train_template
from minitn.heom.propagate import MultiLayer
from minitn.lib.backend import DTYPE, np
from minitn.lib.logging import Logger
from minitn.lib.tools import huffman_tree
from minitn.lib.units import Quantity
from minitn.models.sbm import SpinBoson
from minitn.tensor import Leaf, Tensor
from scipy import linalg as la

from minitn.models.particles import Phonon

# System:
e = Quantity(5000, 'cm-1').value_in_au
v = Quantity(500, 'cm-1').value_in_au
max_tier = 15
rank_heom = 1
temperature = 300
beta = Quantity(1 / temperature, 'K-1').value_in_au if temperature else None
# beta = None: ZT

ph_parameters = [
    #(Quantity(400, 'cm-1').value_in_au, Quantity(500, 'cm-1').value_in_au),
    #(Quantity(800, 'cm-1').value_in_au, Quantity(500, 'cm-1').value_in_au),
    #(Quantity(1200, 'cm-1').value_in_au, Quantity(500, 'cm-1').value_in_au),
    (Quantity(1600, 'cm-1').value_in_au, Quantity(500, 'cm-1').value_in_au),
]
dof = len(ph_parameters)
prefix = 'boson_dof{}_{}K_t{}_'.format(dof, temperature, max_tier)

drude = Drude(
    gamma=Quantity(20, 'cm-1').value_in_au,
    lambda_=Quantity(400, 'cm-1').value_in_au,
    beta=beta,
)

model = SpinBoson(
    sys_ham=np.array([[-0.5 * e, v], [v, 0.5 * e]], dtype=DTYPE),
    sys_op=np.array([[-0.5, 0.0], [0.0, 0.5]], dtype=DTYPE),
    ph_parameters=ph_parameters,
    ph_dims=(dof * [max_tier]),
    #bath_corr=drude,
    #bath_dims=[max_tier],
)

# init state
A, B = 1.0, 1.0
wfn_0 = np.array([A, B]) / np.sqrt(A**2 + B**2)
rho_0 = np.tensordot(wfn_0, wfn_0, axes=0)

# Propagation
dt_unit = Quantity(0.01, 'fs').value_in_au
callback_interval = 10
count = 50_00

def test_heom(fname=None):
    ph_dims = list(np.repeat(model.ph_dims, 2))
    n_dims = ph_dims if model.bath_dims is None else ph_dims + model.bath_dims
    print(n_dims)

    root = tensor_train_template(rho_0, n_dims, rank=rank_heom)
    leaves = root.leaves()
    h_list = model.heom_h_list(leaves[0], leaves[1], leaves[2:], beta=beta)

    solver = MultiLayer(root, h_list)
    solver.ode_method = 'RK45'
    solver.cmf_steps = solver.max_ode_steps  # use constant mean-field
    solver.ps_method = 'unite'
    #solver.svd_err = 1.0e-14

    # Define the obersevable of interest
    logger = Logger(filename=prefix + fname, level='info').logger
    logger2 = Logger(filename=prefix + "en_" + fname, level='info').logger
    for n, (time, r) in enumerate(
            solver.propagator(
                steps=count,
                ode_inter=dt_unit,
                split=True,
            )):
        # renormalized by the trace of rho
        norm = np.trace(np.reshape(np.reshape(r.array, (4, -1))[:, 0], (2, 2)))
        r.set_array(r.array / norm)
        if n % callback_interval == 0:
            t = Quantity(time).convert_to(unit='fs').value
            rho = np.reshape(r.array, (4, -1))[:, 0]
            logger.info("{}    {} {} {} {}".format(t, rho[0], rho[1], rho[2],
                                                   rho[3]))
            en = np.trace(np.reshape(rho, (2, 2)) @ model.h)
            logger2.info('{}    {}'.format(t, en))
    return

def test_mctdh(fname=None):
    n_state = 2
    sys_i = Leaf(name='sys-i')
    sys_j = Leaf(name='sys-j')
    sys_root = Tensor(name='elec', axis=0)

    rank_wfn = 1
    ph_spdos = []
    for n, (omega, g) in enumerate(ph_parameters, 1):
        ph_spdo = Tensor(name='ph{}'.format(n), axis=0)
        ph_spdos.append(ph_spdo)

    def ph_spf():
        t = Tensor(axis=0)
        t.name = 'spf' + str(hex(id(t)))[-4:]
        return t

    graph, ph_root = huffman_tree(ph_spdos, obj_new=ph_spf, n_branch=2)

    graph[sys_root] = [sys_i, sys_j]

    ph_is = []
    ph_js = []
    for n, ph_spdo in enumerate(ph_spdos, 1):
        phi_leaf = Leaf(name='phi{}'.format(n))
        phj_leaf = Leaf(name='phj{}'.format(n))
        graph[ph_spdo] = [phi_leaf, phj_leaf]
        ph_is.append(phi_leaf)
        ph_js.append(phj_leaf)

    root = Tensor(name='root', axis=None)
    graph[root] = [sys_root, ph_root]

    stack = [root]
    while stack:
        parent = stack.pop()
        for child in graph[parent]:
            parent.link_to(parent.order, child, 0)
            if child in graph:
                stack.append(child)

    # Define the detailed parameters for the ML-MCTDH tree
    h_list = model.do_l_list(sys_i, sys_j, ph_is, ph_js)
    solver = MultiLayer(root, h_list)
    bond_dict = {}
    # Leaves
    for s, i, t, j in root.linkage_visitor():
        if t.name.startswith('sys'):
            bond_dict[(s, i, t, j)] = n_state
        else:
            if isinstance(t, Leaf):
                bond_dict[(s, i, t, j)] = max_tier
            else:
                bond_dict[(s, i, t, j)] = rank_wfn
    solver.autocomplete(bond_dict)

    # set initial root array
    sys_root.set_array(np.array([rho_0], dtype=complex))
    for ph_spdo, (omega, _) in zip(ph_spdos, ph_parameters):
        if beta is None:
            # ZT
            array = np.zeros((max_tier, max_tier), dtype=complex)
            array[0, 0] = 1.0
        else:
            # FT
            h = Phonon(max_tier, omega).hamiltonian
            array = la.expm(-beta * h)
        ph_spdo.set_array(np.array([array], dtype=complex))

    solver = MultiLayer(root, h_list)
    solver.ode_method = 'RK45'
    solver.cmf_steps = solver.max_ode_steps  # constant mean-field
    solver.ps_method = 'unite'
    solver.svd_err = 1.0e-14

    # Define the obersevable of interest
    logger = Logger(filename=prefix + fname, level='info').logger
    logger2 = Logger(filename=prefix + 'en_' + fname, level='info').logger
    for n, (time, r) in enumerate(
            solver.propagator(
                steps=count,
                ode_inter=dt_unit,
                split=True,
            )):
        if n % callback_interval == 0:
            t = Quantity(time).convert_to(unit='fs').value
            for node in r.visitor(leaf=False):
                print("{}, {}".format(node, node.shape))

if __name__ == '__main__':
    import os
    import sys

    from matplotlib import pyplot as plt

    f_dir = os.path.abspath(os.path.dirname(__file__))
    os.chdir(os.path.join(f_dir, '2022data'))

    #test_heom(fname='heom.dat')
    test_mctdh(fname='do.dat')


from argparse import ArgumentParser

# ros
import rospy

# local
from .data_recorder import DataRecorder

def parse_args():
    parser = ArgumentParser()
    parser.add_argument("--path", type=str, default="./", help="data root path")
    parser.add_argument("--pro", action="store_true", help="whether to enable professional mode")
    return parser.parse_args()

def main(args=None):
    rospy.init_node("data_collection", anonymous=True)
    if args is None:
        args = parse_args()

    # gif generator
    data_recorder = DataRecorder(path=args.path, pro=args.pro)
    data_recorder.run()

if __name__ == "__main__":
    main()


import logging
import traceback

from django.utils.timezone import now
from rest_framework import response, status, viewsets
from rest_framework.response import Response

from api import authentication
from api.methods import clean_sensitive_data
from opentoilet import exceptions
from opentoilet.exceptions import APINotFound

log = logging.getLogger(__name__)

class BaseAPI(viewsets.GenericViewSet):
    authentication_classes = [authentication.JWTAuthentication]
    REQUIRES_AUTH = False
    ENTITLEMENTS = set()
    MODEL = None
    READ_SERIALIZER = None
    CREATE_SERIALIZER = None
    FILTER = None

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

    def initial(self, request, *args, **kwargs):
        super().initial(request, *args, **kwargs)

        if self.REQUIRES_AUTH:
            self._check_auth(request=request)

        if self.REQUIRES_AUTH and self.ENTITLEMENTS:
            self._check_entitlements(request=request)

    def _check_auth(self, request):
        if request.user.is_anonymous:
            raise exceptions.APIAccessDenied(
                "no_auth", "Missing authentication credentials"
            )

        # Store last activity for authenticated user
        if request.user.is_authenticated:
            request.user.account.last_activity = now()
            request.user.account.save()

    def _check_entitlements(self, request):
        if not self.ENTITLEMENTS.issubset(set(request.user.account.entitlements)):
            raise exceptions.APIAccessDenied(
                "missing_entitlements", f"Missing entitlements: {self.ENTITLEMENTS}"
            )

    def handle_exception(self, exc):
        tb = traceback.TracebackException.from_exception(exc)
        get_data = dict(self.request.GET)
        data = self.request.data
        post_data = []

        # Clean clean clean. Don't want to log this.
        # Also checking whether it's bulk insert or not
        if isinstance(data, list):
            for d in data:
                raw_post_data = dict(d)
                post_data.append(clean_sensitive_data(raw_post_data))
        else:
            raw_post_data = dict(self.request.data)
            post_data = clean_sensitive_data(raw_post_data)

        get_data = clean_sensitive_data(get_data)

        logging.error(
            "Exception=%s api=%s method=%s user=%s data=%s path=%s query=%s",
            "".join(tb.format()),
            self.get_view_name(),
            self.request.method,
            self.request.user.username,
            post_data,
            self.request.path,
            get_data,
        )

        try:
            return super().handle_exception(exc)
        except exceptions.APINotFound as e:
            return response.Response({"error": e.message, "code": e.code}, status=404)
        except (exceptions.APIException, exceptions.APIAccessDenied) as e:
            return response.Response({"error": e.message, "code": e.code}, status=400)
        except Exception:
            return response.Response(
                {"error": "Something went wrong.", "code": "unknown"}, status=500
            )

    def get_queryset(self):
        return self.MODEL.objects.all()

    def _get_instance(self, many=False, **kwargs):
        qs = self.MODEL.objects.get
        if many is True:
            qs = self.MODEL.objects.filter

        try:
            return qs(**kwargs)
        except self.MODEL.DoesNotExist:
            raise APINotFound()

    def _retrieve(self, request, pk):
        try:
            return Response(
                self.READ_SERIALIZER(instance=self._get_instance(pk=pk)).data,
                status=status.HTTP_200_OK,
            )
        except self.MODEL.DoesNotExist:
            raise APINotFound()

    def _list(self, request, **kwargs):
        if self.FILTER:
            f = self.FILTER(request.GET, queryset=self.get_queryset())
            qs = f.qs
        else:
            qs = self.MODEL.objects.filter(**kwargs)
        return self.get_paginated_response(
            self.READ_SERIALIZER(
                self.paginate_queryset(qs),
                many=True,
            ).data
        )

    def _create(self, request):
        serializer = self.CREATE_SERIALIZER(data=request.data)
        serializer.is_valid(raise_exception=True)
        instance = serializer.save()
        instance.actor = request.user
        instance.save()

        return Response(
            self.READ_SERIALIZER(instance=instance).data,
            status=status.HTTP_201_CREATED,
        )

    def _update(self, request, pk):
        instance = self._get_instance(pk=pk)
        serializer = self.CREATE_SERIALIZER(
            data=request.data, instance=instance, partial=True
        )
        serializer.is_valid(raise_exception=True)
        serializer.save()

        return Response(
            self.READ_SERIALIZER(instance=instance).data,
            status=status.HTTP_200_OK,
        )


import socket
import threading

nickname = input("Enter Name: ")

client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.connect(('192.168.214.67', 1234)) # IP address of client and portnumber

def receive():
    while True:
        message = client.recv(1024).decode('utf-8')
        if message == 'Name':
            client.send(nickname.encode('utf-8'))
        else:
            print(message)

def write():
    while True:
        message = '{}: {}'.format(nickname, input(''))
        client.send(message.encode('utf-8'))

receive_thread = threading.Thread(target=receive)
receive_thread.start()

write_thread = threading.Thread(target=write)
write_thread.start()


from datetime import timedelta
from datetime import datetime
from airflow import DAG
from airflow.operators.bash import BashOperator
# from airflow.operators.python_operator import PythonOperator
from airflow.utils.dates import days_ago

# [args]
default_args = {
    'owner': 'airflow',
    'depends_on_past': False,
    'email': ['hwa0327@yonsei.ac.kr'],
    'email_on_failure': False,
    'email_on_retry': False,
    'retries': 1,
    'retry_delay': timedelta(minutes=2),
}

# [DAG]
dag = DAG(
    'Gabage_Collector',
    default_args=default_args,
    description='kill finished chrome process and remove tmp file',
    schedule_interval='50 * * * *',
    start_date=datetime(2021,5,26, 6),
    tags=['gabage_collector'],
)

# [task]

kill_chrome_process = BashOperator(
    task_id='kill_chrome_process',
    depends_on_past=False,
    bash_command="/home/capje/airflow/kill_oldest_chrome.sh ",
    retries=3,
    dag=dag,
)

flush_tmp_dir = BashOperator(
    task_id='flush_tmp_directory',
    depends_on_past=False,
    bash_command="/home/capje/airflow/remove_chrome_tmp.sh ",
    retries=3,
    dag=dag,
)

# [dependency]


"""

Method of showing Regression model taken from Alice ChiaHui Lui (2018)
https://www.kaggle.com/chiahuiliu/avocado-exploratory-and-regression/notebook

FRITZ GERALD SANTOS - 20071968
Building a regression model

"""

import pandas
from sklearn.model_selection import train_test_split
import statsmodels.api as sm
from sklearn.metrics import explained_variance_score

#Import the dataset
dataframe = pandas.read_csv("avocado.csv")

#Cleaning the data
dataframe.rename(columns={'year': 'Year', 'region' : 'Region', 'type' : 'Type'}, inplace=True)

#cleaning - convert the data type of date into datetime.
dataframe["Date"] = pandas.to_datetime(dataframe["Date"])

#Cleaning - convert the data type of region to category
dataframe['Region'] = dataframe['Region'].astype('category')
dataframe['Region'] = dataframe['Region'].cat.codes

#Create dummy variables for type series - have two separate series for conventional and organic (representing 1 if true and 0 if false)
dummy_type = pandas.get_dummies(dataframe['Type'])
dataframe = pandas.concat([dataframe, dummy_type], axis=1)

#Split the dates into quarters of the year.
dataframe['Date_Q'] = dataframe['Date'].apply(lambda x: x.quarter)

#Create X and Y columns
X_columns = ['Total Volume', '4046', '4225', '4770', 'Total Bags', 'Small Bags', 'Large Bags', 'XLarge Bags', 'Year','Region', 'conventional', 'organic', 'Date_Q']
X = dataframe[X_columns]
Y = dataframe['AveragePrice']

#Create X and Y train to use for model
X_train, X_test, y_train, y_test  = train_test_split(X, Y, test_size=0.33, random_state=2018)

#For the regression model.
model = sm.OLS(y_train, X_train)
res = model.fit()
print(res.summary()) #print the regression model.


# -*- coding: utf-8 -*-
# pylint: skip-file
# by Roman Golev

__doc__ = "Returns element Volume for a selected elements. Calculate totals in square meters"
__author__ = 'Roman Golev'
__title__ = "Volume\nQuery"

from types import NoneType
import clr
clr.AddReference("RevitAPI")
import Autodesk
from Autodesk.Revit.DB import *
from Autodesk.Revit.UI import *
from Autodesk.Revit.UI import Selection

import sys

clr.AddReference('System')
clr.AddReference('RevitAPIUI')
import pyrevit
from pyrevit import forms
from System.Collections.Generic import List
from core.selectionhelpers import CustomISelectionFilterByIdInclude, ID_WALLS
from Autodesk.Revit.UI.Selection import ObjectType

doc = __revit__.ActiveUIDocument.Document
uidoc = __revit__.ActiveUIDocument
uiapp = __revit__
app = uiapp.Application
t = Autodesk.Revit.DB.Transaction(doc)

versionNumber = uiapp.Application.VersionNumber
if "2019" in versionNumber or "2020" in versionNumber:
        from Autodesk.Revit.DB import UnitUtils
elif "2021" in versionNumber or "2022" in versionNumber or "2023" in versionNumber or "2024" in versionNumber:
        from Autodesk.Revit.DB import UnitUtils, UnitTypeId
else:
        from Autodesk.Revit.DB import UnitUtils, UnitTypeId

categories = doc.Settings.Categories
n = categories.Size

# Get unput: selected by user elements
def get_selection():
     selobject = uidoc.Selection.GetElementIds()
     if selobject.Count == 0:
          try:
               selection = uidoc.Selection.PickObjects(ObjectType.Element, CustomISelectionFilterByIdInclude(ID_WALLS), "Selection Objects")
          except:
               sys.exit()
     elif selobject.Count != 0:
          selection = selobject
     return selection

selobject = get_selection()
volumes = []

def get_host_volume(elem):
        vl = elem.get_Parameter(BuiltInParameter.HOST_VOLUME_COMPUTED).AsDouble()
        if "2019" in versionNumber or "2020" in versionNumber:
                return UnitUtils.ConvertToInternalUnits(vl, DisplayUnitType.DUT_CUBIC_METERS)
        if "2021" in versionNumber or "2022" in versionNumber or "2023" in versionNumber:
                return UnitUtils.ConvertFromInternalUnits(vl, UnitTypeId.CubicMeters)

def get_ifc_volume(el):
        geo = el.get_Geometry(Options())
        vl = 0
        for i in geo:
                if "Solid" in i.ToString():
                        if "2019" in versionNumber or "2020" in versionNumber:
                                vl += UnitUtils.ConvertToInternalUnits(i.Volume, DisplayUnitType.DUT_CUBIC_METERS)
                        if "2021" in versionNumber or "2022" in versionNumber or "2023" in versionNumber:
                                vl += UnitUtils.ConvertFromInternalUnits(i.Volume, UnitTypeId.CubicMeters)
        return vl

if selobject.Count == 0:
        print('Nothing selected')
elif selobject.Count != 0:
        for element in selobject:
                el = doc.GetElement(element)
                try:
                        vol = get_host_volume(el)
                        if float(vol) == 0.0:
                                raise ValueError
                        volumes.append(vol)
                        print(vol)
                except ValueError:
                        try:
                                vol = get_ifc_volume(el)
                                print(vol)
                                volumes.append(vol)
                        except:
                                print("Couldn't get the volume of element {}".format(el.Id))
                except:
                        print("Couldn't get the volume of element {}".format(el.Id))

print("Summary : {}".format(sum(volumes)) + " m³")

#TODO: Add option to calculate volume in case if it is no Volume property in model
# Based on Geometry


import sys
input = sys.stdin.readline

while True:
    b = input().strip()
    if b == '#':
        break
    p = b[-1]
    b = list(map(int, b[:-1]))
    b.append(0 if ['e', 'o'].index(p) == sum(b)%2 else 1)
    print(*b, sep='')


import heapq
def solution(n, edge):
    answer = 0
    b=dict()
    for i in range(1,n+1):
        b[i]=[]
    for j in edge:
        b[j[0]].append(j[1])
        b[j[1]].append(j[0])
    last_distance=dijkstra(b,n)
    max_distance=max(last_distance)
    for i in last_distance:
        if i==max_distance:
            answer+=1
    return answer

def dijkstra(graph,n):
    distances=[float('inf')]*n
    distances[0]=0
    queue=[]
    heapq.heappush(queue,[distances[0],1])
    print(queue)

    while queue:
        current_distance,current_node=heapq.heappop(queue)
        if distances[current_node-1] <current_distance:
            continue
        for adjacent in graph[current_node]:
            distance=current_distance+1

            if distance <distances[adjacent-1]:
                distances[adjacent-1] = distance
                heapq.heappush(queue,[distance,adjacent])
        print(queue)
    return distances
print(solution(6,[[3, 6], [4, 3], [3, 2], [1, 3], [1, 2], [2, 4], [5, 2]]))


from datetime import datetime
import json
from time import sleep
from typing import Dict, List, Optional, Tuple, Union
from threading import Event, Thread
from argparse import ArgumentParser

import asyncio
import auraxium
from auraxium import ps2
import random

from constants import Team, Classes, MetagameEventType, MetagameEventState, Faction, Vehicles, VEHICLE_WEAPONS, World
from dataclass import OutfitwarsInstance, Outfit, Teams
from events import MetagameEvent, DeathEvent, FacilityControlEvent, VehicleDestroyEvent
from service import get_rabbit, RabbitConnection, RabbitService, Logger
from state import NexusMap
from replay import MetagameReplayEvent, FacilityControlReplayEvent, DeathReplayEvent, VehicleDestroyReplayEvent

logger = Logger.getLogger("Match")

#BLUE_OUTFIT = 37571208657592881 # [HMRD]
#RED_OUTFIT = 37570391403474491  # [UN17]
MAX_CAPTURES = 20

VS_WEAPON = 80   # Orion
NC_WEAPON = 7169 # GD-7F
TR_WEAPON = 7254 # MSW-R
NSO_WEAPON = 6009899 # XMG-100

weapons = {
    Faction.VS: VS_WEAPON,
    Faction.NC: NC_WEAPON,
    Faction.TR: TR_WEAPON,
    Faction.NSO: NSO_WEAPON
}

tiebreaker_points = { Team.RED: 0, Team.BLUE: 0 }
tbpool = 500

def tiebreaker_worker(interval: float, nexus: NexusMap) -> Tuple[Thread, Event]:
    stop = Event()
    def work():
        global tiebreaker_points, tbpool
        while not stop.wait(interval):
            for team in tiebreaker_points:
                points = nexus.count(team) * 2
                if points > tbpool:
                    points = tbpool
                tiebreaker_points[team] += points
                tbpool -= points
    thread = Thread(target=work)
    return thread, stop

def vehicle_destroy_worker(
        interval: float,
        event: OutfitwarsInstance,
        players: Dict[int, List[int]] = {
            37571208657592881: [5428085259783352737],
            37570391403474491: [5428297992006446465, 5428985062301123025]
        },
        delay: float = 0
    ) -> Tuple[Thread, Event]:
    stop = Event()
    def work():
        global weapons
        sleep(delay)
        vehicle_destroy_queue = f'aggregator-outfitwars-{event.world}-{event.zone}-{event.censusInstanceId}-VehicleDestroy'
        thread_rabbit = RabbitService(RabbitConnection())
        while not stop.wait(interval):
            attackerOutfitId = random.choice(list(players.keys()))
            victimOutfitId = random.choice(list(players.keys()))
            i = 0
            while attackerOutfitId == victimOutfitId and i < 5: # 0.5^5 == 3.125% tk rate
                victimOutfitId = random.choice(list(players.keys()))
                i += 1
            attackerOutfit = event.teams.red if event.teams.red.id == attackerOutfitId else event.teams.blue
            attacker = random.choice(players[attackerOutfitId])
            attackerVehicle = random.choice([0, *Vehicles.by_faction(attackerOutfit.faction)])
            if attackerVehicle != 0:
                attackerWeapon = VEHICLE_WEAPONS[attackerVehicle][attackerOutfit.faction]
            else:
                attackerWeapon = weapons[attackerOutfit.faction]
            victimOutfit = event.teams.red if event.teams.red.id == victimOutfitId else event.teams.blue
            victim = random.choice(players[victimOutfitId])
            victimVehicle = random.choice(Vehicles.by_faction(victimOutfit.faction))
            vehicleDestroy = VehicleDestroyEvent(
                world_id=str(int(event.world)),
                attacker_character_id=str(attacker),
                attacker_loadout_id=str(int(random.choice(Classes.by_faction(attackerOutfit.faction)))),
                attacker_team_id='2' if attackerOutfitId == event.teams.blue.id else '3',
                attacker_vehicle_id=str(attackerVehicle),
                attacker_weapon_id=str(attackerWeapon),
                character_id=str(victim),
                faction_id=str(int(victimOutfit.faction)),
                team_id='2' if victimOutfitId == event.teams.blue.id else '3',
                vehicle_id=str(victimVehicle),
                zone_id=(int(event.zoneInstanceId) << 16) | int(event.zone),
            )
            thread_rabbit.send(vehicleDestroy, vehicle_destroy_queue)
    thread = Thread(target=work)
    return thread, stop

def death_worker(
        interval: float,
        event: OutfitwarsInstance,
        players: Dict[int, List[int]] = {
            37571208657592881: [5428085259783352737],
            37570391403474491: [5428297992006446465, 5428985062301123025]
        },
        delay: float = 0
    ) -> Tuple[Thread, Event]:
    stop = Event()
    def work():
        global weapons
        sleep(delay)
        death_queue = f'aggregator-outfitwars-{event.world}-{event.zone}-{event.censusInstanceId}-Death'
        thread_rabbit = RabbitService(RabbitConnection())
        #players = [BLUE_PLAYER, RED_PLAYER, NSO_RED_PLAYER]
        while not stop.wait(interval):
            attackerOutfitId = random.choice(list(players.keys()))
            victimOutfitId = random.choice(list(players.keys()))
            i = 0
            while attackerOutfitId == victimOutfitId and i < 5: # 0.5^5 == 3.125% tk rate
                victimOutfitId = random.choice(list(players.keys()))
                i += 1
            attackerOutfit = event.teams.red if event.teams.red.id == attackerOutfitId else event.teams.blue
            attacker = random.choice(players[attackerOutfitId])
            victimOutfit = event.teams.red if event.teams.red.id == victimOutfitId else event.teams.blue
            victim = random.choice(players[victimOutfitId])
            death = DeathEvent(
                attacker_character_id=str(attacker),
                attacker_loadout_id=str(int(random.choice(Classes.by_faction(attackerOutfit.faction)))),
                attacker_weapon_id=str(weapons[attackerOutfit.faction]),
                attacker_team_id='2' if attackerOutfitId == event.teams.blue.id else '3',
                character_id=str(victim),
                character_loadout_id=str(int(random.choice(Classes.by_faction(victimOutfit.faction)))),
                is_headshot=str(1 if random.random() < 0.25 and attacker != victim else 0),
                team_id='2' if victimOutfitId == event.teams.blue.id else '3',
                world_id=str(event.world),
                zone_id=(int(event.zoneInstanceId) << 16) | int(event.zone)
            )
            thread_rabbit.send(death, death_queue)
    thread = Thread(target=work)
    return thread, stop

def nexus_alert(
    world: int,
    instance: int,
    teams: Teams = Teams(
        Outfit(37570391403474491, name="Un1ty", faction=Faction.TR, alias="UN17"),
        Outfit(37571208657592881, name="Drunk Division", faction=Faction.TR, alias="HMRD")
    ),
    members: Dict[int, List[int]] = {
        37570391403474491: [5428297992006446465, 5428985062301123025],
        37571208657592881: [5428085259783352737]
    },
    capture_rate: int = 5,
    death_rate: float = 0.5,
    first_death_delay: float = 0.0,
    vehicle_destroy_rate: float = 30.0,
    vehicle_destroy_delay: float = 0.0) -> Optional[Tuple[Team, Dict[Team, int]]]:

    metagame_event_queue = f'aggregator-{world}-MetagameEvent'

    zone_id = (instance << 16) | 10
    event = OutfitwarsInstance(
        censusInstanceId=random.randint(1, 65535),
        zoneInstanceId=str(instance),
        world=world,
        zone=10,
        timeStarted=datetime.now(),
        phase=random.randint(1, 3),
        round=random.randint(1, 7),
        teams=teams
    )

    participant_count = len(members[event.teams.blue.id]) + len(members[event.teams.red.id])
    faction_blue = int((len(members[event.teams.blue.id]) / participant_count) * 100)
    faction_red = 100 - faction_blue

    metagame_event = MetagameEvent(
        instance_id=str(event.censusInstanceId),
        metagame_event_id=str(int(MetagameEventType.NEXUS_OUTFIT_WAR)),
        metagame_event_state=str(int(MetagameEventState.STARTED)),
        metagame_event_state_name="started",
        world_id=str(int(event.world)),
        zone_id=(int(event.zoneInstanceId) << 16) | event.zone,
        faction_vs='0',
        faction_nc=str(faction_blue),
        faction_tr=str(faction_red),
    )
    rabbit = get_rabbit()
    rabbit.send(metagame_event, metagame_event_queue)
    death_thread, death_stop_event = None, None
    vehicle_destroy_thread, vehicle_destroy_stop_event = None, None
    tb_thread, tb_stop_event = None, None
    nexus = None
    try:
        nexus = NexusMap(zone_id & 0xFFFF)
        to_capture = 0
        captures = MAX_CAPTURES
        fac_control_queue = f'aggregator-outfitwars-{int(event.world)}-{int(event.zone)}-{event.censusInstanceId}-FacilityControl'
        death_thread, death_stop_event = death_worker(death_rate, event, members, first_death_delay)
        vehicle_destroy_thread, vehicle_destroy_stop_event = vehicle_destroy_worker(vehicle_destroy_rate, event, members, vehicle_destroy_delay)
        tb_thread, tb_stop_event = tiebreaker_worker(60, nexus)
        tb_thread.start()
        death_thread.start()
        vehicle_destroy_thread.start()
        while to_capture not in ['310610', '310600'] and captures > 0:
            sleep(capture_rate)
            team = random.choice([Team.BLUE, Team.RED])
            to_capture = random.choice(nexus.get_capturable(team))
            old_faction = nexus.get_region(to_capture).faction
            nexus.capture(to_capture, team)
            # it appears from the PTS match that outfits might not get credit for captures.
            # Now that we have team id, we can compensate for that on our side ^.^
            if team == Team.BLUE and event.teams.blue.faction == Team.BLUE:
                outfit_id = str(event.teams.blue.id)
            elif team == Team.RED and event.teams.red.faction == Team.RED:
                outfit_id = str(event.teams.red.id)
            else:
                outfit_id = "0"
            facility_control = FacilityControlEvent(
                facility_id=str(to_capture),
                zone_id=str(zone_id),
                world_id=str(int(event.world)),
                old_faction_id=str(int(old_faction)),
                new_faction_id=str(int(team)),
                outfit_id=outfit_id,
            )
            rabbit.send(facility_control, fac_control_queue)
            captures -= 1

        if to_capture in ['310610', '310600']:
            tiebreaker_points[team] += tbpool
            tbpool = 0
    finally:
        metagame_end_event = MetagameEvent(
            instance_id=str(event.censusInstanceId),
            metagame_event_id=str(int(MetagameEventType.NEXUS_OUTFIT_WAR)),
            metagame_event_state=str(int(MetagameEventState.FINISHED)),
            metagame_event_state_name="ended",
            world_id=str(int(event.world)),
            zone_id=(int(event.zoneInstanceId) << 16) | event.zone,
            faction_vs='0',
            faction_nc=str(faction_blue),
            faction_tr=str(faction_red),
        )
        if death_stop_event and death_thread:
            death_stop_event.set()
            if death_thread.is_alive():
                death_thread.join()

        if vehicle_destroy_stop_event and vehicle_destroy_thread:
            vehicle_destroy_stop_event.set()
            if vehicle_destroy_thread.is_alive():
                vehicle_destroy_thread.join()

        if tb_stop_event and tb_thread:
            tb_stop_event.set()
            if tb_thread.is_alive():
                tb_thread.join()

        rabbit.send(metagame_end_event, metagame_event_queue)
        if nexus:
            points = {
                Team.RED: tiebreaker_points[Team.RED],
                Team.BLUE: tiebreaker_points[Team.BLUE]
            }
            winner = Team.RED if nexus.territory(Team.RED) > nexus.territory(Team.BLUE) else Team.BLUE
            return winner, points

def replay_vehicle_destroy_worker(
        event: OutfitwarsInstance,
        vehicle_destroys: List[VehicleDestroyReplayEvent]
    ) -> Thread:
    def work():
        global weapons
        vehicle_destroy_queue = f'aggregator-outfitwars-{event.world}-{event.zone}-{event.censusInstanceId}-VehicleDestroy'
        thread_rabbit = RabbitService(RabbitConnection())
        index = 0
        interval = (datetime.fromtimestamp(int(vehicle_destroys[0].timestamp)) - event.timeStarted).total_seconds()
        while index < len(vehicle_destroys):
            replay_event = vehicle_destroys[index]
            sleep(interval)
            print('Next Vehicle Destroy event in: '+str(interval)+' seconds...')
            vehicleDestroy = VehicleDestroyEvent(
                world_id=str(int(event.world)),
                attacker_character_id=replay_event.attacker_character_id,
                attacker_loadout_id=replay_event.attacker_loadout_id,
                attacker_team_id=replay_event.attacker_team_id,
                attacker_vehicle_id=replay_event.attacker_vehicle_id,
                attacker_weapon_id=replay_event.attacker_weapon_id,
                character_id=replay_event.character_id,
                faction_id=replay_event.faction_id,
                team_id=replay_event.team_id,
                vehicle_id=replay_event.vehicle_definition_id,
                zone_id=replay_event.zone_id,
                timestamp=replay_event.timestamp
            )
            thread_rabbit.send(vehicleDestroy, vehicle_destroy_queue)
            if index + 1 < len(vehicle_destroys):
                interval = (datetime.fromtimestamp(int(vehicle_destroys[index + 1].timestamp)) - datetime.fromtimestamp(int(vehicle_destroys[index].timestamp))).total_seconds()
            index += 1
    thread = Thread(target=work)
    return thread

def replay_death_worker(
        event: OutfitwarsInstance,
        death_replay_events: List[DeathReplayEvent]
    ) -> Thread:
    def work():
        death_queue = f'aggregator-outfitwars-{event.world}-{event.zone}-{event.censusInstanceId}-Death'
        thread_rabbit = RabbitService(RabbitConnection())
        index = 0
        interval = (datetime.fromtimestamp(int(death_replay_events[0].timestamp)) - event.timeStarted).total_seconds()
        #players = [BLUE_PLAYER, RED_PLAYER, NSO_RED_PLAYER]
        while index < len(death_replay_events):
            replay_event = death_replay_events[index]
            print('Next Death event in: '+str(interval)+' seconds...')
            sleep(interval)
            death = DeathEvent(
                attacker_character_id=replay_event.attacker_character_id,
                attacker_loadout_id=replay_event.attacker_loadout_id,
                attacker_weapon_id=replay_event.attacker_weapon_id,
                attacker_team_id=replay_event.attacker_team_id,
                character_id=replay_event.character_id,
                character_loadout_id=replay_event.character_loadout_id,
                is_headshot=replay_event.is_headshot,
                team_id=replay_event.team_id,
                world_id=replay_event.world_id,
                zone_id=replay_event.zone_id,
                timestamp=replay_event.timestamp
            )
            thread_rabbit.send(death, death_queue)
            if index + 1 < len(death_replay_events):
                interval = (datetime.fromtimestamp(int(death_replay_events[index + 1].timestamp)) - datetime.fromtimestamp(int(death_replay_events[index].timestamp))).total_seconds()
            index += 1
    thread = Thread(target=work)
    return thread

def replay_facility_control_worker(
    event: OutfitwarsInstance,
    facility_events: List[FacilityControlReplayEvent]
) -> Thread:
    def work():
        fac_control_queue = f'aggregator-{event.instanceId}-FacilityControl'
        thread_rabbit = RabbitService(RabbitConnection())
        index = 0
        interval = (datetime.fromtimestamp(int(facility_events[0].timestamp)) - event.timeStarted).total_seconds()
        while index < len(facility_events):
            replay_event = facility_events[index]
            print(replay_event)
            print('Next Facility event in: '+str(interval)+' seconds...')
            sleep(interval)
            facility_control = FacilityControlEvent(
                facility_id=replay_event.facility_id,
                zone_id=replay_event.zone_id,
                world_id=replay_event.world_id,
                old_faction_id=replay_event.faction_old,
                new_faction_id=replay_event.faction_new,
                outfit_id=replay_event.outfit_id,
                timestamp=replay_event.timestamp,
                duration_held=replay_event.duration_held
            )
            print('Sending FacilityControl message to queue ' + fac_control_queue)
            print(facility_control)
            thread_rabbit = RabbitService(RabbitConnection())
            thread_rabbit.send(facility_control, fac_control_queue)
            thread_rabbit.close()
            if index + 1 < len(facility_events):
                interval = (datetime.fromtimestamp(int(facility_events[index + 1].timestamp)) - datetime.fromtimestamp(int(facility_events[index].timestamp))).total_seconds()
            index += 1
    thread = Thread(target=work)
    return thread

def replay(
    metagame_replay_event: MetagameReplayEvent,
    metagame_replay_end_event: MetagameReplayEvent,
    facility_events: List[FacilityControlReplayEvent],
    death_events: List[DeathReplayEvent],
    vehicle_destroy_events: List[VehicleDestroyReplayEvent]
):
    world = World(int(metagame_replay_event.world_id))
    censusInstance = int(metagame_replay_event.instance_id)
    metagame_event_queue = f'aggregator-{world.value}-MetagameEvent'

    zone_id = int(metagame_replay_event.zone_id)
    event = OutfitwarsInstance(
        censusInstanceId=censusInstance,
        zoneInstanceId=((zone_id >> 16) & 0xFFFF),
        world=world,
        zone=zone_id & 0xFFFF,
        timeStarted=datetime.fromtimestamp(int(metagame_replay_event.timestamp)),
        phase=1,
        round=2, #really doesn't matter for a replay
        teams=None
    )

    metagame_event = MetagameEvent(
        instance_id=metagame_replay_event.instance_id,
        metagame_event_id=metagame_replay_event.metagame_event_id,
        metagame_event_state=metagame_replay_event.metagame_event_state,
        metagame_event_state_name=metagame_replay_event.metagame_event_state_name,
        world_id=metagame_replay_event.world_id,
        zone_id=metagame_replay_event.zone_id,
        faction_vs=metagame_replay_event.faction_vs,
        faction_nc=metagame_replay_event.faction_nc,
        faction_tr=metagame_replay_event.faction_tr,
    )
    rabbit = get_rabbit()
    rabbit.send(metagame_event, metagame_event_queue)
    death_thread = None
    vehicle_destroy_thread = None
    try:
        death_thread = replay_death_worker(event, death_events)
        vehicle_destroy_thread = replay_vehicle_destroy_worker(event, vehicle_destroy_events)
        facility_control_thread = replay_facility_control_worker(event, facility_events)
        death_thread.start()
        vehicle_destroy_thread.start()
        facility_control_thread.start()
        death_thread.join()
        vehicle_destroy_thread.join()
        facility_control_thread.join()
    finally:
        metagame_replay_end_event
        metagame_end_event = MetagameEvent(
            instance_id=metagame_replay_end_event.instance_id,
            metagame_event_id=metagame_replay_end_event.metagame_event_id,
            metagame_event_state=metagame_replay_end_event.metagame_event_state,
            metagame_event_state_name=metagame_replay_end_event.metagame_event_state_name,
            world_id=metagame_replay_end_event.world_id,
            zone_id=metagame_replay_end_event.zone_id,
            faction_vs=metagame_replay_end_event.faction_vs,
            faction_nc=metagame_replay_end_event.faction_nc,
            faction_tr=metagame_replay_end_event.faction_tr,
            timestamp=metagame_replay_end_event.timestamp
        )
        rabbit.send(metagame_end_event, metagame_event_queue)

async def build_outfit_data(service_id: str, red_outfit_id: int, blue_outfit_id: int):
    if not service_id.startswith("s:"):
        service_id = f"s:{service_id}"
    async with auraxium.Client(service_id=service_id) as client:
        logger.info("Retrieving outfit data...")
        red_outfit, blue_outfit = await asyncio.gather(
            client.get_by_id(ps2.Outfit, red_outfit_id),
            client.get_by_id(ps2.Outfit, blue_outfit_id)
        )
        if red_outfit is None or blue_outfit is None:
            raise ValueError(f"{'Both' if red_outfit is None and blue_outfit is None else ('Red' if red_outfit is None else 'Blue')} outfit(s) do(es) not exist!")

        (red_leader, #standing by
         blue_leader, red_members, blue_members) = await asyncio.gather(
            red_outfit.leader(),
            blue_outfit.leader(),
            red_outfit.members(),
            blue_outfit.members()
        )
        red_leader.character()

        (red_leader_character, blue_leader_character) = await asyncio.gather(
            red_leader.character(),
            blue_leader.character()
        )

        red_outfit_data = Outfit(red_outfit.id, red_outfit.name, red_leader_character.faction_id, red_outfit.alias)
        blue_outfit_data = Outfit(blue_outfit.id, blue_outfit.name, blue_leader_character.faction_id, blue_outfit.alias)
        character_ids = {
            red_outfit.id: [member.id for member in red_members[:48]],
            blue_outfit.id: [member.id for member in blue_members[:48]],
        }

    return Teams(red_outfit_data, blue_outfit_data), character_ids

def start_alert(teams: Teams, members: Dict[int, List], world: int, instance: int, capture_rate: int, death_rate: float, death_delay: float, vehicle_destroy_rate: float, vehicle_destroy_delay: float):

    logger.info(f"Starting alert on world {world}, zone instance {instance}")
    result = nexus_alert(world, instance, teams, members, capture_rate, death_rate, death_delay, vehicle_destroy_rate, vehicle_destroy_delay)
    logger.info(f"Finished alert on world {world}, zone instance {instance}")
    if result is not None:
        logger.info(f"Winner was team {result[0].name.capitalize()}")
    return instance, result

# Simulator that fakes an alert on Nexus with a bunch of random (possible) captures
#   Basically intended to test out various scenarios that could happen with outfit wars
def main():
    global logger
    parser = ArgumentParser(description="A tool that can fake a single outfit wars match between two outfits")

    subparsers = parser.add_subparsers(dest="subparser_name", required=True)
    fake_match_parser = subparsers.add_parser("fake", help="Fake an outfitwars match for testing")
    fake_match_parser.add_argument("service_id", type=str, help="The PS2 Census service id used for querying outfit (member) data")
    fake_match_parser.add_argument("--red-outfit-id", "-r", type=int, default=37570391403474491, help="The outfit id for the red team")
    fake_match_parser.add_argument("--blue-outfit-id", "-b", type=int, default=37571208657592881, help="The outfit id for the blue team")
    fake_match_parser.add_argument("--world", "-w", type=int, choices=[1, 10, 13, 17], default=1, help="The world having the outfit war")
    fake_match_parser.add_argument("--capture-rate", "-c", type=int, default=5, help="The number of seconds between base captures")
    fake_match_parser.add_argument("--death-rate", "-d", type=float, default=1.0, help="The number of seconds between deaths (can be a decimal)")
    fake_match_parser.add_argument("--death-delay", "-l", type=float, default=0.0, help="The decimal number of seconds to delay the first death by")

    replay_parser = subparsers.add_parser("replay", help="Replay an outfitwars match to fix failed matches")
    replay_parser.add_argument("metagame_events_file", type=str)
    replay_parser.add_argument("facility_events_file", type=str)
    replay_parser.add_argument("death_events_file", type=str)
    replay_parser.add_argument("vehicle_destroy_events_file", type=str)
    args = parser.parse_args()

    if args.subparser_name == "fake":
        try:
            teams, members = asyncio.get_event_loop().run_until_complete(build_outfit_data(args.service_id, args.red_outfit_id, args.blue_outfit_id))
        except ValueError as e:
            logger.error(str(e))
            exit(1)
        start_alert(teams, members, args.world, random.randint(0, 0xFFFF), args.capture_rate, args.death_rate, args.death_delay, 5, 5)
    else:
        with open(args.metagame_events_file) as f:
            metagame_events = json.load(f)
        assert len(metagame_events) == 2, "Should have only a start and end event for a single alert!"
        with open(args.facility_events_file) as f:
            facility_events_data = json.load(f)
        with open(args.death_events_file) as f:
            death_events_data = json.load(f)
        with open(args.vehicle_destroy_events_file) as f:
            vehicle_destroy_events_data = json.load(f)

        logger.info("Building replay event objects...")
        metagame_start_event = MetagameReplayEvent.from_json(metagame_events[0])
        metagame_end_event = MetagameReplayEvent.from_json(metagame_events[1])

        facility_events = [FacilityControlReplayEvent.from_json(event) for event in facility_events_data]
        death_events = [DeathReplayEvent.from_json(event) for event in death_events_data]
        vehicle_destroy_events = [VehicleDestroyReplayEvent.from_json(event) for event in vehicle_destroy_events_data]
        logger.info("Built replay event objects")
        logger.info("Starting replay")

        replay(metagame_start_event, metagame_end_event, facility_events, death_events, vehicle_destroy_events)

if __name__ == "__main__":
    main()


import datetime
from kiteconnect import KiteConnect

kite = KiteConnect(api_key="")

def make_zerodha_login(api_key, access_token):
    global kite
    try:
        kite.api_key = api_key
        kite.set_access_token(access_token)
        return 1
    except Exception as e:
        return 0


from django.shortcuts import get_object_or_404, redirect, render

from .forms import CommentForm

from blog.models import Post

from .models import Comment

# Create your views here.

def post_comment(request, post_pk):
    # 这里先过去被评论的文章，因为后面需要把评论和文章关联起来
    # 这里我们使用了django内置的函数get_objects_or_404
    # 从而获取post类 也就是文章详情
    post = get_object_or_404(Post, pk=post_pk)

    # http协议中只有post和get方法，2而表单的数据传输都是用过 post 请求，
    # 所以这里我们选择当 请求 为post是才处理表单数据
    if request.method == 'POST':
        # 用户评论时提交的数据 存在 request.POST
        # 我们利用这个数据结构 CommentForm 的实例，这样生成了field字段的（name email texe url）
        form = CommentForm(request.POST)
        # 生成评论的实例
        comment = Comment()

        # 通过is_vaild()方法，django 会帮我们检测数据内容是否合法
        if form.is_valid():
            # 当数据合法时候，我们调表单的save方法 将其存入数据库
            # commit = False 的作用是 利用表单数据，成成 Comment 模型的实例
            # 意思是 目前不把form里的内容提交到数据库
            comment = form.save(commit=False)
            comment.parent = request.POST.get('comment_parent')
            # 将评论和文关联起来
            comment.post = post
            # 最后将commen模型的数据保存到数据库
            comment.save()

            # 保存之后 重定向网页 重新渲染
            return redirect(post)

        else:
            # 检查到数据不合法，重新渲染本页面，并渲染出表单的错误
            # 因此我们穿了三个变量给 detail 模板，
            # Post （文章） 平路列表 表单Form
            comment_list = post.comment_set.all()
            context = {
                'post': post,
                'form': form,
                'comment_list':comment_list,
            }
            return render(request, 'blog/detail.html', context=context)

    # 不是 psot 请求，说名用户没有提交数据，重定向到文章详情页
    return redirect(post)

def comment_reply(request, post_pk, comment_pk):
    '''
    回复评论
    comment_pk： 父级评论id
    post_pk: 文章post id
    '''
    post = get_object_or_404(Post, pk=post_pk)
    last_comment = Comment.objects.filter(pk=comment_pk)[0]
    form = CommentForm()
    context = {
        'post': post,
        'last_comment': last_comment,
        'form': form,
    }

    return render(request, 'comments/reply.html', context=context)


import pandas as pd
import numpy as np
import sys

data_path = sys.argv[1]
base_name = sys.argv[2]
export_path = sys.argv[3]
split_num = int(sys.argv[4])

input_df = pd.read_csv(f"{data_path}{base_name}.csv")

export_dataframes = np.array_split(input_df, split_num)

for idx, dataframe in enumerate(export_dataframes):
    dataframe.to_csv(f"{export_path}{base_name}{idx}.csv", index=False)


# %%
##########################
####### CHAPTER 11 #######
##########################
### Groupby Operations ###
##########################

# %%
# import modules
import pandas as pd
import numpy as np
import seaborn as sns

# %%
# load data
df = pd.read_csv('./data/gapminder.tsv', sep='\t')

# %%
# calculate the average life expectancy for each year
avg_life_exp_by_year = df.groupby('year').lifeExp.mean()
print(avg_life_exp_by_year)

# %%
# how groupby statements work?
# get a list of unique years in the data
years = df.year.unique()
print(years)

# %%
# subset the data for the year 1952
y1952 = df.loc[df.year == 1952, :]
print(y1952)

# %%
# take the mean of the lifeExp values
y1952_mean = y1952.lifeExp.mean()
print(y1952_mean)

# %%
# repeat this process for every year column
y1957 = df.loc[df.year == 1957, :]
y1957_mean = y1957.lifeExp.mean()

y1962 = df.loc[df.year == 1962, :]
y1962_mean = y1962.lifeExp.mean()

y2007 = df.loc[df.year == 2007, :]
y2007_mean = y2007.lifeExp.mean()

df2 = pd.DataFrame({"year": [1952, 1957, 1962, 2007],
                    "": [y1952_mean, y1957_mean, y1962_mean, y2007_mean]})

print(df2)

# %%
# Aggregation Function
# use custom function into agg
def my_mean(values):
    n = len(values)
    sum = 0
    for value in values:
        sum += value
    return sum / n

agg_my_mean = df.groupby('year')["lifeExp"].agg(my_mean)
print(agg_my_mean)

# %%
# functions that take multiple parameters
def my_mean_diff(values, diff_value):
    n = len(values)
    sum = 0
    for value in values:
        sum += value
    mean = sum / n
    return (mean - diff_value)

# calculate the global average life expectancy mean
global_mean = df["lifeExp"].mean()
print(global_mean)

# %%
# custom aggregation function with multiple parameters
agg_mean_diff = (
    df
    .groupby("year")
    ["lifeExp"]
    .agg(my_mean_diff, diff_value=global_mean)
)

print(agg_mean_diff)

# %%
# Multiple Functions simultaneously
# calculate the count, mean, std of the lifeExp by continent
gdf = (
    df
    .groupby("year")
    ["lifeExp"]
    .agg([np.count_nonzero, np.mean, np.std])
)

print(gdf)

# %%
# use a dictionary on a dataframe to agg different columns
# for each year, calculate the
# average lifeExp, median pop, and median gdpPercap
gdf_dict = df.groupby("year").agg(
    {
        "lifeExp": "mean",
        "pop": "median",
        "gdpPercap": "median"
    }
)

print(gdf_dict)

# %% 11-2
# Transform
# .transform() takes multiple values and returns a one-to-one transformation of the values
# that is, it does not reduce the amount of data
# Z-Score Example
def my_zscore(x):
    return (x-x.mean()) / x.std()

transform_z = df.groupby('year').lifeExp.transform(my_zscore)
print(transform_z.head())

# %%
# Missing Value Example
# randomly pick 4 'total_bill' values and turn them into missing
np.random.seed(42)
tips_10 = sns.load_dataset('tips').sample(10)
tips_10.loc[
    np.random.permutation(tips_10.index)[:4],
    'total_bill'
] = np.NaN

print(tips_10)

# %%
# perhaps the Male and Female values in the sex column have different spending habits
# count the non-missing values by sex
count_sex = tips_10.groupby('sex').count()
print(count_sex)

# %%
def fill_na_mean(x):
    """returns the average of a given vector"""
    avg = x.mean()
    return x.fillna(avg)

# calculate a mean 'total_bill' by 'sex'
total_bill_group_mean = (
    tips_10
    .groupby("sex")
    .total_bill
    .transform(fill_na_mean)
)

# assign to a new column in the original data
# you can also replace the original column by using 'total_bill'
tips_10["fill_total_bill"] = total_bill_group_mean

print(tips_10[['sex', 'total_bill', 'fill_total_bill']])

# %% 11-3
# Filter
# .filter() allows you to split your data by keys,
# and then perform some kind of boolean subsetting on the data.

# load the tips data set
tips = sns.load_dataset('tips')

# note the number of rows in the original data
print(tips.shape)

# look at the frequency counts for the table size
print(tips['size'].value_counts())

# %%
# filter the data such that each group has more than 30 observations
tips_filtered = (
    tips
    .groupby("size")
    .filter(lambda x: x["size"].count() >= 30)
)

# the output shows that our data set was filtered down
print(tips_filtered.shape)
print(tips_filtered['size'].value_counts())

# %% 11-4
# The Object: pandas.core.groupby, DataFrameGroupby
# we will investigate some of the inner workings of grouped objects.

# start with the subsetted tips data set
tips_10 = sns.load_dataset('tips').sample(10, random_state=42)
print(tips_10)

# %%
# save just the grouped object
grouped = tips_10.groupby('sex')
# note that we just get back the object and its memory location
print(grouped)

# %%
# nothing has been actually calculated yet,
# because we never performed an action that requires a calculation
# if we want to actually see the calculated groups, we can call the groups attribute

# see the actual grops of the groupby
# it returns only the index
print(grouped.groups)

# %%
# calculate the mean on relevant columns
avgs = grouped.mean(numeric_only=True)
print(avgs)

# not all the columns reported a mean
# list all the columns
print(tips_10.columns)

# %%
# Selecting a Group

# to extract a particular group, use the .get_group() method
# get the 'Female' group
female = grouped.get_group('Female')
print(female)

# %%
# iterate through our grouped values just like any other container in Python
for sex_group in grouped:
    print(sex_group)

# %%
# you can't really get the 0 element from the grouped object
# the object is still a pandas.core.groupby.DataFrameGroupBy object
print(grouped[0])

# %%
# DataFrameGroupBy consists of tuple (key[string], group[DataFrame])
for key, group in grouped:
    print('* key:', key)
    print('* group:', len(group))
    print(group)
    print('\n')

# %%
# Multiple Groups
# mean by sex and time
bill_sex_time = tips_10.groupby(['sex', 'time'])
group_avg = bill_sex_time.mean(numeric_only=True)
print(group_avg)

# %%
# interesting things happen when we look at the index
print(group_avg.index)

# %%
# to get a regualr flat dataframe back,
# call the .reset_index() method on the results
group_method = (
    tips_10
    .groupby(['sex', 'time'])
    .mean(numeric_only=True)
    .reset_index()
)
print(group_method)

# %%
# alternatively, use the as_index=False parameter in the .groupby() method
group_param = (
    tips_10
    .groupby(['sex', 'time'], as_index=False)
    .mean(numeric_only=True)
)
print(group_param)


import random

from aalpy.automata import Onfsm, Mdp, StochasticMealyMachine
from aalpy.base import Oracle, SUL

automaton_dict = {Onfsm: 'onfsm', Mdp: 'mdp', StochasticMealyMachine: 'smm'}

class RandomWalkEqOracle(Oracle):
    """
    Equivalence oracle where queries contain random inputs. After every step, 'reset_prob' determines the probability
    that the system will reset and a new query asked.
    """

    def __init__(self, alphabet: list, sul: SUL, num_steps=5000, reset_after_cex=True, reset_prob=0.09):
        """

        Args:
            alphabet: input alphabet

            sul: system under learning

            num_steps: number of steps to be preformed

            reset_after_cex: if true, num_steps will be preformed after every counter example, else the total number
                or steps will equal to num_steps

            reset_prob: probability that the new query will be asked
        """

        super().__init__(alphabet, sul)
        self.step_limit = num_steps
        self.reset_after_cex = reset_after_cex
        self.reset_prob = reset_prob
        self.random_steps_done = 0
        self.automata_type = None

    def find_cex(self, hypothesis):
        if not self.automata_type:
            self.automata_type = automaton_dict.get(type(hypothesis), 'det')

        inputs = []
        outputs = []
        self.reset_hyp_and_sul(hypothesis)

        while self.random_steps_done < self.step_limit:
            self.num_steps += 1
            self.random_steps_done += 1

            if random.random() <= self.reset_prob:
                self.reset_hyp_and_sul(hypothesis)
                inputs.clear()
                outputs.clear()

            inputs.append(random.choice(self.alphabet))

            out_sul = self.sul.step(inputs[-1])
            outputs.append(out_sul)

            if self.automata_type == 'det':
                out_hyp = hypothesis.step(inputs[-1])
            else:
                out_hyp = hypothesis.step_to(inputs[-1], out_sul)

            if self.automata_type == 'det' and out_sul != out_hyp:
                if self.reset_after_cex:
                    self.random_steps_done = 0

                self.sul.post()
                return inputs
            elif out_hyp is None:
                if self.reset_after_cex:
                    self.random_steps_done = 0
                self.sul.post()

                if self.automata_type == 'onfsm':
                    return inputs, outputs
                else:
                    # hypothesis is MDP or SMM
                    cex = [hypothesis.initial_state.output] if self.automata_type == 'mdp' else []
                    for i, o in zip(inputs, outputs):
                        cex.extend([i, o])
                    return cex

        return None

    def reset_counter(self):
        if self.reset_after_cex:
            self.random_steps_done = 0


import subprocess
import os
import io
from flask import Flask
from flask import request, jsonify

app = Flask(__name__)

MY_AUTH_TOKEN = os.getenv('CI_TOKEN', None)
PATH_TO_SCRIPT = os.getenv('PATH_TO_SCRIPT', None)

@app.route('/', methods=['POST'])
def MainHandler():
    if request.headers.get('Authorization') != MY_AUTH_TOKEN:
        return jsonify({'message': 'Bad token'}), 401
    else:
        logs = []
        errors = []
        logs.append('run \'' + PATH_TO_SCRIPT + '\'')

        shellscript = subprocess.Popen(["sh", PATH_TO_SCRIPT], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        stdout, stderr = shellscript.communicate()

        for line in stdout.decode("utf-8").split('\n'):
            logs.append(line)
        for line in stderr.decode("utf-8").split('\n'):
            errors.append(line)

        logs.append('done!')
        return jsonify({'message': 'Any work', 'logs': logs, 'errors': errors}), 200

def main():
    app.run(host='127.0.0.1', port=1114)

if __name__ == '__main__':
    main()


import os
from data.raw_data_loader_settings import *
import h5py
import logging
import numpy as np
import pandas as pd
import time

logger = logging.getLogger(__name__)

# TODO: process all files to standard format

class DataProcessor:

    def __init__(self, name, ReferenceDataLoader, RawDataLoader, org_structure=FileOrgStructure.DATECOLUMN):
        t = time.perf_counter()
        self.name = name
        self.outputpath = OutputDataPath
        self.ReferenceDataLoader = ReferenceDataLoader
        self.PVDataLoader = RawDataLoader
        self.org_structure = org_structure
        self.processed_halt_date = None
        self.processed_st_date = None
        self.processed_columns = set()

    def process_with_args(self, args):
        temp_names = args[0]
        temp_data_processors = args[1]
        temp_date = args[2]

        try:
            self.process(temp_data_processors, temp_date, temp_names)
            print("calculating for " + str(temp_date) + " " + str(temp_names))
        except:
            print("skipping for " + str(temp_date) + " " + str(temp_names))

    def process(self, pv_data_processors, date, names, pv_data="1min_PV"):

        if self.processed_halt_date is None:
            self.processed_halt_date = self.process_halt_date()
            self.processed_st_date = self.process_st_date()

        t = time.perf_counter()
        pv_data = self.PVDataLoader.load_pv_data(pv_data, date, date)
        logger.debug(f"load_pv_data finished in {time.perf_counter() - t} seconds")
        for pv_data_processor, name in zip(pv_data_processors, names):
            processed_daily_pv = self.process_1min_pv_to_daily(pv_data_processor, pv_data)
            logger.debug(f"process_1min_pv_to_daily finished in {time.perf_counter() - t} seconds")
            processed_daily_pv = processed_daily_pv.reset_index().rename(columns={"index": "code"})
            processed_daily_pv["code"] = processed_daily_pv["code"].astype(np.string_)
            self.write_data(date, processed_daily_pv, name, self.org_structure)
        logger.debug(f"wrote data in {time.perf_counter() - t} seconds")

    """ Helper Functions """

    def write_data(self, date, daily_df, name, org_structure):
        if org_structure is not FileOrgStructure.DATECOLUMN:
            raise Exception("Method not implemented")

        with h5py.File(self.outputpath + "\\" + name + ".hdf5", 'a') as f:
            dategroup = f.create_group(str(date))
            for column in daily_df.columns:
                try:
                    dategroup.create_dataset(column, data=daily_df[column].to_numpy())
                except:
                    dategroup.create_dataset(column, data=daily_df[column].to_numpy().astype(np.float64))
                    logger.info(f"forced conversion on {date} for {column}")
                self.processed_columns.add(column)
            f.close()

    @classmethod
    def write_performance_data(cls, name, performance_df, org_structure=FileOrgStructure.DATECOLUMN):
        if org_structure is not FileOrgStructure.DATECOLUMN:
            raise Exception("Method not implemented")

        with h5py.File(PerformanceOutputPath + "\\" + name + ".hdf5", 'w') as f:
            for column in performance_df.columns:
                try:
                    f.create_dataset(column, data=performance_df[column].to_numpy())
                except:
                    f.create_dataset(column, data=performance_df[column].to_numpy().astype(np.float64))
            f.close()

    @classmethod
    def write_alpha_data_all(cls, daily_df, name, org_structure=FileOrgStructure.DATECOLUMN):
        if org_structure is not FileOrgStructure.DATECOLUMN:
            raise Exception("Method not implemented")

        all_dates = list(daily_df["date"].drop_duplicates())
        directory = AlphaOutputPath + "\\" + name
        if not os.path.exists(directory):
            os.mkdir(directory)
        for date in all_dates:
            data = daily_df[daily_df["date"] == date]
            with h5py.File(directory + "\\" + str(date) + ".hdf5", 'w') as f:
                for column in daily_df.columns:
                    try:
                        f.create_dataset(column, data=data[column].to_numpy())
                    except:
                        f.create_dataset(column, data=data[column].to_numpy().astype(np.float64))
                f.close()

    @classmethod
    def write_alpha_data(cls, date, daily_df, name, org_structure=FileOrgStructure.DATECOLUMN):
        if org_structure is not FileOrgStructure.DATECOLUMN:
            raise Exception("Method not implemented")

        directory = AlphaOutputPath + "\\" + name
        if not os.path.exists(directory):
            os.mkdir(directory)
        with h5py.File(directory + "\\" + date + ".hdf5", 'w') as f:
            for column in daily_df.columns:
                try:
                    f.create_dataset(column, data=daily_df[column].to_numpy())
                except:
                    f.create_dataset(column, data=daily_df[column].to_numpy().astype(np.float64))
            f.close()

    def process_halt_date(self):
        halt_date = self.ReferenceDataLoader.load_reference_data_by_name("HaltDate").copy()
        halt_date["restart_inclusive"] = halt_date.apply(lambda x: x["restart_time"] > 93000, axis=1)
        trade_date = self.ReferenceDataLoader.load_reference_data_by_name("Calendar").copy()
        trade_date = trade_date[trade_date["is_open"] == 1]

        output = []
        for row in halt_date.iterrows():
            code = row[1]["code"]
            halt_start = row[1]["halt_date"]
            restart_date = row[1]["restart_date"]
            if row[1]["restart_inclusive"]:
                tmp = trade_date[(trade_date["date"] >= halt_start) & (trade_date["date"] <= restart_date)].assign(
                    code=code).assign(is_halt=1)
            else:
                tmp = trade_date[(trade_date["date"] >= halt_start) & (trade_date["date"] < restart_date)].assign(
                    code=code).assign(is_halt=1)
            output.append(tmp[["code", "date", "is_halt"]])
        return pd.concat(output)

    def process_st_date(self):

        trade_date = self.ReferenceDataLoader.load_reference_data_by_name("Calendar").copy()
        trade_date_open = trade_date[trade_date["is_open"] == 1][["date", "is_open"]].set_index("date")
        trade_date_all = trade_date[["date", "is_open"]].set_index("date")

        st_status = self.ReferenceDataLoader.load_reference_data_by_name("STDate").copy()[
            ["code", "status_id", "eff_date"]]
        st_status = st_status.pivot_table(index="eff_date", columns="code", values="status_id")
        st_status = trade_date_open.join(st_status).ffill().drop(columns="is_open")
        st_status = trade_date_all.join(st_status).drop(columns="is_open")
        st_status = st_status.stack().reset_index()
        st_status.columns = ["date", "code", "status_id"]
        st_status["date"] = st_status["date"].astype("int32")
        st_status["status_id"] = st_status["status_id"].astype("int32")
        return st_status

    def process_1min_pv_to_daily(self, pv_data_processor, pv_data):

        processed_1min_pv = pv_data_processor(pv_data)
        current_base = self.ReferenceDataLoader.load_reference_data_by_name("ListDelistDate").copy().set_index("code")
        joined_df = current_base.join(processed_1min_pv)
        return joined_df[processed_1min_pv.columns]

    def add_reference_data(self, processed_daily_pv, date):

        cumulative_adjustment_factor = \
            self.ReferenceDataLoader.load_reference_data_by_name("CumulativeAdjustmentFactor")[
                self.ReferenceDataLoader.load_reference_data_by_name("CumulativeAdjustmentFactor")["date_int"] == int(
                    date)].set_index("code")[
                ["cum_adjf"]]
        cumulative_adjustment_factor["cum_adjf"] = cumulative_adjustment_factor["cum_adjf"].astype(np.float64)

        halt_date = self.processed_halt_date.copy()
        halt_date = halt_date[halt_date["date"] == int(date)].set_index("code")[["is_halt"]]
        halt_date["is_halt"] = halt_date["is_halt"].astype(np.float64)

        current_status = self.ReferenceDataLoader.load_reference_data_by_name("ListDelistDate").copy()[
            ["code", "lst_status", "lst_date", "delst_date"]].set_index("code")
        current_status["lst_status"] = current_status["lst_status"].astype(np.string_)
        current_status["lst_date"] = current_status["lst_date"].astype(np.float64)
        current_status["delst_date"] = current_status["delst_date"].astype(np.float64)

        st_date = self.processed_st_date.copy()
        st_date = st_date[st_date["date"] == int(date)].set_index("code")[["status_id"]]
        st_date["status_id"] = st_date["status_id"].astype(np.float64)

        sector_classification = self.ReferenceDataLoader.load_reference_data_by_name("Sector")[
            self.ReferenceDataLoader.load_reference_data_by_name("Sector")["date_int"] == int(date)].set_index("code")[
            ["sw1",
             "sw2",
             "sw3"]]

        limit_price = self.ReferenceDataLoader.load_reference_data_by_name("LimitPrices")[
            self.ReferenceDataLoader.load_reference_data_by_name("LimitPrices")["date_int"] == int(date)].set_index(
            "code")[["up_limit",
                     "down_limit"]]
        limit_price["up_limit"] = limit_price["up_limit"].astype(np.float64)
        limit_price["down_limit"] = limit_price["down_limit"].astype(np.float64)

        market_value = self.ReferenceDataLoader.load_reference_data_by_name("MarketValue")[
            self.ReferenceDataLoader.load_reference_data_by_name("MarketValue")["date_int"] == int(date)].set_index(
            "code")[["neg_mkt_val",
                     "mkt_val",
                     "neg_shares",
                     "shares"]]
        market_value["neg_mkt_val"] = market_value["neg_mkt_val"].astype(np.float64)
        market_value["mkt_val"] = market_value["mkt_val"].astype(np.float64)
        market_value["neg_shares"] = market_value["neg_shares"].astype(np.float64)
        market_value["shares"] = market_value["shares"].astype(np.float64)

        processed_daily_pv = processed_daily_pv.join(cumulative_adjustment_factor)
        processed_daily_pv = processed_daily_pv.join(halt_date)
        processed_daily_pv = processed_daily_pv.join(current_status)
        processed_daily_pv = processed_daily_pv.join(st_date)

        processed_daily_pv = processed_daily_pv.join(sector_classification)
        processed_daily_pv[["sw1", "sw2", "sw3"]] = processed_daily_pv[["sw1", "sw2", "sw3"]].fillna(value="")
        processed_daily_pv["sw1"] = processed_daily_pv["sw1"].str.encode('utf-8').astype(np.string_)
        processed_daily_pv["sw2"] = processed_daily_pv["sw2"].str.encode('utf-8').astype(np.string_)
        processed_daily_pv["sw3"] = processed_daily_pv["sw3"].str.encode('utf-8').astype(np.string_)

        processed_daily_pv = processed_daily_pv.join(limit_price)
        processed_daily_pv = processed_daily_pv.join(market_value)
        processed_daily_pv = processed_daily_pv.reset_index()
        processed_daily_pv["code"] = processed_daily_pv["code"].astype(np.string_)

        return processed_daily_pv


import jsonschema
import unittest
import pytest
from pathlib import Path
import os
import json

# Load all JSON files in RAM
path_function_dictionaries = os.path.join(
    os.path.dirname(__file__), "..", "sql_translate", "function_dictionaries"
)
path_jsonschema = os.path.join(
    os.path.dirname(__file__), "samples", "jsonschema", "function_dictionaries.json"
)
files_to_review = {}
for path_file in Path(os.path.join(path_function_dictionaries)).rglob("*.json"):
    print(f"Loading {path_file} in memory")
    with open(path_file) as f:
        files_to_review[path_file] = json.load(f)
assert files_to_review  # If empty list, file must have moved

def test_dictionaries_against_jsonschema() -> None:
    with open(path_jsonschema) as f:
        schema = json.load(f)
    for file_to_review, content in files_to_review.items():
        print(f"Validating {file_to_review} against jsonschema")
        assert jsonschema.validate(content, schema) is None

def test_dictionaries_are_sorted() -> None:
    for file_to_review, content in files_to_review.items():
        print(f"Validating {file_to_review} has sorted keys")
        assert list(content.keys()) == sorted(content.keys())