import pandas as pd
import numpy as np
import datetime
import matplotlib.pyplot as plt
from pandas.plotting import radviz
from utils import code_to_word

"""
    构建一个具有1个隐藏层的神经网络，隐层的大小为10
    输入层为4个特征，输出层为3个分类
    (1,0,0)为第一类，(0,1,0)为第二类，(0,0,1)为第三类
"""

# 1.初始化参数
def initialize_parameters(n_x, n_h, n_y):
    np.random.seed(2)

    # 权重和偏置矩阵
    w1 = np.random.randn(n_h, n_x) * 0.01
    b1 = np.zeros(shape=(n_h, 1))
    w2 = np.random.randn(n_y, n_h) * 0.01
    b2 = np.zeros(shape=(n_y, 1))

    # 通过字典存储参数
    parameters = {"w1": w1, "b1": b1, "w2": w2, "b2": b2}

    return parameters

# 2.前向传播
def forward_propagation(X, parameters):
    w1 = parameters["w1"]
    b1 = parameters["b1"]
    w2 = parameters["w2"]
    b2 = parameters["b2"]

    # 通过前向传播来计算a2
    z1 = np.dot(w1, X) + b1  # 这个地方需注意矩阵加法：虽然(w1*X)和b1的维度不同，但可以相加
    a1 = np.tanh(z1)  # 使用tanh作为第一层的激活函数
    z2 = np.dot(w2, a1) + b2
    a2 = 1 / (1 + np.exp(-z2))  # 使用sigmoid作为第二层的激活函数

    # 通过字典存储参数
    cache = {"z1": z1, "a1": a1, "z2": z2, "a2": a2}

    return a2, cache

# 3.计算代价函数
def compute_cost(a2, Y, parameters):
    m = Y.shape[1]  # Y的列数即为总的样本数

    # 采用交叉熵（cross-entropy）作为代价函数
    logprobs = np.multiply(np.log(a2), Y) + np.multiply((1 - Y), np.log(1 - a2))
    cost = -np.sum(logprobs) / m

    return cost

# 4.反向传播（计算代价函数的导数）
def backward_propagation(parameters, cache, X, Y):
    m = Y.shape[1]

    w2 = parameters["w2"]

    a1 = cache["a1"]
    a2 = cache["a2"]

    # 反向传播，计算dw1、db1、dw2、db2
    dz2 = a2 - Y
    dw2 = (1 / m) * np.dot(dz2, a1.T)
    db2 = (1 / m) * np.sum(dz2, axis=1, keepdims=True)
    dz1 = np.multiply(np.dot(w2.T, dz2), 1 - np.power(a1, 2))
    dw1 = (1 / m) * np.dot(dz1, X.T)
    db1 = (1 / m) * np.sum(dz1, axis=1, keepdims=True)

    grads = {"dw1": dw1, "db1": db1, "dw2": dw2, "db2": db2}

    return grads

# 5.更新参数
def update_parameters(parameters, grads, learning_rate=0.1):
    w1 = parameters["w1"]
    b1 = parameters["b1"]
    w2 = parameters["w2"]
    b2 = parameters["b2"]

    dw1 = grads["dw1"]
    db1 = grads["db1"]
    dw2 = grads["dw2"]
    db2 = grads["db2"]

    # 更新参数
    w1 = w1 - dw1 * learning_rate
    b1 = b1 - db1 * learning_rate
    w2 = w2 - dw2 * learning_rate
    b2 = b2 - db2 * learning_rate

    parameters = {"w1": w1, "b1": b1, "w2": w2, "b2": b2}

    return parameters

# 6.模型评估
def predict(parameters, x_test, y_test):
    w1 = parameters["w1"]
    b1 = parameters["b1"]
    w2 = parameters["w2"]
    b2 = parameters["b2"]

    z1 = np.dot(w1, x_test) + b1
    a1 = np.tanh(z1)
    z2 = np.dot(w2, a1) + b2
    a2 = 1 / (1 + np.exp(-z2))

    # 结果的维度
    n_rows = y_test.shape[0]
    n_cols = y_test.shape[1]

    # 预测值结果存储
    output = np.empty(shape=(n_rows, n_cols), dtype=int)
    print(a2)

    for i in range(n_rows):
        for j in range(n_cols):
            if a2[i][j] > 0.5:
                output[i][j] = 1
            else:
                output[i][j] = 0

    print("预测结果：")
    print(code_to_word(output.T))
    print("真实结果：")
    print(code_to_word(y_test.T))

    print("准确率：", accuracy_calculate(output.T, y_test.T))
    return output.T

def accuracy_calculate(predict, real):
    """计算结果准确率"""
    # 结果的维度
    n_rows = predict.shape[0]
    n_cols = predict.shape[1]
    accuracy = 0.00
    for i in range(n_rows):
        count = 0
        for j in range(n_cols):
            if predict[i][j] == real[i][j]:
                count += 1
        print(round(count / n_cols, 2))
        accuracy += round(count / n_cols, 2)
    return round(accuracy / n_rows, 2)

# 建立神经网络并进行训练
def nn_model(X, Y, n_h, n_input, n_output, num_iterations=20000, print_cost=False):
    np.random.seed(3)

    n_x = n_input  # 输入层节点数
    n_y = n_output  # 输出层节点数

    # 1.初始化参数
    parameters = initialize_parameters(n_x, n_h, n_y)
    learning_rate = 0.1
    # 梯度下降循环
    for i in range(0, num_iterations):
        # 2.前向传播
        a2, cache = forward_propagation(X, parameters)
        # 3.计算代价函数
        cost = compute_cost(a2, Y, parameters)
        # 4.反向传播
        grads = backward_propagation(parameters, cache, X, Y)
        # 5.更新参数
        parameters = update_parameters(parameters, grads, learning_rate)

        # 每10000次迭代，输出一次代价函数
        if print_cost and i % 10000 == 0:
            print("迭代第%i次，代价函数为：%f" % (i, cost), "学习率：" + str(learning_rate))
            # 学习率梯度下降
            learning_rate = 0.9 * learning_rate

    return parameters

if __name__ == "__main__":
    # 读取数据
    data_set = pd.read_csv("samples/gift_training.csv", header=None)

    # 第1种取数据方法：
    X = data_set.iloc[:, 0:13].values.T  # 前十三列是特征，T表示转置
    Y = data_set.iloc[:, 13:].values.T  # 后十六列是标签

    Y = Y.astype("uint8")

    # 开始训练
    start_time = datetime.datetime.now()
    # 输入4个节点，隐层10个节点，输出3个节点，迭代200000次
    parameters = nn_model(
        X, Y, n_h=20, n_input=13, n_output=16, num_iterations=200000, print_cost=True
    )
    end_time = datetime.datetime.now()
    print(
        "用时："
        + str((end_time - start_time).seconds)
        + "s"
        + str(round((end_time - start_time).microseconds / 1000))
        + "ms"
    )

    # 对模型进行测试
    data_test = pd.read_csv("samples/gift_test.csv", header=None)
    x_test = data_test.iloc[:, 0:13].values.T
    y_test = data_test.iloc[:, 13:].values.T
    y_test = y_test.astype("uint8")

    result = predict(parameters, x_test, y_test)


"""
╔══════════════════════════════════════════════════════════╗
║                        tasks.py                          ║
╚══════════════════════════════════════════════════════════╝
┌──────────────────────────────────────────────────────────┐
│                        Author                            │
├──────┬────────────────────┬───────┬──────────────────────┤
│ Name │ A S M Saad         │ Email │ asmsaad3@gmail.com   │
├──────┼────────────────────┼───────┼──────────────────────┤
│ Date │ June 6, 2023       │ Github│ asmsaad/mintrower    │
├──────┴────────────────────┴───────┴──────────────────────┤
│                       Description                        │
├──────────────────────────────────────────────────────────┤
│                                                          │
│                                                          │
└──────────────────────────────────────────────────────────┘
"""
from tkinter import *
from tkinter import ttk
from colors import *
from icons import *
from tksupport import *
# from addnewtask import *
from tabs.tktabsupport import *
from tabs.actions import *

class TaskTab:
    def __init__(self, base_canvas):
        self.base_canvas = base_canvas

        self.column_data_details = {
            "Selector": {"width": 50, "text_align": CENTER, "anchor": "center"},
            "ID": {"width": 100, "text_align": LEFT, "anchor": "center"},
            "Website": {"width": 240 - 10, "text_align": LEFT, "anchor": W},
            "Size": {"width": 120, "text_align": LEFT, "anchor": W},
            "Keyword": {"width": 180, "text_align": LEFT, "anchor": W},
            "Proxy": {"width": 150, "text_align": CENTER, "anchor": W},
            "Billing Profile": {"width": 150, "text_align": LEFT, "anchor": W},
            "Status": {"width": 130, "text_align": CENTER, "anchor": W},
            "Actions": {"width": 178 + 30 + 10 - 50, "text_align": LEFT, "anchor": W},
        }

        self.tab_property = TabProperty(self.base_canvas)
        self.tab_property.set_individual_data_control(controls=("run", "edit", "delete"), tab_name="task")
        self.header_frame ,self.data_show_frame , self.total_control_frame = self.tab_property.create_frames(header_height=70,middle_height=540,bottom_height=58)
        #Make Heading
        self.tab_property.tree_view_heading(self.header_frame,self.column_data_details)
        #Total Control Area
        self.total_control_panel(self.total_control_frame)

        ''''
        In this section,  retrieve  the data from the  database that
        was previously imported. The initial  imported  data will be
        displayed at the top  of  the frame. Ensure that the data is
        provided,  as  the  display_data  dictionary   keys  remains
        unchanged.
        '''
        for i in range(25):
            display_data = {
                "ID": str(i+1),
                "Website": "",
                "Size": "",
                "Keyword": "",
                "Proxy": "",
                "Billing Profile": "",
                "Status": "",
            }
            self.tab_property.individual_data(self.data_show_frame,display_data)

    def total_control_panel(self,frame):
        # Total Control button
        self.control_btns_details = {
            "add_new": {"dimension": (138 + 10, 32+10)},
            "delete_all": {"dimension": (129 + 10, 32 + 10)},
            "run_all": {"dimension": (114+10, 32+10)},
            "stop_all": {"dimension": (114+10, 32+10)},
        }

        self.left_control_frmae = Frame(frame, bg=Colors__.color()["working space"]["bg"], border=0, borderwidth=0, highlightthickness=0)
        self.left_control_frmae.pack(side=LEFT, anchor=W)

        self.right_control_frmae = Frame(frame, bg=Colors__.color()["working space"]["bg"], border=0, borderwidth=0, highlightthickness=0)
        self.right_control_frmae.pack(side=RIGHT, anchor=E)

        self.total_control_btns = {}
        for each_control_btn in list(self.control_btns_details.keys())[:2]:
            self.total_control_btns[each_control_btn] = {}
            self.total_control_btns[each_control_btn]["btn_obj"] = TkWidget()
            self.total_control_btns[each_control_btn]["btn"] = self.total_control_btns[each_control_btn]["btn_obj"].image_btn(self.left_control_frmae, imgTk=image__.icons(each_control_btn.lower()), imgTk_hover=image__.icons(each_control_btn.lower()+"_hover"), dimension=self.control_btns_details[each_control_btn]["dimension"], bg=Colors__.color()["working space"]["bg"], activebackground=Colors__.color()["working space"]["bg"])
            self.total_control_btns[each_control_btn]["btn"].pack(side=LEFT, anchor=W)
        # Add New Task Button Call
        self.total_control_btns["add_new"]["btn"]["command"] = lambda root_=self.base_canvas.winfo_toplevel(), task_tab_frame=self.data_show_frame, details=self.tab_property: task_tab_action_add_new_task(root_,task_tab_frame,details)

        # Left Control Button's Frame
        for each_control_btn in list(self.control_btns_details.keys())[2:]:
            self.total_control_btns[each_control_btn] = {}
            self.total_control_btns[each_control_btn]["btn_obj"] = TkWidget()
            self.total_control_btns[each_control_btn]["btn"] = self.total_control_btns[each_control_btn]["btn_obj"].image_btn(self.right_control_frmae, imgTk=image__.icons(each_control_btn.lower()), imgTk_hover=image__.icons(each_control_btn.lower()+"_hover"), dimension=self.control_btns_details[each_control_btn]["dimension"], bg=Colors__.color()["working space"]["bg"], activebackground=Colors__.color()["working space"]["bg"])
            self.total_control_btns[each_control_btn]["btn"].pack(side=LEFT, anchor=W)


# In this script, we will create some basic nlp features.
import time
notebookstart= time.time()

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import os
import gc
print("Data:\n",os.listdir("../input"))

# Models Packages
from sklearn import metrics
from sklearn.metrics import mean_squared_error
from sklearn import feature_selection
from sklearn.model_selection import train_test_split
from sklearn import preprocessing
from sklearn.decomposition import NMF

# Gradient Boosting
import lightgbm as lgb
from sklearn.linear_model import Ridge
from sklearn.cross_validation import KFold

# Tf-Idf
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.pipeline import FeatureUnion
from scipy.sparse import hstack, csr_matrix
from nltk.corpus import stopwords
russian_stop = set(stopwords.words('russian'))
stop_2 = set([w for w in open("../tmp/russian_stopwords.txt", "r").readlines()])
russian_stop = russian_stop.union(stop_2)
import string
punctuations = string.punctuation

# Viz
import re
import string

# I/O
from utils import *

# others
from joblib import Parallel, delayed
from scipy.stats import skew, kurtosis, entropy
from scipy import sparse
from scipy.sparse.linalg import svds
import umap
import gensim
from gensim.models.doc2vec import LabeledSentence
from gensim import corpora, models

NFOLDS = 5
SEED = 42

print("\nData Load Stage")
training = pd.read_csv('../input/train_stemmed.csv', index_col = "item_id")
traindex = training.index
testing = pd.read_csv('../input/test_stemmed.csv', index_col = "item_id")
testdex = testing.index

ntrain = training.shape[0]
ntest = testing.shape[0]

y = pd.read_csv('../input/train.csv', usecols = ["deal_probability"]).deal_probability.values

print('Train shape: {} Rows, {} Columns'.format(*training.shape))
print('Test shape: {} Rows, {} Columns'.format(*testing.shape))

print("Combine Train and Test")
df = pd.concat([training,testing],axis=0)
df["text_all"] = df.description.fillna("") + " "  + df.title.fillna("")
del training, testing
gc.collect()
print('\nAll Data shape: {} Rows, {} Columns'.format(*df.shape))

# Define Functions
def count_upper(x): return len([w for w in x if w.isupper()])
def count_lower(x): return len([w for w in x if w.islower()])
def count_stops(x): return len([w for w in x if w in russian_stop])
def count_punctuations(x): return len([w for w in x if w in punctuations])
def count_numbers(x): return len([w for w in x if w.isnumeric()])
def count_words(x): return len(x.split())
def count_unique_words(x): return len(set(x.split()))
def get_std(x): return np.std(x.todense())
def get_skew(x): return skew(x.todense().T)[0]
def get_kur(x): return kurtosis(x.todense().T)[0]
def get_entropy(x): return entropy(x.todense().T)[0]

# textcols = ["description", "title", "text_all"]#"description"
# for col in textcols:
#     print("Creating features {}...".format(col))
#     df_target = df[[col]].fillna("")
#
#     print("Creating basic tfidf features...")
#     df_target = df[col].fillna("")
#     tfidf_para = {
#         "stop_words": russian_stop,
#         "analyzer": 'word',
#         "token_pattern": r'\w{1,}',
#         "sublinear_tf": True,
#         "dtype": np.float32,
#         "norm": 'l2',
#         "min_df":3,
#         "smooth_idf":True
#     }
#
#     vectorizer = TfidfVectorizer(ngram_range=(1, 2), max_features=17000, **tfidf_para)
#     vecs = vectorizer.fit_transform(df_target.values)

    # df_out = pd.DataFrame()
    # df_out[col + "_stemmed_tfidf_mean"] = Parallel(n_jobs=-1)([delayed(np.mean)(v) for v in vecs])
    # print("mean done")
    # df_out[col + "_stemmed_tfidf_max"] = Parallel(n_jobs=-1)([delayed(np.max)(v) for v in vecs])
    # print("max done")
    # df_out[col + "_stemmed_tfidf_min"] = Parallel(n_jobs=-1)([delayed(np.min)(v) for v in vecs])
    # print("min done")
    # df_out[col + "_stemmed_tfidf_std"] = Parallel(n_jobs=-1)([delayed(get_std)(v) for v in vecs])
    # print("std done")
    # df_out[col + "_stemmed_tfidf_skew"] = Parallel(n_jobs=-1)([delayed(get_skew)(v) for v in vecs])
    # print("skew done")
    # df_out[col + "_stemmed_tfidf_kur"] = Parallel(n_jobs=-1)([delayed(get_kur)(v) for v in vecs])
    # print("kurtoisis done")
    # df_out[col + "_stemmed_tfidf_entropy"] = Parallel(n_jobs=-1)([delayed(get_entropy)(v) for v in vecs])
    # print("entropy done")
    # df_out[col + "_stemmed_tfidf_sum"] = Parallel(n_jobs=-1)([delayed(np.sum)(v) for v in vecs])
    # print("sum done")
    # to_parquet(df_out.iloc[:ntrain, :], "../features/fe_tfidf_stemmed_basic_{}_train.parquet".format(col))
    # to_parquet(df_out.iloc[ntrain:, :], "../features/fe_tfidf_stemmed_basic_{}_test.parquet".format(col))
    # del df_out; gc.collect()
    #
    # oof_sgd(vecs[:ntrain,:],vecs[ntrain:,:],y,"tfidf_stemmed_{}".format(col))
    # oof_lgbm(vecs[:ntrain,:].astype(np.float32),vecs[ntrain:,:].astype(np.float32),y,"tfidf_stemmed_{}".format(col))
    # del vecs; gc.collect()

# for col in textcols:
#     df_target = df[col].fillna("")
#     tfidf_para = {
#         "stop_words": russian_stop,
#         "analyzer": 'word',
#         "token_pattern": r'\w{1,}',
#         "sublinear_tf": True,
#         "dtype": np.float32,
#         "norm": 'l2',
#         "min_df":3,
#         "smooth_idf":False
#     }
#     vectorizer = TfidfVectorizer(ngram_range=(1, 2), max_features=50000, **tfidf_para)
#
#     vecs = vectorizer.fit_transform(df_target.values)
#     # tfidf dimensionality reduction
#     print("Start dimensionality reduction")
#     ## SVD
#     U, S , _ = svds(vecs.tocsc(),k=3)
#     m_svd = [U[i] * S for i in range(U.shape[0])]
#     m_svd = np.array(m_svd)
#     train_svd = pd.DataFrame(m_svd[:ntrain, :], columns=["svd_stemmed_{}_1".format(col+"_tfidf"), "svd_stemmed_{}_2".format(col+"_tfidf"), "svd_stemmed_{}_3".format(col+"_tfidf")])
#     test_svd = pd.DataFrame(m_svd[ntrain:, :], columns=["svd_stemmed_{}_1".format(col+"_tfidf"), "svd_stemmed_{}_2".format(col+"_tfidf"), "svd_stemmed_{}_3".format(col+"_tfidf")])
#     to_parquet(train_svd, "../features/fe_tfidf_stemmed_svd_{}_train.parquet".format(col))
#     to_parquet(test_svd, "../features/fe_tfidf_stemmed_svd_{}_test.parquet".format(col))
#     del m_svd, train_svd, test_svd; gc.collect()
#
#     ## NMF
#     nmf = NMF(n_components=3)
#     X_nmf = nmf.fit_transform(vecs)
#     df_nmf = pd.DataFrame(X_nmf, columns=["nmf_stemmed_{}_1".format(col+"_tfidf"), "nmf_stemmed_{}_2".format(col+"_tfidf"), "nmf_stemmed_{}_3".format(col+"_tfidf")])
#     nmf_train = df_nmf.iloc[:ntrain,:]
#     nmf_test = df_nmf.iloc[ntrain:,:]
#     to_parquet(nmf_train, "../features/fe_tfidf_stemmed_nmf_{}_train.parquet".format(col))
#     to_parquet(nmf_test, "../features/fe_tfidf_stemmed_nmf_{}_test.parquet".format(col))
#     del df_nmf, nmf_train, nmf_test; gc.collect()

# Meta Text Features

kfolds = get_kfolds()

print("Creating Ridge Features...")
class SklearnWrapper(object):
    def __init__(self, clf, seed=0, params=None, seed_bool = True):
        if(seed_bool == True):
            params['random_state'] = seed
        self.clf = clf(**params)

    def train(self, x_train, y_train):
        self.clf.fit(x_train, y_train)

    def predict(self, x):
        return self.clf.predict(x)

def get_oof(clf, x_train, y, x_test):
    oof_train = np.zeros((ntrain,))
    oof_test = np.zeros((ntest,))
    oof_test_skf = np.empty((NFOLDS, ntest))

    for i, (train_index, test_index) in enumerate(kfolds):
        print('\nFold {}'.format(i))
        x_tr = x_train[train_index]
        y_tr = y[train_index]
        x_te = x_train[test_index]

        clf.train(x_tr, y_tr)

        oof_train[test_index] = clf.predict(x_te)
        oof_test_skf[i, :] = clf.predict(x_test)

    oof_test[:] = oof_test_skf.mean(axis=0)
    return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1)

def cleanName(text):
    try:
        textProc = text.lower()
        textProc = " ".join(map(str.strip, re.split('(\d+)',textProc)))
        regex = re.compile(u'[^[:alpha:]]')
        textProc = regex.sub(" ", textProc)
        textProc = " ".join(textProc.split())
        return textProc
    except:
        return "name error"

df['title'] = df['title'].apply(lambda x: cleanName(x)).fillna("")
df["description"]   = df["description"].apply(lambda x: cleanName(x)).fillna("")

# textfeats = ["description", "title"]
# tfidf_para = {
#     "stop_words": russian_stop,
#     "analyzer": 'word',
#     "token_pattern": r'\w{1,}',
#     "sublinear_tf": True,
#     "dtype": np.float32,
#     "norm": 'l2',
#     "min_df":3,
#     "smooth_idf":False
# }
#
# def get_col(col_name): return lambda x: x[col_name]
#
# vectorizer = FeatureUnion([
#         ('description',TfidfVectorizer(
#             ngram_range=(1, 2),
#             max_features=17000,
#             **tfidf_para,
#             preprocessor=get_col('description'))),
#         ('title',CountVectorizer(
#             ngram_range=(1, 2),
#             stop_words = russian_stop,
#             preprocessor=get_col('title')))
#     ])
#
# #Fit my vectorizer on the entire dataset instead of the training rows
# #Score improved by .0001
# vectorizer.fit(df.to_dict('records'))
#
# ready_df = vectorizer.transform(df.to_dict('records'))
# tfvocab = vectorizer.get_feature_names()
#
# # Drop Text Cols
# textfeats = ["description", "title"]
#
#
# from sklearn.metrics import mean_squared_error
# from math import sqrt
#
# ridge_params = {'alpha':20.0, 'fit_intercept':True, 'normalize':False, 'copy_X':True,
#                 'max_iter':None, 'tol':0.001, 'solver':'auto', 'random_state':SEED}
#
# ridge = SklearnWrapper(clf=Ridge, seed = SEED, params = ridge_params)
# ridge_oof_train, ridge_oof_test = get_oof(ridge, ready_df[:ntrain], y, ready_df[ntrain:])
# train_out = pd.DataFrame(ridge_oof_train,columns=["ridge_oof_base_stemmed_2"])
# test_out = pd.DataFrame(ridge_oof_test,columns=["ridge_oof_base_stemmed_2"])
#
# to_parquet(train_out, "../features/oof_ridge_tfidf_stemmed_2_train.parquet")
# to_parquet(test_out, "../features/oof_ridge_tfidf_stemmed_2_test.parquet")
#

tfidf_para = {
    "stop_words": russian_stop,
    "analyzer": 'word',
    "token_pattern": r'\w{1,}',
    "sublinear_tf": True,
    "dtype": np.float32,
    "norm": 'l2',
    "min_df":3,
    "smooth_idf":False
}

def get_col(col_name): return lambda x: x[col_name]

vectorizer = FeatureUnion([
        ('description',TfidfVectorizer(
            ngram_range=(1, 3),
            max_features=20000,
            **tfidf_para,
            preprocessor=get_col('description'))),
        ('title',CountVectorizer(
            ngram_range=(1, 3),
            stop_words = russian_stop,
            preprocessor=get_col('title')))
    ])

#Fit my vectorizer on the entire dataset instead of the training rows
#Score improved by .0001
vectorizer.fit(df.to_dict('records'))

ready_df = vectorizer.transform(df.to_dict('records'))
tfvocab = vectorizer.get_feature_names()

# Drop Text Cols
textfeats = ["description", "title"]
df.drop(textfeats, axis=1,inplace=True)

from sklearn.metrics import mean_squared_error
from math import sqrt

ridge_params = {'alpha':20.0, 'fit_intercept':True, 'normalize':False, 'copy_X':True,
                'max_iter':None, 'tol':0.001, 'solver':'auto', 'random_state':SEED}

ridge = SklearnWrapper(clf=Ridge, seed = SEED, params = ridge_params)
ridge_oof_train, ridge_oof_test = get_oof(ridge, ready_df[:ntrain], y, ready_df[ntrain:])
train_out = pd.DataFrame(ridge_oof_train,columns=["ridge_oof_base_stemmed_3"])
test_out = pd.DataFrame(ridge_oof_test,columns=["ridge_oof_base_stemmed_3"])

to_parquet(train_out, "../features/oof_ridge_tfidf_stemmed_3_train.parquet")
to_parquet(test_out, "../features/oof_ridge_tfidf_stemmed_3_test.parquet")


# Move a robot along a line given a start and end point by steps
# This macro shows three different ways of programming a robot using a Python script and the RoboDK API

# More information about the RoboDK API here:
# https://robodk.com/doc/en/RoboDK-API.html
# For more information visit:
# https://robodk.com/doc/en/PythonAPI/robodk.html#robolink-py

# Default parameters:
P_START = [1755, -500, 2155]  # Start point with respect to the robot base frame
P_END = [1755, 600, 2155]  # End point with respect to the robot base frame
NUM_POINTS = 10  # Number of points to interpolate

# Function definition to create a list of points (line)
def MakePoints(xStart, xEnd, numPoints):
    """Generates a list of points"""
    if len(xStart) != 3 or len(xEnd) != 3:
        raise Exception("Start and end point must be 3-dimensional vectors")
    if numPoints < 2:
        raise Exception("At least two points are required")

    # Starting Points
    pt_list = []
    x = xStart[0]
    y = xStart[1]
    z = xStart[2]

    # How much we add/subtract between each interpolated point
    x_steps = (xEnd[0] - xStart[0]) / (numPoints - 1)
    y_steps = (xEnd[1] - xStart[1]) / (numPoints - 1)
    z_steps = (xEnd[2] - xStart[2]) / (numPoints - 1)

    # Incrementally add to each point until the end point is reached
    for i in range(numPoints):
        point_i = [x, y, z]  # create a point
        #append the point to the list
        pt_list.append(point_i)
        x = x + x_steps
        y = y + y_steps
        z = z + z_steps
    return pt_list

#---------------------------------------------------
#--------------- PROGRAM START ---------------------
from robodk.robolink import *  # API to communicate with RoboDK
from robodk.robomath import *  # basic matrix operations

# Generate the points curve path
POINTS = MakePoints(P_START, P_END, NUM_POINTS)

# Initialize the RoboDK API
RDK = Robolink()

# turn off auto rendering (faster)
RDK.Render(False)

# Automatically delete previously generated items (Auto tag)
list_names = RDK.ItemList()  # list all names
for item_name in list_names:
    if item_name.startswith('Auto'):
        RDK.Item(item_name).Delete()

# Promt the user to select a robot (if only one robot is available it will select that robot automatically)
robot = RDK.ItemUserPick('Select a robot', ITEM_TYPE_ROBOT)

# Turn rendering ON before starting the simulation
RDK.Render(True)

# Abort if the user hits Cancel
if not robot.Valid():
    quit()

# Retrieve the robot reference frame
reference = robot.Parent()

# Use the robot base frame as the active reference
robot.setPoseFrame(reference)

# get the current orientation of the robot (with respect to the active reference frame and tool frame)
pose_ref = robot.Pose()
print(Pose_2_TxyzRxyz(pose_ref))
# a pose can also be defined as xyzwpr / xyzABC
#pose_ref = TxyzRxyz_2_Pose([100,200,300,0,0,pi])

#-------------------------------------------------------------
# Option 1: Move the robot using the Python script

# We can automatically force the "Create robot program" action using a RUNMODE state
# RDK.setRunMode(RUNMODE_MAKE_ROBOTPROG)

# Iterate through all the points
for i in range(NUM_POINTS):
    # update the reference target with the desired XYZ coordinates
    pose_i = pose_ref
    pose_i.setPos(POINTS[i])

    # Move the robot to that target:
    robot.MoveJ(pose_i)

# Done, stop program execution
quit()

#-------------------------------------------------------------
# Option 2: Create the program on the graphical user interface
# Turn off rendering
RDK.Render(False)
prog = RDK.AddProgram('AutoProgram')

# Iterate through all the points
for i in range(NUM_POINTS):
    # add a new target and keep the reference to it
    ti = RDK.AddTarget('Auto Target %i' % (i + 1))
    # use the reference pose and update the XYZ position
    pose_i = pose_ref
    pose_i.setPos(POINTS[i])
    ti.setPose(pose_i)
    # force to use the target as a Cartesian target
    ti.setAsCartesianTarget()

    # Optionally, add the target as a Linear/Joint move in the new program
    prog.MoveL(ti)

# Turn rendering ON before starting the simulation
RDK.Render(True)

# Run the program on the simulator (simulate the program):
prog.RunProgram()
# prog.WaitFinished() # wait for the program to finish

# We can create the program automatically
# prog.MakeProgram()

# Also, if we have the robot driver we could use the following call to provoke a "Run on robot" action (simulation and the robot move simultaneously)
# prog.setRunType(PROGRAM_RUN_ON_ROBOT)

# Done, stop program execution
quit()

#-------------------------------------------------------------
# Option 3: Move the robot using the Python script and detect if movements can be linear
# This is an improved version of option 1
#
# We can automatically force the "Create robot program" action using a RUNMODE state
# RDK.setRunMode(RUNMODE_MAKE_ROBOTPROG)

# Iterate through all the points
ROBOT_JOINTS = None
for i in range(NUM_POINTS):
    # update the reference target with the desired XYZ coordinates
    pose_i = pose_ref
    pose_i.setPos(POINTS[i])

    # Move the robot to that target:
    if i == 0:
        # important: make the first movement a joint move!
        robot.MoveJ(pose_i)
        ROBOT_JOINTS = robot.Joints()
    else:
        # test if we can do a linear movement from the current position to the next point
        if robot.MoveL_Collision(ROBOT_JOINTS, pose_i) == 0:
            robot.MoveL(pose_i)
        else:
            robot.MoveJ(pose_i)
        ROBOT_JOINTS = robot.Joints()

# Done, stop program execution
quit()

#-------------------------------------------------------------
# Option 4: Create a follow curve project

# First we need to create an object from the provided points or add the points to an existing object and optionally project them on the surface

# Create a new object given the list of points (the 3xN vector can be extended to 6xN to provide the normal)
object_points = RDK.AddPoints(POINTS)

# Alternatively, we can project the points on the object surface
# object = RDK.Item('Object', ITEM_TYPE_OBJECT)
# object_points = object.AddPoints(POINTS, PROJECTION_ALONG_NORMAL_RECALC)
# Place the points at the same location as the reference frame of the object
# object_points.setParent(object.Parent())

# Set the name of the object containing points
object_points.setName('AutoPoints n%i' % NUM_POINTS)

path_settings = RDK.AddMillingProject("AutoPointFollow settings")
prog, status = path_settings.setMillingParameters(part=object_points)
# At this point, we may have to manually adjust the tool object or the reference frame

# Run the create program if success
prog.RunProgram()

# Done
quit()

#-------------------------------------------------------------
# Option 5: Create a follow points project (similar to Option 4)

# First we need to create an object from the provided points or add the points to an existing object and optionally project them on the surface

# Create a new object given the list of points:
object_curve = RDK.AddCurve(POINTS)

# Alternatively, we can project the points on the object surface
# object = RDK.Item('Object', ITEM_TYPE_OBJECT)
# object_curve = object.AddCurve(POINTS, PROJECTION_ALONG_NORMAL_RECALC)
# Place the curve at the same location as the reference frame of the object
# object_curve.setParent(object.Parent())

# Set the name of the object containing points
object_curve.setName('AutoPoints n%i' % NUM_POINTS)

# Create a new "Curve follow project" to automatically follow the curve
path_settings = RDK.AddMillingProject("AutoCurveFollow settings")
prog, status = path_settings.setMillingParameters(part=object_curve)
# At this point, we may have to manually adjust the tool object or the reference frame

# Run the create program if success
prog.RunProgram()

# Done
quit()

#-------------------------------------------------------------
# Option 6: Move the robot using the Python script and measure using an external measurement system
# This example is meant to help validating robot accuracy through distance measurements and using a laser tracker or a stereo camera

if robot.ConnectSafe() <= 0:
    raise Exception("Can't connect to robot")

RDK.setRunMode(RUNMODE_RUN_ROBOT)  # It is redundant if connection worked successfully

POINTS_NOMINAL = []
POINTS_ACCURATE = []
STABILIZATION_TIME = 2  # stabilization time in seconds before taking a measurement
for i in range(NUM_POINTS):
    # build a target using the reference orientation and the XYZ coordinates
    pose_i = pose_ref
    pose_i.setPos(LINE[i])

    # Move to the target with the nomrinal kinematics
    RDK.RunMessage('Moving to point %i (Nominal kinematics)' % (i + 1))
    RDK.RunMessage(str(Pose_2_KUKA(pose_i)))
    # Solve nominal inverse kinematics
    robot.setAccuracyActive(False)
    ji = robot.SolveIK(pose_i)
    robot.MoveL(ji)
    robot.Pause(STABILIZATION_TIME)

    # Take the measurement
    while True:
        pose1, pose2, npoints1, npoints2, time, errors = RDK.StereoCamera_Measure()
        if errors != 0 or npoints1 < 4 or npoints2 < 4:
            print('Invalid measurement. Retrying in 2 seconds...')
            pause(2)
        else:
            # calculate the pose of the tool with respect to the reference frame
            measured = invH(pose1) * pose2
            # retrieve XYZ value of the measurement
            POINTS_NOMINAL = measured.Pos()
            break

    # Move to the target with the accurate kinematics
    RDK.RunMessage('Moving to point %i (Accurate kinematics)' % (i + 1))
    robot.setAccuracyActive(True)
    ji = robot.SolveIK(pose_i)
    robot.MoveL(ji)
    robot.Pause(STABILIZATION_TIME)
    while True:
        pose1, pose2, npoints1, npoints2, time, errors = RDK.StereoCamera_Measure()
        if errors != 0 or npoints1 < 4 or npoints2 < 4:
            print('Invalid measurement. Retrying in 2 seconds...')
            pause(2)
        else:
            # calculate the pose of the tool with respect to the reference frame
            measured = invH(pose1) * pose2
            # retrieve XYZ value of the measurement
            POINTS_ACCURATE = measured.Pos()
            break

# At this point we can check the accurate vs the nominal kinematics and create the following table:
print('pa\tpb\tdist\tdist nom\tdist acc\terror nom\terror acc')
for i in range(numPoints):
    for j in range(numPoints + 1, numPoints):
        dist_program = distance(LINE[i], LINE[j])
        dist_nominal = distance(POINTS_NOMINAL[i], POINTS_NOMINAL[j])
        dist_accurate = distance(POINTS_ACCURATE[i], POINTS_ACCURATE[j])
        error_nominal = abs(dist_nominal - dist_program)
        error_accurate = abs(dist_accurate - dist_program)
        print('%i\t%i\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f' % (i + 1, j + 1, dist_program, dist_nominal, dist_accurate, error_nominal, error_accurate))

quit()


import argparse
import os.path as osp
from typing import Union

import numpy as np
import torch
import torch.nn.functional as F
from yacs.config import CfgNode as CN

UTILS_DIR = osp.dirname(osp.abspath(__file__))
DRL_DIR = osp.join(osp.dirname(UTILS_DIR))
CONFIGS_DIR = osp.join(DRL_DIR, "configs")
DEFAULT_CONFIG_FILE = osp.join(CONFIGS_DIR, "default.yaml")

def get_default_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser()
    parser.add_argument("-o", "--opts", nargs="*", default=[])
    return parser

def construct_config(config_path=None, opts=None):
    if config_path is None:
        config_path = DEFAULT_CONFIG_FILE
    if opts is None:
        opts = []
    config = CN()
    config.set_new_allowed(True)
    config.merge_from_file(config_path)
    config.merge_from_list(opts)

    return config

@torch.jit.script
def mse_loss(tensor1, tensor2):
    """Do F.mse_loss but ensure shapes match"""
    assert tensor1.shape == tensor2.shape, f"{tensor1.shape} != {tensor2.shape}"
    return F.mse_loss(tensor1, tensor2)

def get_num_reward_terms(envs, num_envs):
    # Count the amount of reward terms by doing a dummy step
    actions = torch.as_tensor(envs.action_space.sample())
    if actions.dim() == 1:
        actions = actions.repeat(num_envs, 1)
    _, _, _, infos = envs.step(actions)
    assert "reward_terms" in infos, "Key 'reward_terms' must be in info for EPPO!"
    num_reward_terms = infos["reward_terms"].shape[1]
    return num_reward_terms

def reparameterize_action(mu, sigma, actions):
    eps = (actions - mu) / sigma
    reparam_action = mu + sigma * eps.detach()
    return reparam_action

class MeanReturns:
    def __init__(self, window_size=50):
        self.window_size = window_size
        self.total_returns = None
        self.curr_returns = None

    def update(
        self,
        rewards: Union[np.ndarray, torch.Tensor],
        dones: Union[np.ndarray, torch.Tensor],
    ):
        rewards_tensor = torch.as_tensor(rewards).flatten()
        if self.curr_returns is None:
            self.curr_returns = torch.zeros_like(rewards_tensor)

        dones_mask = torch.as_tensor(dones).bool().flatten()

        # Update buffer of last window_size total returns
        if self.total_returns is None:
            self.total_returns = self.curr_returns[dones_mask]
        else:
            self.total_returns = torch.cat(
                [self.total_returns, self.curr_returns[dones_mask]]
            )
        self.total_returns = self.total_returns[-self.window_size :]

        self.curr_returns[dones_mask] = 0
        self.curr_returns += rewards_tensor

    def mean(self):
        if self.total_returns is None or self.total_returns.nelement() == 0:
            return 0
        return self.total_returns.mean().item()


from ds import queue
class tree(object):
	class treenode(object):
		def __init__(self, v):
			if type(v) is not int:
				print('need int as node value')
				return
			self.t = v
			self.left = None
			self.right = None
		def __repr__(self):
			return('val: {}, left: {}, right: {}'.format(self.t, self.left, self.right))

	def __init__(self, l):
		#default order
		s = queue()
		if l:
			s.add(self.treenode(l[0]))
			self.root = s.top()
		else:
			print('empty tree imported')
		i = 1
		while i < len(l):
			curnode = s.pop()
			ll = l[i]
			if ll != '#':
				newnode = self.treenode(ll)
				s.add(newnode)
				curnode.left = newnode
			i += 1
			if i >= len(l):
				break
			ll = l[i]
			if ll != '#':
				newnode = self.treenode(ll)
				s.add(newnode)
				curnode.right = newnode

			i += 1
		print('tree import completed')

	def add(self, a):
		r = self.root
		while r:
			if a < r.t:
				if not r.left:
					r.left = self.treenode(a)
					break
				else:
					r = r.left
			elif a > r.t:
				if not r.right:
					r.right = self.treenode(a)
					break
				else:
					r = r.right
			else:
				print('node value cannot be equal with')
	def depth(self):
		r = self.root
		def depth_(node):
			if not node:
				return 0
			else:
				return max(depth_(node.left), depth_(node.right)) + 1
		return depth_(r)
	def preorder(self):
		r = self.root
		q = queue()
		if r:
			q.add(r)
		else:
			print('empty tree')
		l = []
		while not q.isempty():
			n = q.pop()
			if n:
				l.append(n.t)
			else:
				l.append('#')
				continue
			q.add(n.left)
			q.add(n.right)
		while l[-1] == '#':
			l.pop()
		return l


import torch
from   torchmetrics import Metric
class MyAccuracy(Metric):
    def __init__( self):
        super().__init__()
        self.add_state("correct" , default = torch.zeros(3,3), dist_reduce_fx = "sum")
        self.add_state("total", default = torch.tensor(0), dist_reduce_fx = "sum")
    def update(self, precision):
        self.correct += precision
        self.total   += 1

    def compute(self):
        return self.correct / self.total

def top_precision(pred, target, L_left, L_right = None ):
    precisions = {}
    T,T        = target.shape
    mask_mat   = torch.ones(T,T)
    if(L_right == None):L_right = T

    L_preds    = pred.tril(L_right - 1).triu(L_left)
    mask_mat   = mask_mat.tril(L_right - 1).triu(L_left)
    L_preds    = torch.squeeze(L_preds.view(1,-1))
    target     = torch.squeeze(target.view(1,-1))

    true_L = torch.tensor(T)
    for k in [1,2,5]:
        top_k          = torch.ceil(true_L/k)
        values, indics = torch.topk(L_preds, k = int(top_k))
        right_count    = torch.gather(target, -1 , indics)
        pred_count = torch.gather(L_preds, -1, indics)
        precision      = torch.sum(right_count) / top_k
        precisions[f'precision_L{k}']  = precision

    return precisions

def all_range_presicion(pred, target):
    pred   = torch.squeeze(pred)
    target  = torch.squeeze(target)
    T,T     = target.shape
    acc_mat = torch.zeros(3,3)
    sep     = [6,12,24,T]
    for index in range(3):
        precisions = top_precision( pred  = pred
                                  , target = target
                                  , L_left = sep[index]
                                  , L_right= sep[index +1 ])
        acc_mat[index][0] = precisions[f'precision_L1']
        acc_mat[index][1] = precisions[f'precision_L2']
        acc_mat[index][2] = precisions[f'precision_L5']

    return acc_mat

def negative_probalbility(contact_maps, predict_prob, k = 6):

    B,T,T     = contact_maps.size()

    mask_mat   = torch.ones(T,T)
    mask_mat   = mask_mat.triu(k)
    num = 2 * torch.count_nonzero(mask_mat)

    logsoftmax = torch.nn.LogSoftmax()
    log_pred = logsoftmax(predict_prob)

    mask_c0_up = log_pred[0][0].triu(k)
    mask_c0_down = log_pred[0][0].tril(-k)
    mask_c0 = mask_c0_up + mask_c0_down

    mask_c1_up = log_pred[0][1].triu(k)
    mask_c1_down = log_pred[0][1].tril(-k)
    mask_c1 = mask_c1_up + mask_c1_down

    mask_c0 = mask_c0.unsqueeze(0)
    mask_c0 = mask_c0.unsqueeze(0)
    mask_c1 = mask_c1.unsqueeze(0)
    mask_c1 = mask_c1.unsqueeze(0)
    output = torch.cat((mask_c0, mask_c1), 1)
    target_up = torch.triu(contact_maps, diagonal = k)
    target_down = torch.tril(contact_maps, diagonal = -k)
    target = target_up + target_down

    nll = torch.nn.NLLLoss(reduction = 'sum')
    prob = nll(output, target) / num
    return prob

class MyLoss(Metric):
    def __init__(self):
        super().__init__()
        self.add_state("loss_sum"
                       , default        = torch.tensor(0, dtype=torch.float)
                       , dist_reduce_fx = "sum")
        self.add_state("total"
                       , default        = torch.tensor(0, dtype=torch.float)
                       , dist_reduce_fx = "sum")

    def update(self, loss: torch.tensor):
        self.loss_sum += loss
        self.total    += 1

    def compute(self):
        return self.loss_sum.float() / self.total


#!/usr/bin/env python3 -u
import time
import sys
from vwradio import avrclient
from vwradio.constants import Keys

client = avrclient.make_client()

if __name__ == '__main__':
    # build list of keycodes to try (one bit per key, max 32 bits)
    keycodes = []
    keycode = 1
    while len(keycodes) < 32:
        keycodes.append(keycode)
        keycode = keycode << 1

    client.set_auto_key_passthru(False)
    for keycode in keycodes:
        # put the display into a known state
        client.hit_key(Keys.MODE_AM)
        client.hit_key(Keys.MODE_FM)
        client.hit_key(Keys.PRESET_1)

        # split the 32-bit keycode to 4 key data bytes
        key_data = []
        for i in range(3, -1, -1):
            key_data.append((keycode >> (8*i) & 0xFF))

        # try the key
        if key_data == [0,0,0,0x40]:
            print('skipped initial')
        elif key_data == [0,0,0x02,0]:
            print('skipped scan')
        elif key_data == [0,0,0x20,0]:
            print('skipped code')
        elif key_data == [0,0x10,0,0]:
            print('skipped bal')
        elif key_data == [0,0,0,0x80]:
            print('skipped no code')
        else:
            sys.stdout.write("\n%r%s" % (key_data, ' ' * 25,))
            sys.stdout.flush()
            client.emulated_upd_load_key_data(key_data)
            time.sleep(20)
            reading_1 = client.read_lcd()
            client.emulated_upd_load_key_data([0, 0, 0, 0])
            time.sleep(0.25)
            reading_2 = client.read_lcd()

            if (reading_1 != 'FM11 891MHZ') or (reading_2 != 'FM11 891MHZ'):
                sys.stdout.write("\r%r down:%r up:%r\n" %
                    (key_data, reading_1, reading_2))
                sys.stdout.flush()
    client.set_auto_key_passthru(True)


'''字典就像一本地址簿，如果你知道了他或她的姓名，你就可以在这里找到其地址或是能够联系上对方的更多详细信息，换言之，我们将键值（Keys）（即姓名）与值（Values）（即地址等详细信息）联立到一起。在这里要注意到键值必须是唯一的，正如在现实中面对两个完全同名的人你没办法找出有关他们的正确信息。
另外要注意的是你只能使用不可变的对象（如字符串）作为字典的键值，但是你可以使用可变或不可变的对象作为字典中的值。基本上这段话也可以翻译为你只能使用简单对象作为键值。
在字典中，你可以通过使用符号构成 d = {key : value1 , key2 : value2} 这样的形式，来成对地指定键值与值。在这里要注意到成对的键值与值之间使用冒号分隔，而每一对键值与值则使用逗号进行区分，它们全都由一对花括号括起。'''
# 字典是属于 dict 类下的实例或对象。

# “ab”是地址（Address）簿（Book）的缩写

ab = {
    'Swaroop': 'swaroop@swaroopch.com',
    'Larry': 'larry@wall.org',
    'Matsumoto': 'matz@ruby-lang',
    'Spammer': 'spammer@hotmail.com'
}

print("Swaroop's address is", ab['Swaroop'])

# 删除一对键值-值配对
del ab['Spammer']

print('\n There are {} contacts in thee address-book\n'.format(len(ab)))

# item方法访问键值与值的信息
for name, address in ab.items():
    print('{}Contact {}'.format(name, address))
# 字典遍历
for key in ab:
    print(f'{key}  {ab[key]}')

# 添加一对键值-值配对
ab['Guido'] = 'guido@python.org'

# 判定值是否存在
if 'Guido' in ab:
    print("\nGuido's address is", ab['Guido'])


from django.shortcuts import render, get_object_or_404
from django.contrib.auth.decorators import login_required, permission_required
from .models import *
from taggit.models import Tag
from django.db.models import Count

from django.contrib.contenttypes.models import ContentType

def courselist(request, tag_slug=None):
    #course = Course.objects.all()
    course = Course.objects.filter(id__in=[1, 2, 3])
    free_course = Course.objects.filter(course_type="FREE")
    tag = None
    if tag_slug:
        tag = get_object_or_404(Tag, slug=tag_slug)
        course = course.filter(tags__in=[tag])

    context = {
              "course": course,
              'free':free_course,
              'tag': tag,

            }
    return render (request, 'mycourse/course_list.html',context)

def course_detail(request, slug):
    course = Course.objects.filter(course_slug=slug).first()
    section = Section.objects.filter(course=course)
    lecture = Lecture.objects.filter(course=course)

    course_tags_ids =course.tags.values_list('id', flat=True)
    similar_course = Course.objects.filter(tags__in=course_tags_ids).exclude(id=course.id)
    similar_course = similar_course.annotate(same_tags=Count('tags')).order_by('-same_tags')[:4]

    context ={

        "course":course,
        "section":section,
        "lecture":lecture,
        'similar_course':similar_course,

    }

    return render(request, 'mycourse/course_detail.html', context)


from http import HTTPStatus

from pydantic import BaseModel, ValidationError

from nrlf.core.errors import NRLF_TO_SPINE_4XX_ERROR
from nrlf.core.nhsd_codings import (
    META,
    IssueSeverity,
    IssueType,
    NrlfCoding,
    SpineCoding,
)
from nrlf.core.validators import json_loads
from nrlf.producer.fhir.r4.model import (
    CodeableConcept,
    Coding,
    OperationOutcome,
    OperationOutcomeIssue,
)

HTTP_STATUS_CODE_MAPPING = {
    SpineCoding.RESOURCE_NOT_FOUND: HTTPStatus.NOT_FOUND,
    SpineCoding.VALIDATION_ERROR: HTTPStatus.BAD_REQUEST,
    SpineCoding.NOT_ACCEPTABLE: HTTPStatus.FORBIDDEN,
    SpineCoding.MISSING_HEADER: HTTPStatus.BAD_REQUEST,
    SpineCoding.MISSING_VALUE: HTTPStatus.BAD_REQUEST,
    SpineCoding.ACCESS_DENIED_LEVEL: HTTPStatus.FORBIDDEN,
    SpineCoding.SERVICE_ERROR: HTTPStatus.INTERNAL_SERVER_ERROR,
    SpineCoding.INVALID_RESOURCE_ID: HTTPStatus.BAD_REQUEST,
    SpineCoding.INVALID_VALUE: HTTPStatus.CONFLICT,
}

class Response(BaseModel):
    status_code: int
    result: dict

def _get_error_from_exception(exception: Exception) -> tuple[int, SpineCoding]:
    coding: SpineCoding = NRLF_TO_SPINE_4XX_ERROR.get(
        exception.__class__, SpineCoding.SERVICE_ERROR
    )
    http_status: HTTPStatus = HTTP_STATUS_CODE_MAPPING[coding]
    return http_status, coding

def _operation_outcome(
    id: str,
    issue_type: IssueType,
    coding: SpineCoding,
    severity: IssueSeverity,
    diagnostics: str,
) -> dict:
    issue_details = CodeableConcept(coding=[coding.value])
    issue = [
        OperationOutcomeIssue(
            code=issue_type.value,
            severity=severity.value,
            diagnostics=diagnostics,
            details=issue_details,
        )
    ]
    return OperationOutcome(
        resourceType=OperationOutcome.__name__, id=id, meta=META, issue=issue
    ).dict(exclude_none=True)

def get_error_message(exception: Exception) -> str:
    return (
        _format_validation_error_message(exception=exception)
        if type(exception) is ValidationError
        else str(
            exception
        )  ### TODO: better error message, e.g. error type in the message?
    )

def _format_validation_error_message(exception: ValidationError) -> str:
    errors = json_loads(exception.json())
    first_error = errors[0]
    model_name = (
        exception.model.public_alias()
        if exception.model.__dict__.get("public_alias")
        else exception.model.__name__
    )
    error_msg_prefix = f"{model_name} validation failure"
    first_bad_property = f"Invalid {first_error['loc'][0]}"
    message_components = [error_msg_prefix, first_bad_property]
    if first_error["type"] == "value_error":
        message_components.append(first_error["msg"])
    return " - ".join(message_components)

def operation_outcome_ok(transaction_id: str, coding: NrlfCoding) -> dict:
    _coding: Coding = coding.value
    operation_outcome = _operation_outcome(
        id=transaction_id,
        coding=coding,
        diagnostics=_coding.display,
        issue_type=IssueType.INFORMATIONAL,
        severity=IssueSeverity.INFORMATION,
    )
    return operation_outcome

def operation_outcome_not_ok(
    transaction_id: str, exception: Exception
) -> tuple[int, dict]:
    status_code, coding = _get_error_from_exception(exception=exception)
    diagnostics = get_error_message(exception=exception)
    operation_outcome = _operation_outcome(
        id=transaction_id,
        coding=coding,
        diagnostics=diagnostics,
        issue_type=IssueType.PROCESSING,
        severity=IssueSeverity.ERROR,
    )
    return status_code, operation_outcome


'''
Text Preprocessor Unit Test Base
Author: Shahrukh Khan
'''
import unittest

class TestPreprocessorBase(unittest.TestCase):
    """ Base class for Test Preprocessor to supply parameters to Unit Test Class
    """
    def __init__(self, methodName='test_preprocessor', preprocess_config=None, data=None, groundtruth=None, log=None):
        super(TestPreprocessorBase, self).__init__(methodName)

        self.preprocess_config = preprocess_config
        self.data = data
        self.groundtruth = groundtruth
        self.log = log

    @staticmethod
    def parametrize(test_processor, preprocess_config=None, data=None, groundtruth=None, log=None):
        """ Create a suite containing all tests taken from the given
            subclass, passing them the parameters
        """
        # Get child names of this class
        testloader = unittest.TestLoader()
        testnames = testloader.getTestCaseNames(test_processor)
        suite = unittest.TestSuite()

        # Delegate passed parameter forward to child class
        for name in testnames:
            suite.addTest(test_processor(
                    name, preprocess_config=preprocess_config, data=data, groundtruth=groundtruth, log=log))

        return suite


import urllib.parse
import urllib.request
import urllib.response
import json

def get_metadata(chosen_title: str) -> dict:
    """
    Render resource in JSON format,
    convert JSON string to Pythonic object
    """
    url = f"http://dbpedia.org/data/{chosen_title}.json"
    # format chosen title into URL
    request = urllib.request.urlopen(url)
    request = request.read()
    request_dictionary = json.loads(request)
    for key, value in request_dictionary.items():
        if key == f"http://dbpedia.org/resource/{chosen_title}":
            # this is the "master key"
            metadata_dictionary = value
            # the value of this key is again a dictionary
    return metadata_dictionary

def get_abstract(metadata_dictionary: dict) -> str:
    """
    Iterate through metadata dictionary,
    return abstract of chosen title in English
    """
    if "http://dbpedia.org/ontology/abstract" in metadata_dictionary:
        # account for the possibility of an abstract not being available
        for key, value in metadata_dictionary.items():
            if key == "http://dbpedia.org/ontology/abstract":
                # the abstract can be found under this key
                list_of_abstracts = value
                # the value of this key is a list of dictionaries
                for abstract_dictionary in list_of_abstracts:
                    for key, value in abstract_dictionary.items():
                        # each dictionary contains two keys:
                        # "value": the actual abstract
                        # "lang": the language of the abstract
                        if key == "lang":
                            language = value
                            if language == "en":
                                # extract only the abtract in English
                                for key, value in abstract_dictionary.items():
                                    if key == "value":
                                        abstract = value
                                        print("\n")
                                        print("Abstract", ":", abstract)
        return abstract
    else:
        print("\n")
        print("Abstract: Abstract not found.")

def get_publication_date(metadata_dictionary: dict) -> str:
    """
    Iterate through metadata dictionary,
    return publication date of chosen title
    """
    if "http://dbpedia.org/ontology/publicationDate" in metadata_dictionary:
        # account for the possibility of a publication date not being available
        for key, value in metadata_dictionary.items():
            if key == "http://dbpedia.org/ontology/publicationDate":
                # the publication date can be found under this key
                list_of_publication_dates = value
                # the value of this key is a list of dictionaries
                for publication_date_dictionary in list_of_publication_dates:
                    for key, value in publication_date_dictionary.items():
                        if key == "value":
                            # the actual publication date can again be found under a key named "value"
                            publication_date = value
                            print("\n")
                            print("Publication date", ":", publication_date)
        return publication_date
    else:
        print("\n")
        print("Publication date: Publication date not found.")

def get_author(metadata_dictionary: dict) -> str:
    """
    Iterate through metadata dictionary,
    return author of chosen title
    """
    if "http://dbpedia.org/ontology/author" in metadata_dictionary:
        # account for the possibility of an author being unknown
        for key, value in metadata_dictionary.items():
            if key == "http://dbpedia.org/ontology/author":
                # the author URI can be found under this key
                list_of_authors = value
                # the value of this key is a list of dictionaries
                for author_dictionary in list_of_authors:
                    for key, value in author_dictionary.items():
                        if key == "value":
                            # the actual author URI can again be found under a key named "value"
                            author_URI = value
                            author = author_URI.rsplit('/', 1)[-1]
                            # extract the name of the author by splitting the URI at the last slash
                            # e.g. http://dbpedia.org/resource/J._K._Rowling -> J._K._Rowling
                            author_with_spaces = author.replace("_", " ")
                            # replace all underscores by spaces
                            print("\n")
                            print("Author", ":", author_with_spaces)
        return author
        # return the name of the author with underscores instead of spaces
    else:
        print("\n")
        print("Author: Author unkown.")


#!/usr/bin/python

import sys

###########################
# build pools
def get_name(info,tag):
	lst = []
	if tag == 'fa':
		for i in range(len(info)):
			header = info[i].split('\n')[0]
			if header not in lst:
				lst.append(header)
	elif tag == 'sam':
		for i in range(len(info)):
			header = info[i].split('\t')[0]
			if header not in lst:
				lst.append(header)
	return (lst)

def intersection(lst1, lst2):
	lst3 = [value for value in lst1 if value in lst2]
	return lst3

def remove_values_from_list(the_list, val):
	return [value for value in the_list if value != val]

def check_pool(lst,inters,tag):
	out = []
	if tag == 'fa':
		for i in range(len(lst)):
			header = lst[i].split('\n')[0]
			if header in inters:
				inters = remove_values_from_list(inters,header)
				out.append('>'+lst[i])
	elif tag == 'sam':
		for i in range(len(lst)):
			header = lst[i].split('\t')[0]
			if header in inters:
				inters = remove_values_from_list(inters,header)
				out.append(lst[i] + '\n')
	return (out)

####################################
if __name__ == '__main__':
	forwd = sys.argv[1]
	rev = sys.argv[2]
	with open(forwd,'r') as f:
        	forward = f.read()
	with open(rev,'r') as f:
        	reverse = f.read()
	#data processing
	tag = 'sam'
	if forward[0] == '>' and reverse[0] == '>':
        	fd = forward.split('>')[1:]
        	rv = reverse.split('>')[1:]
        	tag = 'fa'
	elif forward[0] != '>' and reverse[0] != '>':
        	fd = forward.split('\n')[:-1]
        	rv = reverse.split('\n')[:-1]
	else:
        	fd = ''
        	rv = ''
	forward = get_name(fd,tag)
	reverse = get_name(rv,tag)
	overlap = intersection(forward,reverse)
	fd = check_pool(fd,overlap,tag)
	rv = check_pool(rv,overlap,tag)
	# output files
	f = open(forwd+'-pair','w')
	r = open(rev+'-pair','w')
	string = ''.join(fd)
	f.write(string)
	string = ''.join(rv)
	r.write(string)
	f.close()
	r.close()


# Instructions
# Prime numbers are numbers that can only be cleanly divided by themselves and 1.

# https://en.wikipedia.org/wiki/Prime_number

# You need to write a function that checks whether if the number passed into it is a prime number or not.

# e.g. 2 is a prime number because it's only divisible by 1 and 2.

# But 4 is not a prime number because you can divide it by 1, 2 or 4.

prime = int(input("Enter the number which you want to check if it is a prime number or not"))
isPrime = True
x = 2

while prime%x==0 or x==prime-1:
    x+=1
    if prime%x == 0:
        isPrime = False
        break
    elif prime%x != 0:
        isPrime = True

if isPrime == True:
    print("It is a prime number")
else:
    print("It is not a prime number")


# external imports
import unittest
# local imports
import nautilus.models as models
from nautilus.contrib.graphene_peewee import PeeweeObjectType
from ...util import MockModel

class TestUtil(unittest.TestCase):

    def setUp(self):

        # the base model to test
        TestModel = MockModel()

        # the object type based on the models
        class TestObjectType(PeeweeObjectType):
            class Meta:
                model = TestModel

        # save the mocks to the test case
        self.model = TestModel
        self.object_type = TestObjectType

    def test_generated_object_has_model_fields(self):
        # the list of fields in the service object
        service_object_fields = {field.default_name \
                                    for field in self.object_type._meta.fields}
        # the list of fields in the models
        model_fields = {field.name for field in self.model.fields()}
        # make sure the two lists are the same
        assert model_fields == service_object_fields, (
            "PeeweeObjectType does not have the same fields as the model"
        )


import frappe
import json

from frappe import _

def get_context(context):
    if frappe.session.user == 'Guest':
        frappe.local.flags.redirect_location = '/'
        raise frappe.Redirect

    context.user = frappe.session.user
    context.csrf_token = frappe.sessions.get_csrf_token()

    context.employee = frappe.db.get_value("Employee", {"user_id": context.user}, "name") or ""
    context.holiday_list = frappe.db.get_value("Employee", {"user_id": context.user}, "holiday_list") or ""

    valid_roles = ['Leave Approver']

    if not('Leave Approver' in frappe.get_roles() or 'HR Manager' in frappe.get_roles()):
        frappe.throw(_('Only users with {0} role can access').format(', '.join(valid_roles)),
			frappe.PermissionError)

    allow_delete_leave_history = 0
    allow_cancel_leave_history = 0
    is_hr_manager = 0

    if 'HR Manager' in frappe.get_roles():
        # allow_delete_leave_history = 1
        allow_cancel_leave_history = 1
        is_hr_manager = 1

    context.allow_delete_leave_history = allow_delete_leave_history
    context.allow_cancel_leave_history = allow_cancel_leave_history
    context.is_hr_manager = is_hr_manager

    context.leave_requests = frappe.db.sql("""
		select la.*, f.file_name, f.file_url
        from `tabLeave Application` la
        left join `tabFile` f on f.attached_to_doctype = "Leave Application" and f.attached_to_name = la.name
		where la.docstatus = 0 and la.leave_approver = %(user)s
	""",{
        "user": frappe.session.user
    }, as_dict=True, debug=1)

    condition_history = "la.docstatus = 1"

    if not is_hr_manager:
        condition_history += " and la.leave_approver = '{0}'".format(frappe.session.user)

    context.leave_historys = frappe.db.sql("""
		select la.*, f.file_name, f.file_url
        from `tabLeave Application` la
        left join `tabFile` f on f.attached_to_doctype = "Leave Application" and f.attached_to_name = la.name
		where {0}
	""".format(condition_history), as_dict=True, debug=1)

    context.list_employee = frappe.db.get_all("Employee",fields=["name", "employee_name"])
    context.list_leave_type = frappe.db.get_all("Leave Type",fields=["name"])
    context.list_leave_status = ["Open","Approved","Rejected","Cancelled"]

    return context


from jax.experimental import host_callback
from tqdm.std import tqdm as tqdm_tp

import jax
from jax.experimental import host_callback
import jax.numpy as jnp

def progress_bar_factory(tqdm_bar: tqdm_tp, num_iters):
    """Factory that builds a progress bar decorator along
    with the `set_tqdm_description` and `close_tqdm` functions
    """
    if num_iters > 100:
        print_rate = int(num_iters / 100)
    else:
        print_rate = 1

    # remainder = num_iters % print_rate

    total_calls = 0

    def _update_tqdm(arg, transform):
        nonlocal total_calls
        total_calls += 1
        thinning, chain_ids = arg
        num_chains = len(chain_ids)
        # print(total_calls, thinning * num_chains, arg)
        tqdm_bar.set_description(f"Running chain", refresh=True)
        tqdm_bar.update(thinning * num_chains)

    def _close_tqdm(arg, transform, device):
        tqdm_bar.update(arg)
        tqdm_bar.close()

    def _update_progress_bar(iter_num, chain_id) -> int:
        """Updates tqdm progress bar of a JAX loop only if the iteration number is a multiple of the print_rate
        Usage: carry = progress_bar((iter_num, print_rate), carry)
        """
        from jax.tree_util import tree_all
        cond = jnp.sum((chain_id == 1) & (chain_id == 0)).astype(bool)
        ret = jax.lax.cond(
            (iter_num + 1) % print_rate == 0,
            lambda : host_callback.id_tap(_update_tqdm, (print_rate, chain_id), result=iter_num+1),
            lambda : iter_num,
        )
        return ret

    return _update_progress_bar


# Definition for a binary tree node.
# class TreeNode:
#     def __init__(self, x):
#         self.val = x
#         self.left = None
#         self.right = None

# 因为是搜索二叉树，左小右大，顺着二叉树搜索，分叉点就是近的祖先
class Solution:
    def lowestCommonAncestor(self, root: 'TreeNode', p: 'TreeNode', q: 'TreeNode') -> 'TreeNode':
        p, q = (p, q) if p.val <= q.val else (q, p)
        def dfs(root, p ,q):
            if p.val <= root.val <= q.val: # 出现分叉情况
                return root
            if p.val <= q.val < root.val:
                t = dfs(root.left, p, q)
            elif root.val < p.val <= q.val:
                t = dfs(root.right, p, q)
            return t
        return dfs(root, p, q)

# # 和236相同的递归算法(这题不推荐，比较慢)
# class Solution:
#     def lowestCommonAncestor(self, root: 'TreeNode', p: 'TreeNode', q: 'TreeNode') -> 'TreeNode':
#         if not root: return None
#         if root.val == p.val or root.val == q.val:
#             return root
#         le = self.lowestCommonAncestor(root.left, p, q)
#         ri = self.lowestCommonAncestor(root.right, p, q)
#         if le and ri: return root
#         return le or ri


from .command import Command
import itertools
from tabulate import tabulate

class OutboxCommand(Command):
    def __init__(self, args):
        super().__init__(args)
        self.offset = args.offset
        self.limit = args.limit

    def run(self):
        actor = self.logged_in_actor()
        if actor is None:
            raise Exception("Not logged in")
        outbox = actor.get("outbox", None)
        if outbox is None:
            raise Exception("No outbox found")
        outbox_id = self.to_id(outbox)
        slice = itertools.islice(
            self.items(outbox_id), self.offset, self.offset + self.limit
        )
        rows = []
        for item in slice:
            id = self.to_id(item)
            type = item.get("type", None)
            summary = self.text_prop(item, "summary")
            published = item.get("published")
            rows.append([id, type, summary, published])
        print(tabulate(rows, headers=["id", "type", "summary", "published"]))


import math
import logging
from functools import partial
from collections import OrderedDict
from copy import deepcopy

import torch
import torch.nn as nn
import torch.nn.functional as F

from timm.data import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD
from timm.models.layers import PatchEmbed, Mlp, DropPath, trunc_normal_, lecun_normal_

_logger = logging.getLogger(__name__)

class Attention(nn.Module):
    def __init__(self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0., proj_drop=0.):
        super().__init__()
        self.num_heads = num_heads
        head_dim = dim // num_heads
        self.scale = qk_scale or head_dim ** -0.5

        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
        self.attn_drop = nn.Dropout(attn_drop)
        self.proj = nn.Linear(dim, dim)
        self.proj_drop = nn.Dropout(proj_drop)

    def forward(self, x):
        B, N, C = x.shape
        qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)
        q, k, v = qkv[0], qkv[1], qkv[2]   # make torchscript happy (cannot use tensor as tuple)

        attn = (q @ k.transpose(-2, -1)) * self.scale
        attn = attn.softmax(dim=-1)
        attn = self.attn_drop(attn)

        x = (attn @ v).transpose(1, 2).reshape(B, N, C)
        attn_ft = (attn @ v)

        x = self.proj(x)
        x = self.proj_drop(x)
        return x, attn, attn_ft

class Block(nn.Module):

    def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop=0., attn_drop=0.,
                 drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm):
        super().__init__()
        self.norm1 = norm_layer(dim)
        self.attn = Attention(
            dim, num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop, proj_drop=drop)
        # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here
        self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()
        self.norm2 = norm_layer(dim)
        mlp_hidden_dim = int(dim * mlp_ratio)
        self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop)

    def forward(self, x):
        skip1 = x
        x = self.norm1(x)
        x, attn, attn_ft = self.attn(x)
        x = self.drop_path(x)
        x = skip1 + x
        x1 = x

        skip2 = x
        x = self.norm2(x)
        x = self.mlp(x)
        x = self.drop_path(x)
        x = skip2 + x
        x2 = x

        return x1, x2, attn, attn_ft

class VisionTransformer(nn.Module):
    """ Vision Transformer

    A PyTorch impl of : `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale`
        - https://arxiv.org/abs/2010.11929

    Includes distillation token & head support for `DeiT: Data-efficient Image Transformers`
        - https://arxiv.org/abs/2012.12877
    """

    def __init__(self, img_size=224, patch_size=16, in_chans=3, num_classes=1000, embed_dim=768, depth=12,
                 num_heads=12, mlp_ratio=4., qkv_bias=True, qk_scale=None, representation_size=None, distilled=False,
                 drop_rate=0., attn_drop_rate=0., drop_path_rate=0., embed_layer=PatchEmbed, norm_layer=None,
                 act_layer=None, weight_init=''):
        """
        Args:
            img_size (int, tuple): input image size
            patch_size (int, tuple): patch size
            in_chans (int): number of input channels
            num_classes (int): number of classes for classification head
            embed_dim (int): embedding dimension
            depth (int): depth of transformer
            num_heads (int): number of attention heads
            mlp_ratio (int): ratio of mlp hidden dim to embedding dim
            qkv_bias (bool): enable bias for qkv if True
            qk_scale (float): override default qk scale of head_dim ** -0.5 if set
            representation_size (Optional[int]): enable and set representation layer (pre-logits) to this value if set
            distilled (bool): model includes a distillation token and head as in DeiT models
            drop_rate (float): dropout rate
            attn_drop_rate (float): attention dropout rate
            drop_path_rate (float): stochastic depth rate
            embed_layer (nn.Module): patch embedding layer
            norm_layer: (nn.Module): normalization layer
            weight_init: (str): weight init scheme
        """
        super().__init__()
        self.num_classes = num_classes
        self.num_features = self.embed_dim = embed_dim  # num_features for consistency with other models
        self.num_tokens = 2 if distilled else 1
        norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6)
        act_layer = act_layer or nn.GELU

        self.patch_embed = embed_layer(
            img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim)
        num_patches = self.patch_embed.num_patches

        self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))
        self.dist_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) if distilled else None
        self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + self.num_tokens, embed_dim))
        self.pos_drop = nn.Dropout(p=drop_rate)

        dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)]  # stochastic depth decay rule
        self.blocks = nn.Sequential(*[
            Block(
                dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale,
                drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer, act_layer=act_layer)
            for i in range(depth)])
        self.norm = norm_layer(embed_dim)

        # Representation layer
        if representation_size and not distilled:
            self.num_features = representation_size
            self.pre_logits = nn.Sequential(OrderedDict([
                ('fc', nn.Linear(embed_dim, representation_size)),
                ('act', nn.Tanh())
            ]))
        else:
            self.pre_logits = nn.Identity()

        # Classifier head(s)
        self.head = nn.Linear(self.num_features, num_classes) if num_classes > 0 else nn.Identity()
        self.head_dist = None
        if distilled:
            self.head_dist = nn.Linear(self.embed_dim, self.num_classes) if num_classes > 0 else nn.Identity()

        # Weight init
        assert weight_init in ('jax', 'jax_nlhb', 'nlhb', '')
        head_bias = -math.log(self.num_classes) if 'nlhb' in weight_init else 0.
        trunc_normal_(self.pos_embed, std=.02)
        if self.dist_token is not None:
            trunc_normal_(self.dist_token, std=.02)
        if weight_init.startswith('jax'):
            # leave cls token as zeros to match jax impl
            for n, m in self.named_modules():
                _init_vit_weights(m, n, head_bias=head_bias, jax_impl=True)
        else:
            trunc_normal_(self.cls_token, std=.02)
            self.apply(_init_vit_weights)

    def _init_weights(self, m):
        # this fn left here for compat with downstream users
        _init_vit_weights(m)

    @torch.jit.ignore
    def no_weight_decay(self):
        return {'pos_embed', 'cls_token', 'dist_token'}

    def get_classifier(self):
        if self.dist_token is None:
            return self.head
        else:
            return self.head, self.head_dist

    def reset_classifier(self, num_classes, global_pool=''):
        self.num_classes = num_classes
        self.head = nn.Linear(self.embed_dim, num_classes) if num_classes > 0 else nn.Identity()
        if self.num_tokens == 2:
            self.head_dist = nn.Linear(self.embed_dim, self.num_classes) if num_classes > 0 else nn.Identity()

    def forward_features(self, x):
        zs = []
        attns = []
        attn_fts = []

        x = self.patch_embed(x)
        cls_token = self.cls_token.expand(x.shape[0], -1, -1)  # stole cls_tokens impl from Phil Wang, thanks
        if self.dist_token is None:
            x = torch.cat((cls_token, x), dim=1)
        else:
            x = torch.cat((cls_token, self.dist_token.expand(x.shape[0], -1, -1), x), dim=1)
        x = self.pos_drop(x + self.pos_embed)

        zs.append(x)

        for i, block in enumerate(self.blocks):
            x1, x2, attn, attn_ft = block(x)
            x = x2

            zs.append(x1)
            zs.append(x2)
            attns.append(attn)
            attn_fts.append(attn_ft)

        x = self.norm(x)
        if self.dist_token is None:
            return self.pre_logits(x[:, 0]), zs, attns, attn_fts
        else:
            return x[:, 0], x[:, 1]

    def forward(self, x):
        x, zs, attns, attn_fts = self.forward_features(x)
        if self.head_dist is not None:
            x, x_dist = self.head(x[0]), self.head_dist(x[1])  # x must be a tuple
            if self.training and not torch.jit.is_scripting():
                # during inference, return the average of both classifier predictions
                return x, x_dist
            else:
                return (x + x_dist) / 2
        else:
            x = self.head(x)

        zs.append(x)
        return x, zs, attns, attn_fts

def _init_vit_weights(m, n: str = '', head_bias: float = 0., jax_impl: bool = False):
    """ ViT weight initialization
    * When called without n, head_bias, jax_impl args it will behave exactly the same
      as my original init for compatibility with prev hparam / downstream use cases (ie DeiT).
    * When called w/ valid n (module name) and jax_impl=True, will (hopefully) match JAX impl
    """
    if isinstance(m, nn.Linear):
        if n.startswith('head'):
            nn.init.zeros_(m.weight)
            nn.init.constant_(m.bias, head_bias)
        elif n.startswith('pre_logits'):
            lecun_normal_(m.weight)
            nn.init.zeros_(m.bias)
        else:
            if jax_impl:
                nn.init.xavier_uniform_(m.weight)
                if m.bias is not None:
                    if 'mlp' in n:
                        nn.init.normal_(m.bias, std=1e-6)
                    else:
                        nn.init.zeros_(m.bias)
            else:
                trunc_normal_(m.weight, std=.02)
                if m.bias is not None:
                    nn.init.zeros_(m.bias)
    elif jax_impl and isinstance(m, nn.Conv2d):
        # NOTE conv was left to pytorch default in my original init
        lecun_normal_(m.weight)
        if m.bias is not None:
            nn.init.zeros_(m.bias)
    elif isinstance(m, nn.LayerNorm):
        nn.init.zeros_(m.bias)
        nn.init.ones_(m.weight)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Jun  3 19:17:40 2019

@author: ajit
"""
import xlrd

# excel files have rather long names, lets shorten them
sName = {
        'Chemical Engineering' : 'CHD',
        'Civil Engineering' : 'CED',
        'Computer Science and Engineering' : 'CSE',
        'Electrical Engineering' : 'EEE',
        'Mechanical Engineering' : 'MED',
        'Art & Performing Art' : 'ADP',
        'Communication' : 'COM',
        'Design' : 'DES',
        'Economics' : 'ECO',
        'English' : 'ENG',
        'History' : 'HIS',
        'International Relations and Governance Studies' : 'INT',
        'Sociology' : 'SOC',
        'Management': 'MGT',
        'Chemistry' : 'CHY',
        'Life Sciences' : 'BIO',
        'Mathematics' : 'MAT',
        'Physics' : 'PHY',
        'Archaeology' : 'HIS', # TODO: check which dept offers history
        'Biotechnology' : 'BIO',
        'Chemistry' : 'CHY',
        'Dance' : 'ADP',  # TODO: check which dept offers dance
        'Data Analytics': 'MGT', # TODO: check which dept offers data analytics
        'Electronics and Communication Engineering' : 'ECE',
        'International Relations and Public Affairs': 'INT',
        'Electrical and Electronics Engineering': 'EEE',
        'International Relations': 'INT',
        'Art Design and Performing Arts': 'ADP',
        'Bachelor of Management Studies': 'BMS',
        'Under Graduate': 'UGG',
        'OHM': 'MGT',
        'General Management': 'MGT',
        'FAC': 'MGT',
        'STM': 'MGT',
        'DOM': 'MGT'
        }

# read excel file
# data stored as a dictionary in courseList
#---------------------------------------------

COSfilePath = "COS_backUp_Jul05.xlsx"
wb = xlrd.open_workbook(COSfilePath)
sheet = wb.sheet_by_index(0) # get first sheet

# column heading numbers
School	=	0
Department	=	1
CourseCode	=	2
CourseTitle	=	3
Credits	=	4
CourseType	=	5
OpenasUWE	=	6
PartofMinoras	=	7
Programs	=	[8, 9, 10, 11, 12, 13, 14, 15, 16, 17]
CourseCapacity	=	18
LectureHoursPerWeek	=	19
TutorialHoursPerWeek	=	20
PracticalHoursPerWeek	=	21
LectureDuration	=	22
LectureSections	=	23
TutorialSections	=	24
PracticalSections	=	25
LectureInstructors	=	26
TutorialInstructors	=	27
PracticalInstructors	=	28
LabRoomNumber	=	29
LabCapacity	=	30
SpecialRequirement	=	31
LectureSlottedoutsideMonFriand95	=	32
TutorialSlottedoutsideMonFriand95	=	33
PracticalSlottedoutsideMonFriand95	=	34
LectureRemarks	=	35
TutorialRemarks	=	36
PracticalRemarks	=	37
CourseCoordinator	=	38
LectureCredits	=	39
TutorialCredits	=	40
PracticalCredits	=	41
AssessmentTypeWeightage	=	42
CrossListedwith	=	43
Prerequisite	=	44
Crosslistedcourseoffered	=	45
Lasttimeofferedsemester	=	46

courseList = dict()

for i in range(1,sheet.nrows):

    # loop over courseCode column, make a dictionary for each course
    # if a cell in this column is empty, that means that row has details of
    # last defined course code

    # skip over empty cells
    if (sheet.cell_type(i,CourseCode) == xlrd.XL_CELL_EMPTY):
        continue

    j = 0 # How many EXTRA rows for a particular course
    if ((i+j+1) < sheet.nrows):
        while (sheet.cell_type(i+j+1,CourseCode) == xlrd.XL_CELL_EMPTY):
            j = j+1

    cCode = sheet.cell_value(i,CourseCode)

    courseList[cCode] = dict()

    courseI = courseList[cCode]
    courseI['Credits'] = int(sheet.cell_value(i,Credits))
    courseI['Title'] = sheet.cell_value(i,CourseTitle)
    courseI['CourseType'] = sheet.cell_value(i,CourseType)

    if (courseI['CourseType'] != 'CCC'):
        courseI['level'] = cCode[-3]
    else:
        courseI['level'] = 0

    courseI['OpenasUWE'] = sheet.cell_value(i,OpenasUWE)
    courseI['PartofMinoras'] = sheet.cell_value(i,PartofMinoras)
    courseI['CourseCapacity'] = int(sheet.cell_value(i,CourseCapacity))

    # read target audience of the course
    courseI['programs'] = set()
    for p in Programs:
        if (sheet.cell_type(i,p) != xlrd.XL_CELL_EMPTY):
            cellVal = sheet.cell_value(i,p)
#            cellVal = cellVal[:-1]+cellVal[-1]
            courseI['programs'].add(cellVal)

    # read number lecture hours
    if (sheet.cell_type(i,LectureHoursPerWeek) == xlrd.XL_CELL_EMPTY):
        courseI['LectureHoursPerWeek']=0
    else:
        courseI['LectureHoursPerWeek']=int(2*float(sheet.cell_value(i,LectureHoursPerWeek)))
        courseI['LectureDuration']=int(2*float(sheet.cell_value(i,LectureDuration)))

    #read number of tutorials hours
    if (sheet.cell_type(i,TutorialHoursPerWeek) == xlrd.XL_CELL_EMPTY):
        courseI['TutorialHoursPerWeek']=0
    else:
        courseI['TutorialHoursPerWeek']=int(2*float(sheet.cell_value(i,TutorialHoursPerWeek)))

    #read number of lab hours
    if (sheet.cell_type(i,PracticalHoursPerWeek) == xlrd.XL_CELL_EMPTY):
        courseI['PracticalHoursPerWeek']=0
    else:
        courseI['PracticalHoursPerWeek']=int(2*float(sheet.cell_value(i,PracticalHoursPerWeek)))

    # read lecture sections and respective instructors
    if (courseI['LectureHoursPerWeek'] >= 1):

        courseI['lecSections'] = dict()
        if (j == 0):
            seci = 'LEC1'
            courseI['lecSections'][seci] = dict()
            courseI['lecSections'][seci]['instructors'] = sheet.cell_value(i,LectureInstructors).split(',\n')[:-1]

            courseI['lecSections'][seci]['students'] = set()
        else:
            for k in range(i,i+j+1):
                if (sheet.cell_type(k,LectureSections) != xlrd.XL_CELL_EMPTY):
                    seci = sheet.cell_value(k,LectureSections)
                    courseI['lecSections'][seci] = dict()
                    courseI['lecSections'][seci]['instructors'] = sheet.cell_value(k,LectureInstructors).split(',\n')[:-1]
                    courseI['lecSections'][seci]['students'] = set()

    # read tut sections and respective instructors
    if (courseI['TutorialHoursPerWeek'] >= 1):
        courseI['tutSections'] = dict()

        if (j == 0) :
            seci = 'TUT1'
            courseI['tutSections'][seci] = dict()
            courseI['tutSections'][seci]['instructors'] = sheet.cell_value(i,TutorialInstructors).split(',\n')[:-1]
            courseI['tutSections'][seci]['students'] = set()
        else:
            for k in range(i,i+j+1):
                if (sheet.cell_type(k,TutorialSections) != xlrd.XL_CELL_EMPTY):
                    seci = sheet.cell_value(k,TutorialSections)
                    courseI['tutSections'][seci] = dict()
                    courseI['tutSections'][seci]['instructors'] = sheet.cell_value(k,TutorialInstructors).split(',\n')[:-1]
                    courseI['tutSections'][seci]['students'] = set()

    # read Lab sections, respective instructors, and lab room number
    if (courseI['PracticalHoursPerWeek'] >= 1):
       courseI['labSections'] = dict()
       if (j == 0):
           praci = 'PRAC1'
           courseI['labSections'][praci] = dict()
           courseI['labSections'][praci]['instructors'] =  sheet.cell_value(i,PracticalInstructors).split(',\n')[:-1]
           courseI['labSections'][praci]['students'] =  set()
           courseI['labSections'][praci]['room'] =  sheet.cell_value(i,LabRoomNumber)
           courseI['labSections'][praci]['students'] = set()

       else:
           for k in range(i,i+j+1):
               if (sheet.cell_type(k,PracticalSections) != xlrd.XL_CELL_EMPTY):
                    praci = sheet.cell_value(k,PracticalSections)
                    courseI['labSections'][praci] = dict()
                    courseI['labSections'][praci]['instructors'] =  sheet.cell_value(k,PracticalInstructors).split(',\n')[:-1]
                    courseI['labSections'][praci]['room'] =  sheet.cell_value(k,LabRoomNumber)
                    courseI['labSections'][praci]['students'] = set()

# create a dictionary major electives, minor electives and UWEs
majorElectives = dict()
minorElectives = dict()
UWEnotInMinors = dict()
UWEs = dict()

for cIndex, c in courseList.items():
    d = cIndex[0:3] # dept
    y = cIndex[3]   # year

    # combine many business school minor in one
    if (d == 'MKT') or (d == 'DOM') or (d == 'FAC') or (d == 'OHM') or (d == 'STM'):
        d = 'MGT'

    if (int(y) >= 4):
        y = '4'

    #combine year and dept again
    d = d+y

    if 'Yes' in c['OpenasUWE']:
        if d in UWEs:
            UWEs[d].add(cIndex)
        else:
            UWEs[d] = set()
            UWEs[d].add(cIndex)

    # update the list of major electives
    if 'Major Elective' in c['CourseType']:
        if d in majorElectives:
            majorElectives[d].add(cIndex)
        else:
            majorElectives[d] = set()
            majorElectives[d].add(cIndex)

    # update list of minor electives
    # treat UWE which are not part of minors as minor electives
    #    for time-tabling purpose
    if 'Elective' in c['PartofMinoras']:
        if d in minorElectives:
            minorElectives[d].add(cIndex)
        else:
            minorElectives[d] = set()
            minorElectives[d].add(cIndex)

    if ('Not a part' in c['PartofMinoras']) and ('Yes' in c['OpenasUWE']):
        if d in UWEnotInMinors:
            UWEnotInMinors[d].add(cIndex)
        else:
            UWEnotInMinors[d] = set()
            UWEnotInMinors[d].add(cIndex)

# extract a list of CCC
CCCcourses = set()
for cIndex in courseList:
    if 'CCC' in cIndex:
        CCCcourses.add(cIndex)

# read target audience for each major electives
majorElectiveTakers = dict()
for ci in courseList:
    c = courseList[ci]
    if ('Major Elective' in c['CourseType']):

        if (ci in majorElectiveTakers):
            majorElectiveTakers[ci].add(c['programs'])
        else:
            majorElectiveTakers[ci] = set()
            majorElectiveTakers[ci] = majorElectiveTakers[ci].union(c['programs'])

# for each group create a set of major elective course
majorElectivePool = dict()
for ci in majorElectiveTakers:
    stSeti = majorElectiveTakers[ci]

    for sti in stSeti:
        if (sti in majorElectivePool):
            majorElectivePool[sti].add(ci)
        else:
            majorElectivePool[sti] = set()
            majorElectivePool[sti].add(ci)


# Configuration file for the Sphinx documentation builder.

import os
import sys
sys.path.append(os.path.abspath('../../src/'))

# -- Project information

project = 'GAMSPy'
copyright = "2023,  GAMS Development Corporation"
author = "Muhammet Soyturk"

release = '0.1'
version = '0.1.0'

# -- General configuration

extensions = [
    'sphinx.ext.duration',
    'sphinx.ext.doctest',
    'sphinx.ext.autodoc',
    'sphinx.ext.autosummary',
    'sphinx.ext.intersphinx',
    'sphinx.ext.mathjax',
    'sphinx.ext.todo',
    "nbsphinx",
    "sphinx_copybutton",
    "numpydoc"
]

intersphinx_mapping = {
    'python': ('https://docs.python.org/3/', None),
    'sphinx': ('https://www.sphinx-doc.org/en/master/', None),
}
intersphinx_disabled_domains = ['std']

templates_path = ['../_templates']
exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
autosummary_generate = True
todo_include_todos = True

# -- Options for HTML output

html_theme = 'sphinx_rtd_theme'

# -- Options for EPUB output
epub_show_urls = 'footnote'

# Configuration file for the Sphinx documentation builder.
#
# For the full list of built-in configuration values, see the documentation:
# https://www.sphinx-doc.org/en/master/usage/configuration.html

# -- Project information -----------------------------------------------------
# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information

# project = "GAMSpy"
# copyright = "2023, Muhammet Soyturk"
# author = "Muhammet Soyturk"
# release = "0.1.0"

# # -- General configuration ---------------------------------------------------
# # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration

# extensions = [
#     "sphinx.ext.autodoc",
#     "sphinx.ext.viewcode",
#     "numpydoc",
#     "sphinx.ext.intersphinx",
#     "sphinx.ext.coverage",
#     "sphinx.ext.doctest",
#     "sphinx.ext.autosummary",
#     "sphinx.ext.graphviz",
#     "sphinx.ext.ifconfig",
#     "matplotlib.sphinxext.plot_directive",
#     "IPython.sphinxext.ipython_console_highlighting",
#     "IPython.sphinxext.ipython_directive",
#     "sphinx.ext.mathjax",
#     "sphinx_design",
#     "sphinx_copybutton",
#     "nbsphinx"
# ]

# templates_path = ["_templates"]
# exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]

# # -- Options for HTML output -------------------------------------------------
# # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output

# html_theme = 'sphinx_rtd_theme'
# html_static_path = ["_static"]
# html_show_sourcelink = False
# html_theme_options = {
#     "icon_links": [
#         {
#             "name": "GitHub",
#             "url": "https://github.com/GAMS-dev",
#             "icon": "fa-brands fa-square-github",
#             "type": "fontawesome",
#         },
#         {
#             "name": "Twitter",
#             "url": "https://twitter.com/GamsSoftware",
#             "icon": "fa-brands fa-square-twitter",
#         },
#     ],
#     "logo": {
#         "image_light": "_static/logo.png",
#         "image_dark": "_static/logo.png",
#     }
# }


import os

class Translator():
    def __init__(self):
        self.heatmaps = []
        self.position_dict = {
            "cb": "Center Back",
            "rb": "Right Back",
            "lb": "Left Back",
            "cm": "Center Midfield",
            "rm": "Right Midfield",
            "lm": "Left Midfield",
            "dm": "Defensive Midfield",
            "am": "Attacking Midfield",
            "rw": "Right Wing",
            "lw": "Left Wing",
            "fw": "Center Forward"
        }
        self.load_heatmaps()

    def load_heatmaps(self):
        for data in os.listdir("../data/scaled_heatmaps"):
            self.heatmaps.append(data[:-4])

    def translate_position(self,pos_ab,formation):
        position = self.position_dict[pos_ab[:2]]
        modifier = pos_ab[-1]
        if position in ["Center Back","Center Midfield","Defensive Midfield","Attacking Midfield","Center Forward"]:
            left = "Left " + position + "_" + formation
            centre = position + "_" + formation
            right = "Right " + position + "_" + formation
            if modifier == "1":
                if left in self.heatmaps:
                    return left
                elif centre in self.heatmaps:
                    return centre
                # else:
                #     print("Translation failed for: " + pos_ab + "_" + formation)
                #     return ""
            if modifier == "2":
                if centre in self.heatmaps:
                    return centre
                elif right in self.heatmaps:
                    return right
                # else:
                #     print("Translation failed for: " + pos_ab + "_" + formation)
                #     return ""
            if modifier == "3":
                if right in self.heatmaps:
                    return right
                elif centre in self.heatmaps:
                    return centre
                # else:
                #     print("Translation failed for: " + pos_ab + "_" + formation)
                #     return ""
        if position == "Right Back" or position == "Left Back":
            fullback = position + "_" + formation
            wingback = position.split(" ")[0] + " Wing Back" + "_" + formation
            if fullback in self.heatmaps:
                return fullback
            elif wingback in self.heatmaps:
                return wingback
            # else:
            #     print("Translation failed for: " + pos_ab + "_" + formation)
            #     return ""
        if position == "Right Wing Back" or position == "Left Wing Back":
            fullback = position.split(" ")[0] + " Back" + "_" + formation
            wingback = position + "_" + formation
            if wingback in self.heatmaps:
                return wingback
            elif fullback in self.heatmaps:
                return fullback
            # else:
            #     print("Translation failed for: " + pos_ab + "_" + formation)
            #     return ""
        if position == "Right Midfield" or position == "Left Midfield":
            midfield = position + "_" + formation
            wing = position.split(" ")[0] + " Wing" + "_" + formation
            if midfield in self.heatmaps:
                return midfield
            elif wing in self.heatmaps:
                return wing
            # else:
            #     print("Translation failed for: " + pos_ab + "_" + formation)
            #     return ""
        if position == "Right Wing" or position == "Left Wing":
            midfield = position.split(" ")[0] + " Midfield" + "_" + formation
            wing = position + "_" + formation
            if wing in self.heatmaps:
                return wing
            elif midfield in self.heatmaps:
                return midfield
            # else:
            #     print("Translation failed for: " + pos_ab + "_" + formation)
            #     return ""
        # print("Translation failed for: " + pos_ab + "_" + formation + " returning default...")
        return [heatmap for heatmap in self.heatmaps if position in heatmap][0]


import dataclasses
import logging
import csv
import requests
from typing import List
from urllib.request import urlopen

CSV_URL = 'https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/java-packages.csv'
TRACK1_PACKAGE_PREFIX = 'azure-mgmt-'
TRACK2_PACKAGE_PREFIX = 'azure-resourcemanager-'

MIGRATION_LINK = 'https://github.com/Azure/azure-sdk-for-java/blob/main/sdk/resourcemanager/docs/MIGRATION_GUIDE.md'

CSV_FILENAME = 'deprecate_java_packages.csv'

SERVICE_NAME_MAP = {
    'cosmosdb': 'cosmos',
    'datalake-analytics': 'datalakeanalytics',
    'datalake-store': 'datalakestore',
    'datalake-store-uploader': 'datalakestore',
    'devices': 'deviceprovisioningservices',
    'devtestlab': 'devtestlabs',
    'documentdb': 'cosmos',
    'eventhub': 'eventhubs',
    'features': 'resources',
    'graph-rbac': 'authorization',
    'insights': 'monitor',
    'locks': 'resources',
    'machinelearning': 'machinelearningservices',
    'media': 'mediaservices',
    'dbformariadb': 'mariadb',
    'policy': 'resources',
    'powerbi': 'powerbidedicated',
    'samples': 'resources',
    'scheduler': 'resources',
    'subscriptions': 'resources',
    'traffic-manager': 'trafficmanager',
    'utility': 'resources',
    'website': 'appservice',
    'websites': 'appservice',
}

@dataclasses.dataclass
class Sdk:
    service: str
    package: str
    artifact: str

@dataclasses.dataclass
class SdkInfo:
    service: str
    t1_package: str
    t1_artifact: str
    t2_artifact: str

    def to_row(self) -> List[str]:
        return [self.service, 'Java', self.t1_maven_url(), self.t2_maven_url(), MIGRATION_LINK]

    def t1_maven_url(self):
        return f'https://mvnrepository.com/artifact/{self.t1_package}/{self.t1_artifact}'

    def t2_maven_url(self):
        return f'https://mvnrepository.com/artifact/com.azure.resourcemanager/{self.t2_artifact}'

def process_java_packages_csv() -> List[SdkInfo]:
    logging.info(f'query csv: {CSV_URL}')
    with urlopen(CSV_URL) as csv_response:
        csv_data = csv_response.read()
        csv_str = csv_data.decode('utf-8')
        csv_reader = csv.DictReader(csv_str.split('\n'), delimiter=',', quotechar='"')

    t1_sdks = []
    t2_sdks = {}
    for row in csv_reader:
        package = row['Package']
        namespace = row['GroupId']
        if package.startswith(TRACK1_PACKAGE_PREFIX):
            service = package[len(TRACK1_PACKAGE_PREFIX):]
            if service in SERVICE_NAME_MAP:
                service = SERVICE_NAME_MAP[service]
            sdk = Sdk(service, namespace, package)
            t1_sdks.append(sdk)
        elif package == 'azure':
            service = ''
            sdk = Sdk(service, namespace, package)
            t1_sdks.append(sdk)

        if package.startswith(TRACK2_PACKAGE_PREFIX) and package != TRACK2_PACKAGE_PREFIX + 'parent':
            service = package[len(TRACK2_PACKAGE_PREFIX):]
            sdk = Sdk(service, namespace, package)
            t2_sdks[service] = sdk
        elif package == 'azure-resourcemanager':
            service = ''
            sdk = Sdk(service, namespace, package)
            t2_sdks[service] = sdk

    t1_sdks.sort(key=lambda r: r.service)

    sdk_info_list = []
    for sdk in t1_sdks:
        t2_sdk = None
        if sdk.service in t2_sdks:
            t2_sdk = t2_sdks[sdk.service]
        if not t2_sdk:
            print(sdk.service)

        sdk_info = SdkInfo(sdk.service, sdk.package, sdk.artifact, t2_sdk.artifact if t2_sdk else '')
        sdk_info_list.append(sdk_info)

    # sdk_info_list.append(SdkInfo('', 'com.microsoft.azure', 'azure', 'azure-resourcemanager'))

    return sdk_info_list

def run():
    logging.basicConfig(level=logging.INFO)
    sdk_info_list = process_java_packages_csv()
    write_csv(sdk_info_list)

    validate_maven_packages(sdk_info_list)

def write_csv(sdk_info_list: List[SdkInfo]):
    logging.info(f'write csv: {CSV_FILENAME}')
    with open(CSV_FILENAME, 'w', newline='') as f:
        writer = csv.writer(f, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)

        writer.writerow(['Service', 'Language',
                         'Legacy package (package being replaced)',
                         'New Packages that replace the legacy package on the left',
                         'Migration Guide link'])
        for item in sdk_info_list:
            writer.writerow(item.to_row())

def validate_maven_packages(sdk_info_list: List[SdkInfo]):
    for sdk_info in sdk_info_list:
        check_maven_url(sdk_info.t1_maven_url())
        check_maven_url(sdk_info.t2_maven_url())

def check_maven_url(url: str):
    url = url\
              .replace('mvnrepository.com', 'mvnrepository#com')\
              .replace('.', '/')\
              .replace('mvnrepository#com', 'mvnrepository.com')\
              .replace('https://mvnrepository.com/artifact/', 'https://repo1.maven.org/maven2/') + '/maven-metadata.xml'
    r = requests.get(url)
    logging.info(f'check url {url}, status code {r.status_code}')
    r.raise_for_status()

if __name__ == "__main__":
    run()