#!/usr/bin/env python
# coding: utf-8

# In[174]:

import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import GridSearchCV
from itertools import combinations

# ### Part 1: Get the wine dataset¶
# Describe the dataset
#
# Load in the training and test splits of the dataset

# In[2]:

df_train = pd.read_csv('/Users/eiwi/Practical-ML-DS/Chapter 1. Machine Learning Foundations/DATA/winequality-red-train.csv',index_col=0)
df_test = pd.read_csv('/Users/eiwi/Practical-ML-DS/Chapter 1. Machine Learning Foundations/DATA/winequality-red-test.csv', index_col=0)

# In[3]:

# The charts showing the relationship between each feature and the target
fig, ax = plt.subplots(len(df_train.columns),1, sharey = True, figsize = (18,18))
for i in range(len(df_train.columns)):
    current_col = df_train[df_train.columns[i]]
    ax[i].barh(df_train['quality'], current_col)
    ax[i].set_xlabel(str(df_train.columns[i]))
    ax[i].set_ylabel('quality')
plt.show()

# In[4]:

f, ax = plt.subplots(figsize=(12, 10))
ax = sns.heatmap(df_train.corr(), annot=True)

# In[5]:

print("All the features given in this dataset are numeric.")
print('The most correlated features are: fixed acidity, density and citric acid, total sulfur dioxide and free sulfur dioxide, negatively correlated: fixed acidity and pH, citric and volatile acidity.')

# In[6]:

print('The Quality paremeter is mostly correlated to alcohol (positively) and volatile acidity (negatively).')

# In[7]:

# Separate target and predictors:

# In[8]:

target_train = df_train.pop('quality')
target_test = df_test.pop('quality')

# In[9]:

## Further split train dataset into train and validation sets

X_train, X_validation, y_train, y_validation = train_test_split(df_train, target_train, test_size=0.33, random_state=42)

# In[10]:

# Set the indices in the new datasets as the subquential range of numbers

def reset_indices(df,target):
    lst_idx = []
    for idx,_ in enumerate(df.iloc[:,0]):
        lst_idx.append(idx)
    df['index'] = lst_idx
    df.set_index(df['index'], inplace = True)
    df.drop(columns='index', inplace=True)
    target.index = lst_idx
    return df, target

# In[11]:

X_train, y_train = reset_indices(X_train, y_train)

# In[14]:

X_validation, y_validation = reset_indices(X_validation, y_validation)

# In[15]:

X_test, y_test = reset_indices(df_test, target_test)

# In[16]:

print(len(X_train))
print(len(y_train))
print(len(X_validation))
print(len(y_validation))
print(len(X_test))
print(len(y_test))

# In[17]:

print('The data: train set of {} rows, validation set of {} rows, test set of {} rows.'.format(len(X_train), len(X_validation), len(df_test)), end = '\n')
print('Number of predictors: {}. List of predictors: {}.  Target: {}. Possible classes : {}.'.format(len(X_train.columns), ', '.join(X_train.columns), 'column "quality"', str(set(target_train))), end = '\n')
print('Missing values for train set: {}, for validation set: {}, for test set: {}.'.format(X_train.isna().sum().sum(), X_validation.isna().sum().sum(), df_test.isna().sum().sum()))

# In[18]:

# The data requires scaling due to the difference in units:
X_train.describe()

# In[19]:

# Data scaling
scaler = StandardScaler()
X_train_std = pd.DataFrame(scaler.fit_transform(X_train), columns = X_train.columns)
X_validation_std = pd.DataFrame(scaler.fit_transform(X_validation),  columns = X_validation.columns)
X_test_std = pd.DataFrame(scaler.fit_transform(df_test), columns = df_test.columns)

# In[20]:

fig, ax = plt.subplots(1,2, figsize = (12,8))
ax[0].plot(X_train)
ax[0].set_title('Features before scaling')
ax[1].plot(X_train_std)
ax[1].set_title('Features after scaling')
plt.legend(X_train.columns, loc="lower center", bbox_to_anchor=(0, -0.3),  ncol= 3)
plt.show()

# In[21]:

# Now, the data is ready to be fed to the model.
# I will treat this as a classification problem and will try several basic models to
# get an idea of the most suitable algorithm and hyper parameters.

# ### Part 2: Fit models to the wine dataset and test performance
# Make sure you are comfortable with passing the data through a model.
#
# Evaluate the performance of the model.
#
# Make sure you are testing it on the right set of data.
#

# In[23]:

results_dict = {'model':[],
                'params':[],
                'train_score':[],
                'validation_score':[]
               }

# In[24]:

from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.neural_network import MLPClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier

models_lst = [
    LogisticRegression(random_state=42),
    SVC(gamma=2, random_state=42),
    MLPClassifier(max_iter=1000, random_state=42),
    KNeighborsClassifier(n_jobs=-1),
    DecisionTreeClassifier(random_state=42),
   ]

# In[25]:

params = [
    {'penalty':('l1','l2', 'elasticnet', 'none'), 'C':(0.025, 0.01,1,10), 'solver': ('newton-cg', 'lbfgs', 'liblinear','sag','saga')},
    {'C':(0.025, 0.01,1,10), 'kernel':('linear', 'rbf'), 'decision_function_shape':('ovr','ovo')},
    {'activation':('relu', 'logistic'), 'solver':('sgd','lbfgs'), 'alpha':(0.001,0.01,0.1,1), 'learning_rate': ('adaptive','constant')},
    {'n_neighbors':(3,5,10), 'weights':('distance','uniform'), 'algorithm':('auto','ball_tree','kd_tree')},
    {'criterion':('gini','entropy'), 'splitter':('best','random'),'max_depth':(5,10,25), 'min_samples_split':(2,3,5)}
]

# In[26]:

for i in range(len(models_lst)):
    model = models_lst[i]
    parameters = params[i]
    grid_search_model = GridSearchCV(model, parameters)
    grid_search_model.fit(X_train,y_train)
    train_score =  grid_search_model.best_score_
    validation_score = grid_search_model.best_estimator_.score(X_validation,y_validation)
    results_dict['model'].append(grid_search_model.best_estimator_)
    results_dict['params'].append(grid_search_model.best_params_)
    results_dict['train_score'].append(train_score)
    results_dict['validation_score'].append(validation_score)

# In[27]:

results_dict

# In[28]:

classifiers_comparison = pd.DataFrame.from_dict(results_dict)

# In[29]:

classifiers_comparison.to_csv('/Users/eiwi/Practical-ML-DS/Chapter 1. Machine Learning Foundations/PROJECT/classifiers_comparison.csv')

# In[30]:

classifiers_comparison

# In[31]:

best_train_score = classifiers_comparison['train_score'].max()
print('best_train_score: ', best_train_score)
print('best_train_score_model: ', classifiers_comparison['model'][classifiers_comparison['train_score']==best_train_score], end = '\n')

print('\n')

second_best_train_score = list(classifiers_comparison['train_score'].sort_values(ascending = False))[1]
print('second_best_train_score: ', second_best_train_score)
print('second_best_train_score_model: ', classifiers_comparison['model'][classifiers_comparison['train_score']==second_best_train_score], end = '\n')

print('\n')

best_validation_score = classifiers_comparison['validation_score'].max()
print('best_validation_score: ', best_validation_score)
print('best_validation_score_model: ', classifiers_comparison['model'][classifiers_comparison['validation_score']==best_validation_score], end = '\n')

print('\n')

second_best_validation_score = list(classifiers_comparison['validation_score'].sort_values(ascending = False))[1]
print('second_best_validation_score: ', second_best_validation_score)
print('second_best_validation_score_model: ', classifiers_comparison['model'][classifiers_comparison['validation_score']==second_best_validation_score], end = '\n')

# In[32]:

print('So far the best models were {} and {}, so I will try to combine them into an ensemble.'.format('DecisionTreeClassifier', 'KNeighborsClassifier' ))

# ### Part 3: Improve your performance by ensembling some of your models
# Combine the results of more than one model
#
# Evaluate the performance of the ensemble

# In[138]:

from sklearn.ensemble import VotingClassifier, StackingClassifier, AdaBoostClassifier, BaggingClassifier, RandomForestClassifier

# In[139]:

ensembles_results = {
    'ensemble':[],
    'train_score':[],
    'validation_score':[]
}

# In[140]:

# First, I will combine DecisionTreeClassifier + KNeighborsClassifier
# using ensemble called VotingClassifier a) with hard vote b) with soft vote

# In[141]:

# a) VotingClassifier with a hard vote: the classifier uses
# predicted class labels for majority rule voting.

# In[142]:

knn_estimator =  KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',
                       metric_params=None, n_jobs=-1, n_neighbors=10, p=2)

dtc_estimator = DecisionTreeClassifier(ccp_alpha=0.0, class_weight=None, criterion='entropy',
                         max_depth=25, max_features=None, max_leaf_nodes=None,
                         min_impurity_decrease=0.0, min_impurity_split=None,
                         min_samples_leaf=1, min_samples_split=3,
                         min_weight_fraction_leaf=0.0,splitter='best')

# In[143]:

hard_voting_ensemble = VotingClassifier(estimators=[('knn_estimator',knn_estimator), ('dtc_estimator',dtc_estimator)],
                                   voting='hard')

# In[144]:

hard_voting_ensemble.fit(X_train, y_train)

ensembles_results['ensemble'].append('VotingClassifier_hard')
ensembles_results['train_score'].append(hard_voting_ensemble.score(X_train, y_train))
ensembles_results['validation_score'].append(hard_voting_ensemble.score(X_validation,y_validation))

# In[145]:

# b) VotingClassifier with a soft vote: predicts the class label based on the argmax
# of the sums of the predicted probabilities

# In[146]:

soft_voting_ensemble = VotingClassifier(estimators=[('knn_estimator',knn_estimator), ('dtc_estimator',dtc_estimator)],
                                   voting='soft')

# In[147]:

soft_voting_ensemble.fit(X_train, y_train)

ensembles_results['ensemble'].append('VotingClassifier_soft')
ensembles_results['train_score'].append(soft_voting_ensemble.score(X_train, y_train))
ensembles_results['validation_score'].append(soft_voting_ensemble.score(X_validation,y_validation))

# In[148]:

# Secondly, I will use StackingClassifier with the same DecisionTreeClassifier + KNeighborsClassifier

stacking_ensemble = StackingClassifier(estimators=[('knn_estimator',knn_estimator), ('dtc_estimator',dtc_estimator)])

stacking_ensemble.fit(X_train, y_train)

ensembles_results['ensemble'].append('StackingClassifier')
ensembles_results['train_score'].append(stacking_ensemble.score(X_train, y_train))
ensembles_results['validation_score'].append(stacking_ensemble.score(X_validation,y_validation))

# In[149]:

# Compare StackingClassifier with final AdaBoostClassifier
ada_stacking_ensemble = StackingClassifier(estimators=[('knn_estimator',knn_estimator),
                                                       ('dtc_estimator',dtc_estimator)],
                                           final_estimator=None,
                                           stack_method='auto',
                                           n_jobs=-1)

ada_stacking_ensemble.fit(X_train, y_train)

ensembles_results['ensemble'].append('StackingClassifier_AdaBoost')
ensembles_results['train_score'].append(ada_stacking_ensemble.score(X_train, y_train))
ensembles_results['validation_score'].append(ada_stacking_ensemble.score(X_validation,y_validation))

# In[150]:

# Then compare ensemble perfomance with a BaggingClassifier
# using multiple KNeighborsClassifiers OR DecisionTreeClassifiers

# In[151]:

# a) BaggingClassifier with DecisionTreeClassifier

# In[152]:

dtc_bagging_ensemble = BaggingClassifier(base_estimator = dtc_estimator,
                                         n_estimators=100,
                                         random_state=42)

dtc_bagging_ensemble.fit(X_train, y_train)

# In[153]:

ensembles_results['ensemble'].append('DTC_BaggingClassifier')
ensembles_results['train_score'].append(dtc_bagging_ensemble.score(X_train, y_train))
ensembles_results['validation_score'].append(dtc_bagging_ensemble.score(X_validation,y_validation))

# In[154]:

# b) BaggingClassifier with KNeighborsClassifiers

# In[155]:

knn_bagging_ensemble = BaggingClassifier(base_estimator=knn_estimator,
                                     n_estimators=100,
                                     random_state=42)

# In[156]:

knn_bagging_ensemble.fit(X_train, y_train)

ensembles_results['ensemble'].append('KNN_BaggingClassifier')
ensembles_results['train_score'].append(knn_bagging_ensemble.score(X_train, y_train))
ensembles_results['validation_score'].append(knn_bagging_ensemble.score(X_validation,y_validation))

# In[157]:

# As DecisionTreeclassifier was the most cussessful in previous ensembles,
# now I will try out RandomForestClassifier

# In[158]:

rfc_ensemble = RandomForestClassifier(n_estimators=100, random_state=0)
rfc_ensemble.base_estimator = dtc_estimator

# In[159]:

rfc_ensemble.fit(X_train, y_train)

ensembles_results['ensemble'].append('RandomForestClassifier')
ensembles_results['train_score'].append(rfc_ensemble.score(X_train, y_train))
ensembles_results['validation_score'].append(rfc_ensemble.score(X_validation,y_validation))

# In[160]:

#Compare with AdaBoostClassifier based on DecisionTreeClassifiers only
# (as KNeighborsClassifier doesn't support sample_weight)

# In[161]:

dtc_ada_ensemble = AdaBoostClassifier(base_estimator= dtc_estimator, n_estimators=50,
                                      learning_rate=1.0, algorithm='SAMME.R',
                                      random_state=42)
dtc_ada_ensemble.fit(X_train, y_train)

ensembles_results['ensemble'].append('AdaBoost_dtc_ensemble')
ensembles_results['train_score'].append(dtc_ada_ensemble.score(X_train, y_train))
ensembles_results['validation_score'].append(dtc_ada_ensemble.score(X_validation, y_validation))

# In[162]:

# Now let's compare these ensembles

# In[163]:

ensembles_results

# In[164]:

ensembles_comparison = pd.DataFrame.from_dict(ensembles_results)

# In[165]:

ensembles_comparison

# In[166]:

# Conclusion: the highest validation score of 0.822695 was achieved with DTC_BaggingClassifier
# The second best score of 0.817967 was achieved with RandomForestClassifier
# Surprisingly, VotingClassifier and StackingClassifier combining different base models
# have worse perfomanse than ensembles with 1 estimator (RandomForestClassifier, DTC_BaggingClassifier)

# In[362]:

dtc_ada_ensemble.__class__.__name__

# In[324]:

lst = list(combinations([3,6,7], 2))
for t in lst:
    print(list(t))

(1,2)

# ### Part 4: Write an algorithm to find the best combination of models and hyperparameters.
#
# There are an infinite number of ways that you could combine different models with different hyperparameters, but some will perform better than others.
#
# List the different parameters which you test over, as well as the ranges which you test
#
# Describe the search procedure which your algorithm implements

# In[484]:

from sklearn.model_selection import cross_val_score

# In[533]:

def algo_name(est):
    return est.__class__.__name__
def algo_voting_soft_classifier(estimators_list, param_grid_list, N, X, y, scoring='f1'):
    """
        This function searches for best combination of model and hyperparameters to be used in VotingClasifier(soft)

        estimators_list — list of non-fitted estimators
        param_grid_list — list of param grids for GridSearch, must be same size as estimators_list
        N — number of models to use in ensemble
        X — features
        y — target
        scoring — scoring function name (string)
    """

    # Lists must be of same size
    assert len(estimators_list) == len(param_grid_list)

    # Can't use more estimators than we have
    assert N <= len(estimators_list)

    # We will keep list of best fitted estimators
    best_estimators = []

    # Let's GridSearch em all
    for estimator, param_grid in zip(estimators_list, param_grid_list):
        print("Searching best params for " + algo_name(estimator))
        gs = GridSearchCV(estimator, param_grid, scoring=scoring)
        gs.fit(X,y)
        best_estimators.append(gs.best_estimator_)
        print("Best score = " + str(gs.best_score_))

    # Here we have all possible combinations of N estimators
    all_combinations = list(combinations(best_estimators, N))

    # Keep track of best ensemble yet
    top_score = 0
    top_ensemble = None

    # Go over all combinations
    for t in all_combinations:
        voting_estimators = []
        voting_estimators_names = []

        # Construct list of estimators for VotingClassifier
        for estimator in list(t):
            name = algo_name(estimator)
            voting_estimators_names.append(name)
            voting_estimators.append((name, estimator))

        print(f"\n\nChecking combination of {N} estimators:")
        print("\t\n".join(voting_estimators_names))
        # Cross-validating ensemble of N estimators
        soft_voting_ensemble = VotingClassifier(estimators=voting_estimators, voting='soft')
        cross_vals = cross_val_score(soft_voting_ensemble, X,y, scoring=scoring)

        # Selecting top ensemble
        current_score = max(cross_vals)

        print(f"Score ({scoring}) of this combination is {current_score}")
        if (current_score>top_score):
            top_score = current_score
            top_ensemble = soft_voting_ensemble

    return top_ensemble

# In[534]:

estimators_list = [
    LogisticRegression(random_state=42, n_jobs=-1),
    SVC(gamma=2, random_state=42, probability=True),
#     MLPClassifier(max_iter=1000, random_state=42),
#     KNeighborsClassifier(n_jobs=-1),
    DecisionTreeClassifier(random_state=42)
]

param_grids_list = [
    {'penalty':('l1','l2', 'elasticnet', 'none'), 'C':(0.025, 0.01,1,10), 'solver': ('newton-cg', 'lbfgs', 'liblinear','sag','saga')},
    {'C':(0.025, 0.01,1,10), 'kernel':('linear', 'rbf'), 'decision_function_shape':('ovr','ovo')},
#     {'activation':('relu', 'logistic'), 'solver':('sgd','lbfgs'), 'alpha':(0.001,0.01,0.1,1), 'learning_rate': ('adaptive','constant')},
#     {'n_neighbors':(3,5,10), 'weights':('distance','uniform'), 'algorithm':('auto','ball_tree','kd_tree')},
    {'criterion':('gini','entropy'), 'splitter':('best','random'),'max_depth':(5,10,25,50), 'min_samples_split':(2,3,5,10)}
]

# In[535]:

demo_X = X_train[:100]
demo_y = y_train[:100]

# In[536]:

import warnings
warnings.filterwarnings("ignore")
best_voter = algo_voting_soft_classifier(estimators_list, param_grids_list, 2, demo_X, demo_y, scoring='accuracy')
warnings.filterwarnings("default")

# In[518]:

best_voter

# In[519]:

warnings.filterwarnings("ignore")
best_voter_3 = algo_voting_soft_classifier(estimators_list, param_grids_list, 3, demo_X, demo_y, scoring='accuracy')
warnings.filterwarnings("default")

# In[520]:

best_voter_3

# ### Part 5: Present your results
#
# The final part of the project requires you to attempt to summarise your work. It should be clear how we could replicate the results by implementing exactly the same ensemble (models and hyperparameters).
#
# Please try to communicate and display your results with any graphs or charts.
# If you have any insights into why certain ensembles or models perform better of worse than others, and would like to write a paragraph to explain this, we'd love to read it!
# Please also write a summary paragraph that describes the best permutation that you found.

# In[ ]:

# Based on my observation, the individual models  show lower score than ensembles

# In[676]:

print( 'The average score among individual models is equal to {}, maximum validation score - {} (provided by {}).'.format(round(classifiers_comparison['validation_score'].mean(),3), round(classifiers_comparison['validation_score'],3).max(), str(classifiers_comparison[classifiers_comparison['validation_score'] == classifiers_comparison['validation_score'].max()]['model'])[5:27]))

# In[710]:

print('The average score among ensembles is equal to {}, maximum validation score - {} (provided by {}).'.format(round(ensembles_comparison['validation_score'].mean(),3),
                        round(ensembles_comparison['validation_score'].max(),3),
                        str(ensembles_comparison[ensembles_comparison['validation_score'] == ensembles_comparison['validation_score'].max()]['ensemble'])[5:27]))

# In[ ]:

# The most powerful individual models are DecisionTreeClassifier and KNeighborsClassifier.
# The most powerful ensembles are DTC_BaggingClassifier and RandomForestClassifier (both are based on DecisionTreeClassifier)

# In[ ]:

# Single models VS ensembles

# In[718]:

fig, ax = plt.subplots(1, 2, figsize = (16,6), sharey = True)

ax[0].bar(classifiers_comparison.index, classifiers_comparison['train_score'], width = -0.3, align = 'edge', color='navy')
ax[0].bar(classifiers_comparison.index, classifiers_comparison['validation_score'], width = 0.3, align = 'edge', color='orange')
ax[0].set_title('Score of the individual models')

ax[1].bar(ensembles_comparison.index, ensembles_comparison['train_score'], width = -0.3, align = 'edge', color='navy')
ax[1].bar(ensembles_comparison.index, ensembles_comparison['validation_score'], width = 0.3, align = 'edge' , color='orange' )
ax[1].set_title('Score of the ensembles')

plt.title(label = 'Difference in score between individual models and ensembles', loc='left')
plt.show()

# In[719]:

# Interestingly, while individual models tend to show slightly higher score on the validation set,
# while ensembles tend to do the opposite: their train score is higher than validation one.

# In[720]:

# It is also interesting to compare the ensembles combining the different models VS homogeneous ones

# In[727]:

ensembles_comparison

# In[725]:

fig, ax = plt.subplots()
ax.set_title('Ensembles score')
ax.bar(ensembles_comparison.index, ensembles_comparison['train_score'], width = -0.3, align = 'edge', color='navy')
ax.bar(ensembles_comparison.index, ensembles_comparison['validation_score'], width = 0.3, align = 'edge' , color='orange')
ax.set_ylabel('Score')
ax.set_xlabel('Ensemble')

# In[ ]:

# After performing a series of test, I made a conclusion that the best possible score can be achieved with
# the ensemble methods that combine multiple versions of the same estimator (BaggingClassifier, RandomForestClassifier, ) rather than
# the ones combining a list of differents estimators (VotingClassifier and StackingClassifier).

# In[ ]:

# So for the further iteartions I would use BaggingClassifier or RandomForestClassifier.
# The key for reproducibility the would be to stick to the repeatable method for every step of data analysis:
# so it is reasonable to create the standard functions for data processing and cleaning, scaling and modelling.
# Also, in order to replicate the results I would pass an argument 'random_state = 42' to the models
# which allows to get the same results while running the model again.

# ### Part 6:
# A stakeholder asks you which features most affect the response variable (output). Describe how you would organise a test to determine this.

# In[521]:

# I would consider the influence of each feature to the output as a change that occurs in target value
# with change of the output with one unit

# In[525]:

get_ipython().system("['Approach for features importance calculation']('Change_in_var.png')")

# In[745]:

# I would test this approach on desicion-tree based model in order to compare the results with the built function:
feature_importances = list(dtc_estimator.fit(demo_X, demo_y).feature_importances_)
features = demo_X.columns

# In[757]:

feat_imp = pd.DataFrame()
feat_imp['features'] = features
feat_imp['feature_importances'] = feature_importances

# In[762]:

feat_imp.sort_values(by = 'feature_importances', ascending = False)

# In[ ]:

# The inbuilt function suggests that alcohol has the strongest effect on the target (quality of  wine).


import json
import logging

from twitterauth.session import Session
from twitterauth.test import APITestCase
from twitterauth.configs import settings
from twitterauth.utils import helper
from twitterauth.utils import payload

LOGGER = logging.getLogger('twitter')
grant_type_missing_err_msg = 'Missing required parameter: grant_type'
invalid_creds_err_msg = 'Unable to verify your credentials'
invalid_grant_type_err_msg = 'invalid_grant_type_value parameter is invalid'

class TestTwitterAuthAPIInputValidation(APITestCase):

    @classmethod
    def setUpClass(self):
        self.session = Session().get_session()
        self.base_url = settings.api.url

    def test_oauth_without_grant_type(self):
        """
        Verify twitter oauth without grant_type
        """
        headers = payload.get_oauth_headers(helper.getBase64Value())

        # Get bearer token using /oauth2/token
        response = self.session.post(self.base_url + "/oauth2/token",
                                     headers=headers)

        # Verify error status code and error message
        assert response.status_code == 403
        LOGGER.info(response.status_code)
        parsed_response = json.loads(response.text)
        assert parsed_response["errors"][0]["message"] == \
               grant_type_missing_err_msg

    def test_oauth_with_invalid_grant_type(self):
        """
        Verify twitter oauth with invalid grant_type
        """
        headers = payload.get_oauth_headers(helper.getBase64Value())
        data = payload.get_oauth_data(grant_type="invalid_grant_type_value")

        # Get bearer token using /oauth2/token
        response = self.session.post(self.base_url + "/oauth2/token",
                                     data=data,
                                     headers=headers)

        # Verify error status code and error message
        assert response.status_code == 400
        LOGGER.info(response.status_code)
        parsed_response = json.loads(response.text)
        assert parsed_response["errors"][0]["message"] == \
               invalid_grant_type_err_msg

    def test_oauth_without_content_type(self):
        """
        Verify twitter oauth without content_type
        """
        headers = payload.get_oauth_headers(helper.getBase64Value(),
                                            content_type="")

        # Get bearer token using /oauth2/token
        response = self.session.post(self.base_url + "/oauth2/token",
                                     headers=headers)

        # Verify error status code
        assert response.status_code == 403
        LOGGER.info(response.status_code)

    def test_oauth_without_authorization(self):
        """
        Verify twitter oauth without authorization
        """
        headers = payload.get_oauth_headers(helper.getBase64Value())
        data = payload.get_oauth_data()

        headers.pop('Authorization', None)

        # Get bearer token using /oauth2/token
        response = self.session.post(self.base_url + "/oauth2/token",
                                     data=data,
                                     headers=headers)
        assert response.status_code == 403
        LOGGER.info(response.text)

        # Verify error message in response
        parsed_response = json.loads(response.text)
        assert parsed_response["errors"][0]["message"] == \
               invalid_creds_err_msg

    def test_oauth_invalid_url(self):
        """
        Verify twitter oauth with invalid oauth url
        """
        headers = payload.get_oauth_headers(helper.getBase64Value())
        data = payload.get_oauth_data()

        # Get bearer token using /oauth2/token
        response = self.session.post(self.base_url + "/oauth2",
                                     data=data,
                                     headers=headers)
        assert response.status_code == 404


import argparse
import csv

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument("firstCol", help = "Number of the first column to match", type=int)
    parser.add_argument("secondCol", help = "Number of the second column to match, multiple content", type=int)
    parser.add_argument("-b", "--fromBeginning", help="Whether full csv", action="store_false")
    parser.add_argument("path", help="Path to the file")
    parser.add_argument("name", help="Name of the file to save")
    args = parser.parse_args()

    return args.firstCol, args.secondCol, args.fromBeginning, args.path, args.name

def open_csv(pathname, fromBeginning):
    with open(pathname, newline = '') as csfile:
        reader = csv.reader(csfile, delimiter=',')
        index = 1 if fromBeginning else 0
        result = [row for row in list(reader)[index:]]

        return result

def create_list_for_table_with_duplicates(csfile, col1, col2):
    result = []

    for row in csfile:
        temp = [row[col1], row[col2]]
        result.append(temp)

    doubled = []

    for row in result:
        res_col_2 = row[1].split(", ")

        for tinyRow in res_col_2:
            doubled.append([row[0], tinyRow])

    return doubled

def save_csv(result, name):
    with open(name + ".csv", 'w') as opcsv:
        writer = csv.writer(opcsv)
        writer.writerows(result)

if __name__ == '__main__':
    first, second, begin, path, name = parse_args()
    res = open_csv(path, begin)
    result = create_list_for_table_with_duplicates(res, first, second)
    print(result)
    save_csv(result, name)


# This program maps IPs to the ports open on them

with open("results.txt",'r') as f:
    lines = f.readlines()
count = 0
mapping = {}
for line in lines:

    if "Nmap scan report for" in line:
        target = count
        ports = []
        while("_____" not in lines[target]):
            if "/" in lines[target] and "open" in lines[target]:
                ports.append(lines[target].split()[2])
            target+=1
        if "(" in line:
            mapping[line.split("(")[1].strip(")\n")] = ports
print(mapping)
print(count)


from citations import *
import sys, os

if __name__ == "__main__":

    if len(sys.argv) < 2:
        print("È necessario indicare come argomento export.xslx, il file Excel ottenuto")
        print("esportando le proprie pubblicazioni seguendo questi passi:")
        print("")
        print(" 1. Cercarsi su https://arpi.unipi.it/browse?type=author")
        print(" 2. Selezionare un numero sufficiente di risultati per pagina per vedere tutte le proprie pubblicazioni")
        print(" 3. Cliccare su Esportazione > Excel")
        print("")
        print("P.S.: Conviene ordinare le pubblicazioni in modo decrescente, così da non escludere quelle rilevanti per la VQR")
        print("anche nel caso se ne abbiano > 100")

        sys.exit(1)

    years = [ 2015, 2016, 2017, 2018, 2019 ]

    class_names = [
        "-",
        "A",
        "B",
        "C",
        "D",
        "E",
    ]

    print("Inserire il settore (e.g, MAT01, ..., MAT08): ", end = "")
    sector = input()

    db_mcq = { year: db for (year, db) in zip(
        years,
        map(lambda x : get_journals_classification(x, "scopus", sector, "MCQ-SCOPUS", "article_MCQ"), years)
    ) }

    db_sjr = { year: db for (year, db) in zip(
        years,
        map(lambda x : get_journals_classification(x, "scopus", sector, "SJR-SNIP", "article_SJR"), years)
    ) }

    db_snip = { year: db for (year, db) in zip(
        years,
        map(lambda x : get_journals_classification(x, "scopus", sector, "SJR-SNIP", "article_SNIP"), years)
    ) }

    db_wos = { year: db for (year, db) in zip(
        years,
        map(lambda x : get_journals_classification(x, "wos", sector, "MCQ-WOS", "article_MCQ"), years)
    ) }

    data = pandas.read_excel(sys.argv[1])

    # We prepare the data for the output
    out = {
      'Titolo': [],
      'MCQ': [],
      'SJR': [],
      'SNIP': [],
      'Anno': [],
      'Rivista': [],
      '# citazioni': [],
      '# autocitazioni': [],
      'Autori': []
    }

    print("\033[1mMCQ SJR SNIP WOS Titolo                                   Anno  #cit. #autocit. #cit.WOS Rivista                  Autori \033[0m")

    for j in range(len(data)):
        scopus_id = data["Identificativo Scopus"][j]
        wos_id = data["Identificativo WOS"][j]
        year = data["Data pubblicazione"][j]

        journal = data["Rivista"][j]

        if year < 2015 or year >= 2020 or not isinstance(journal, str):
            continue
        else:
            journal = journal.lower()

        (cit, self_cit) = get_scopus_citations(scopus_id)
        (cit_wos, self_cit_wos) = get_wos_citations(wos_id)
        cl_mcq = get_classification(journal, cit, db_mcq[year])
        cl_sjr = get_classification(journal, cit, db_sjr[year])
        cl_snip = get_classification(journal, cit, db_snip[year])
        cl_wos = get_classification(journal, cit_wos, db_wos[year])

        # Se le autocitazioni superano il 50% formattiamo in rosso
        if self_cit >= cit / 2:
          self_cit_s = "\033[31;1m" + str(self_cit).rjust(8) + "\033[0m"
        else:
          self_cit_s = str(self_cit).rjust(8)

        if self_cit_wos >= cit_wos / 2:
          self_cit_s_wos = "\033[31;1m" + str(self_cit_wos).rjust(8) + "\033[0m"
        else:
          self_cit_s_wos = str(self_cit_wos).rjust(8)

        class_mcq = class_names[cl_mcq+1]
        class_sjr = class_names[cl_sjr+1]
        class_snip = class_names[cl_snip+1]
        class_wos = class_names[cl_wos+1]

        print("%s   %s   %s    %s   %s  %s  %s  %s  %s %s  %s" % (
          format_class(class_mcq), format_class(class_sjr), format_class(class_snip),
          format_class(class_wos),
          truncate(data["Titolo"][j], 40), year, str(cit).rjust(4), self_cit_s,
          str(cit_wos).rjust(8),
          truncate(journal, 24), data['Autori'][j])
        )

        out['Titolo'].append(data["Titolo"][j])
        out['Anno'].append(year)
        out['# citazioni'].append(cit)
        out['# autocitazioni'].append(self_cit)
        out['Rivista'].append(journal)
        out['Autori'].append(data['Autori'][j])
        out['MCQ'].append(class_mcq)
        out['SJR'].append(class_sjr)
        out['SNIP'].append(class_snip)

    df = pandas.DataFrame(data = out)

    print("")
    if os.path.exists('output.xlsx'):
      print("Il file output.xlsx esiste già, sovrascriverlo? [yn] ", end = '')
      ans = input()

      if ans.lower() != 'y':
        sys.exit(0)

    df.to_excel("output.xlsx")
    print("> I dati sono stati salvati in output.xlsx")


import pandas as pd
import numpy as np
import pickle
import torch
import random
import os
import sys
from pathlib import Path
sys.path.append(os.path.dirname(os.path.abspath(__file__)) + './../..')

# MAX_LEN=12
# MAX_COND_SEQ=56
# MAX_PROC_SEQ=40
# MAX_MED_SEQ=15#37
# MAX_LAB_SEQ=899
# MAX_BMI_SEQ=118

def create_vocab(file):
    with open ('./data/dict/'+file, 'rb') as fp:
        condVocab = pickle.load(fp)
    condVocabDict={}
    condVocabDict[0]=0
    for val in range(len(condVocab)):
        condVocabDict[condVocab[val]]= val+1

    return condVocabDict

def gender_vocab():
    genderVocabDict={}
    genderVocabDict['<PAD>']=0
    genderVocabDict['M']=1
    genderVocabDict['F']=2

    return genderVocabDict

def create_batches(batch_size,chart_flag):
    with open ('./data/dict/'+'hadmDic', 'rb') as fp:
        hids = pickle.load(fp)

    batchDict={}
    with open ('./data/dict/'+'dataDic', 'rb') as fp:
        dataDic = pickle.load(fp)
    if chart_flag:
        batchChartDict={}
        with open ('./data/dict/'+'dataChartDic', 'rb') as fp:
            dataChartDic = pickle.load(fp)

    batch_idx=0
    ids=range(0,len(hids))
    for i in range(0,int(len(hids)/batch_size)):
        rids=random.sample(ids, batch_size)
        ids=list(set(ids)-set(rids))
        batch_hids=hids[rids]
        batchDict[batch_idx]=dict((k, dataDic[k]) for k in batch_hids)
        if chart_flag:
            batchChartDict[batch_idx]=dict((k, dataChartDic[k]) for k in batch_hids)
        batch_idx=batch_idx+1
    if chart_flag:
        return batchDict,batchChartDict
    else:
        return batchDict

def init(diag_flag,proc_flag,out_flag,chart_flag,med_flag,lab_flag):
        condVocabDict={}
        procVocabDict={}
        medVocabDict={}
        outVocabDict={}
        chartVocabDict={}
        labVocabDict={}
        ethVocabDict={}
        ageVocabDict={}
        genderVocabDict={}
        insVocabDict={}

        ethVocabDict=create_vocab('ethVocab')
        with open('./data/dict/'+'ethVocabDict', 'wb') as fp:
            pickle.dump(ethVocabDict, fp)

        ageVocabDict=create_vocab('ageVocab')
        with open('./data/dict/'+'ageVocabDict', 'wb') as fp:
            pickle.dump(ageVocabDict, fp)

        genderVocabDict=gender_vocab()
        with open('./data/dict/'+'genderVocabDict', 'wb') as fp:
            pickle.dump(genderVocabDict, fp)

        insVocabDict=create_vocab('insVocab')
        with open('./data/dict/'+'insVocabDict', 'wb') as fp:
            pickle.dump(insVocabDict, fp)

        if diag_flag:
            file='condVocab'
            with open ('./data/dict/'+file, 'rb') as fp:
                condVocabDict = pickle.load(fp)
        if proc_flag:
            file='procVocab'
            with open ('./data/dict/'+file, 'rb') as fp:
                procVocabDict = pickle.load(fp)
        if med_flag:
            file='medVocab'
            with open ('./data/dict/'+file, 'rb') as fp:
                medVocabDict = pickle.load(fp)
        if out_flag:
            file='outVocab'
            with open ('./data/dict/'+file, 'rb') as fp:
                outVocabDict = pickle.load(fp)
        if chart_flag:
            file='chartVocab'
            with open ('./data/dict/'+file, 'rb') as fp:
                chartVocabDict = pickle.load(fp)
        if lab_flag:
            file='labsVocab'
            with open ('./data/dict/'+file, 'rb') as fp:
                labVocabDict = pickle.load(fp)

        return len(condVocabDict),len(procVocabDict),len(medVocabDict),len(outVocabDict),len(chartVocabDict),len(labVocabDict),ethVocabDict,genderVocabDict,ageVocabDict,insVocabDict

def init_read(batch_size,diag_flag,proc_flag,out_flag,chart_flag,med_flag,lab_flag):
        condVocabDict={}
        procVocabDict={}
        medVocabDict={}
        outVocabDict={}
        chartVocabDict={}
        labVocabDict={}
        ethVocabDict={}
        ageVocabDict={}
        genderVocabDict={}
        insVocabDict={}

        with open('./data/dict/'+'ethVocabDict', 'rb') as fp:
            ethVocabDict= pickle.load(fp)

        with open('./data/dict/'+'ageVocabDict', 'rb') as fp:
            ageVocabDict= pickle.load(fp)

        with open('./data/dict/'+'genderVocabDict', 'rb') as fp:
            genderVocabDict= pickle.load(fp)

        with open('./data/dict/'+'insVocabDict', 'rb') as fp:
            insVocabDict= pickle.load(fp)

        if diag_flag:
            file='condVocab'
            with open ('./data/dict/'+file, 'rb') as fp:
                condVocabDict = pickle.load(fp)
        if proc_flag:
            file='procVocab'
            with open ('./data/dict/'+file, 'rb') as fp:
                procVocabDict = pickle.load(fp)
        if med_flag:
            file='medVocab'
            with open ('./data/dict/'+file, 'rb') as fp:
                medVocabDict = pickle.load(fp)
        if out_flag:
            file='outVocab'
            with open ('./data/dict/'+file, 'rb') as fp:
                outVocabDict = pickle.load(fp)
        if chart_flag:
            file='chartVocab'
            with open ('./data/dict/'+file, 'rb') as fp:
                chartVocabDict = pickle.load(fp)
        if lab_flag:
            file='labsVocab'
            with open ('./data/dict/'+file, 'rb') as fp:
                labVocabDict = pickle.load(fp)

        return len(condVocabDict),len(procVocabDict),len(medVocabDict),len(outVocabDict),len(chartVocabDict),len(labVocabDict),ethVocabDict,genderVocabDict,ageVocabDict,insVocabDict

def get_meta():
    with open ('./data/dict/'+'metaDic', 'rb') as fp:
        meta = pickle.load(fp)
    if "Lab" in meta.keys():
        return meta['Med'],meta['Cond'],meta['Proc'],0,0,meta['Lab']
    else:
        return meta['Med'],meta['Cond'],meta['Proc'],meta['Out'],meta['Chart'],0

def get_batches():
    with open ('./data/dict/'+'batchDict', 'rb') as fp:
        batchDict = pickle.load(fp)
    return batchDict

def get_batch_data(key,data,diag_flag,proc_flag,out_flag,chart_flag,med_flag,lab_flag):
    conds=[]
    procs=[]
    meds=[]
    meds_rate=[]
    meds_amount=[]
    outs=[]
    charts=[]
    charts_val=[]
    labs=[]
    labs_val=[]
    labels=[]
    eth=[]
    age=[]
    gender=[]

    with open ('./data/dict/'+'metaDic', 'rb') as fp:
        meta = pickle.load(fp)
    if diag_flag:
        with open ('./data/dict/'+'condVocabDict', 'rb') as fp:
            condVocabDict = pickle.load(fp)
    if proc_flag:
        with open ('./data/dict/'+'procVocabDict', 'rb') as fp:
            procVocabDict = pickle.load(fp)
    if med_flag:
        with open ('./data/dict/'+'medVocabDict', 'rb') as fp:
            medVocabDict = pickle.load(fp)
    if out_flag:
        with open ('./data/dict/'+'outVocabDict', 'rb') as fp:
            outVocabDict = pickle.load(fp)
    if lab_flag:
        with open ('./data/dict/'+'labVocabDict', 'rb') as fp:
            labVocabDict = pickle.load(fp)

    with open ('./data/dict/'+'ethVocabDict', 'rb') as fp:
        ethVocabDict = pickle.load(fp)
    with open ('./data/dict/'+'genderVocabDict', 'rb') as fp:
        genderVocabDict = pickle.load(fp)
    with open ('./data/dict/'+'ageVocabDict', 'rb') as fp:
        ageVocabDict = pickle.load(fp)

    if chart_flag:
        with open ('./data/dict/'+'chartVocabDict', 'rb') as fp:
            chartVocabDict = pickle.load(fp)
        with open ('./data/dict/'+'batchChartDict', 'rb') as fp:
            batchChartDict = pickle.load(fp)
            batchChartDict=batchChartDict[key]

        for hid, hid_data in batchChartDict.items():
        #print(hid)
            for feature, feat_data in hid_data.items():
                #print(feature)
                if feature=='Chart':
                    #print(list(feat_data.keys()))
                    fids=list(map(chartVocabDict.get, list(feat_data['signal'].keys())))
                    fids_pad=list(np.zeros(meta['Chart']))
                    fids_pad[0:len(fids)]=fids
                    #fids=list(pd.Series(feat_data.keys()).map(medVocabDict))
                    #print(fids)
                    #meds.append(fids_pad)
                    chart_len=list(feat_data['signal'].values())
                    chart_pad=np.asarray(fids_pad)
                    chart_pad=np.expand_dims(chart_pad, axis=1)
                    #print(med_pad)
                    zeros = [ [0] * meta['LOS'] for _ in range(meta['Chart'])]
                    zeros[0:len(chart_len)]=chart_len
                    #print(zeros)
                    zeros = chart_pad * np.asarray(zeros)
                    charts.append(zeros)

                    val_len=list(feat_data['val'].values())
                    val_pad=np.asarray(fids_pad)
                    val_pad=np.expand_dims(val_pad, axis=1)

                    zeros = [ [0] * meta['LOS'] for _ in range(meta['Chart'])]
                    for i in range(0,len(val_len )):
                        zeros[i][0:len(val_len[i])]=val_len[i]

                    #zeros = rate_pad * np.asarray(zeros)
                    charts_val.append(zeros)

    for hid, hid_data in data.items():
        #print(hid)
        for feature, feat_data in hid_data.items():
            #print(feature)
            if feature=='Med':
                #print(list(feat_data.keys()))
                fids=list(map(medVocabDict.get, list(feat_data['signal'].keys())))
                fids_pad=list(np.zeros(meta['Med']))
                fids_pad[0:len(fids)]=fids
                #fids=list(pd.Series(feat_data.keys()).map(medVocabDict))
                #print(fids)
                #meds.append(fids_pad)
                med_len=list(feat_data['signal'].values())
                med_pad=np.asarray(fids_pad)
                med_pad=np.expand_dims(med_pad, axis=1)
                #print(med_pad)
                zeros = [ [0] * meta['LOS'] for _ in range(meta['Med'])]
                zeros[0:len(med_len)]=med_len
                #print(zeros)
                zeros = med_pad * np.asarray(zeros)
                meds.append(zeros)

                if 'rate' in feat_data.keys():
                    rate_len=list(feat_data['rate'].values())
                    rate_pad=np.asarray(fids_pad)
                    rate_pad=np.expand_dims(rate_pad, axis=1)

                    zeros = [ [0] * meta['LOS'] for _ in range(meta['Med'])]
                    for i in range(0,len(rate_len )):
                        zeros[i][0:len(rate_len[i])]=rate_len[i]

                    #zeros = rate_pad * np.asarray(zeros)
                    meds_rate.append(zeros)

                    amount_len=list(feat_data['amount'].values())
                    amount_pad=np.asarray(fids_pad)
                    amount_pad=np.expand_dims(amount_pad, axis=1)

                    zeros = [ [0] * meta['LOS'] for _ in range(meta['Med'])]
                    for i in range(0,len(amount_len)):
                        zeros[i][0:len(amount_len[i])]=amount_len[i]

                    #zeros = amount_pad * np.asarray(zeros)
                    meds_amount.append(zeros)
                else:
                    rate_len=list(feat_data['val'].values())
                    rate_pad=np.asarray(fids_pad)
                    rate_pad=np.expand_dims(rate_pad, axis=1)

                    zeros = [ [0] * meta['LOS'] for _ in range(meta['Med'])]
                    for i in range(0,len(rate_len )):
                        zeros[i][0:len(rate_len[i])]=rate_len[i]

                    #zeros = rate_pad * np.asarray(zeros)
                    meds_rate.append(zeros)

            if feature=='Lab':
                #print(list(feat_data.keys()))
                fids=list(map(labVocabDict.get, list(feat_data['signal'].keys())))
                fids_pad=list(np.zeros(meta['Lab']))
                fids_pad[0:len(fids)]=fids
                #fids=list(pd.Series(feat_data.keys()).map(medVocabDict))
                #print(fids)
                #meds.append(fids_pad)
                lab_len=list(feat_data['signal'].values())
                lab_pad=np.asarray(fids_pad)
                lab_pad=np.expand_dims(lab_pad, axis=1)
                #print(med_pad)
                zeros = [ [0] * meta['LOS'] for _ in range(meta['Lab'])]
                zeros[0:len(lab_len)]=lab_len
                #print(zeros)
                zeros = lab_pad * np.asarray(zeros)
                labs.append(zeros)

                val_len=list(feat_data['val'].values())
                val_pad=np.asarray(fids_pad)
                val_pad=np.expand_dims(val_pad, axis=1)

                zeros = [ [0] * meta['LOS'] for _ in range(meta['Lab'])]
                for i in range(0,len(val_len )):
                    zeros[i][0:len(val_len[i])]=val_len[i]

                #zeros = rate_pad * np.asarray(zeros)
                labs_val.append(zeros)

            if feature=='Proc':
                #print(list(feat_data.keys()))
                fids=list(map(procVocabDict.get, list(feat_data.keys())))
                fids_pad=list(np.zeros(meta['Proc']))
                fids_pad[0:len(fids)]=fids
                #fids=list(pd.Series(feat_data.keys()).map(medVocabDict))
                #print(fids)
                #meds.append(fids_pad)
                proc_len=list(feat_data.values())
                fids_pad=np.asarray(fids_pad)
                fids_pad=np.expand_dims(fids_pad, axis=1)

                zeros = [ [0] * meta['LOS'] for _ in range(meta['Proc'])]
                zeros[0:len(proc_len)]=proc_len

                zeros = fids_pad * np.asarray(zeros)
                procs.append(zeros)

            if feature=='Out':
                #print(list(feat_data.keys()))
                fids=list(map(outVocabDict.get, list(feat_data.keys())))
                fids_pad=list(np.zeros(meta['Out']))
                fids_pad[0:len(fids)]=fids
                #fids=list(pd.Series(feat_data.keys()).map(medVocabDict))
                #print(fids)
                #meds.append(fids_pad)
                out_len=list(feat_data.values())
                fids_pad=np.asarray(fids_pad)
                fids_pad=np.expand_dims(fids_pad, axis=1)

                zeros = [ [0] * meta['LOS'] for _ in range(meta['Out'])]
                zeros[0:len(out_len)]=out_len

                zeros = fids_pad * np.asarray(zeros)
                outs.append(zeros)

            if feature=='Cond':
                #print(list(feat_data.keys()))
                fids=list(pd.Series(feat_data['fids']).map(condVocabDict))
                fids_pad=list(np.zeros(meta['Cond']))
                fids_pad[0:len(fids)]=fids
                conds.append(fids_pad)

        labels.append(hid_data['label'])
        eth.append(ethVocabDict[hid_data['ethnicity']])
        age.append(ageVocabDict[hid_data['age']])
        gender.append(genderVocabDict[hid_data['gender']])

    return [meds,meds_rate],procs,outs,[charts,charts_val],[labs,labs_val],conds,[eth,gender,age],labels


import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from torch.autograd import Variable

import numpy as np
from baselines.common.schedules import LinearSchedule
from baselines import logger

from model import NoisyDistDuelingConv, NoisyDistDuelingMLP, NoisyDuelingConv, NoisyDuelingMLP
from replay_buffer import ReplayBuffer_NStep, PrioritizedReplayBuffer_NStep, PrioritizedReplayBuffer

class TestAgent(object):
    def __init__(self, ob_shape, num_action, args):
        self.args = args
        self.num_action = num_action
        self.ob_shape = ob_shape

        self.dtype = torch.FloatTensor
        self.atype = torch.LongTensor
        if self.args.cuda:
            self.dtype = torch.cuda.FloatTensor
            self.atype = torch.cuda.LongTensor

        # self.model = NoisyDistDuelingConv(self.nb_atoms, ob_shape[0], num_action, self.dtype, args.sigma_init)
        # self.target_model = NoisyDistDuelingConv(self.nb_atoms, ob_shape[0], num_action, self.dtype, args.sigma_init)
        # self.model = NoisyDuelingConv(ob_shape[0], num_action, self.dtype, args.sigma_init)
        # self.target_model = NoisyDuelingConv(ob_shape[0], num_action, self.dtype, args.sigma_init)
        self.model = NoisyDuelingMLP(ob_shape[0], num_action, self.dtype, 0.17)
        self.target_model = NoisyDuelingMLP(ob_shape[0], num_action, self.dtype, 0.17)

        if self.args.cuda:
            self.model.cuda()
            self.target_model.cuda()
        self.optimizer = optim.Adam(self.model.parameters(), lr=self.args.lr, eps=self.args.adam_eps)
        self.criterion = nn.MSELoss(reduce=False)
        self.huber_loss = nn.SmoothL1Loss(reduce=False)

    def sample_noise(self):
        self.model.sample_noise()
        self.target_model.sample_noise()

    def update_target(self):
        self.target_model.load_state_dict(self.model.state_dict())

    def act(self, ob):
        ob_var = Variable(torch.from_numpy(ob).contiguous().type(self.dtype)).view(-1, *self.ob_shape)
        q_out = self.model(ob_var)
        _, deterministic_actions = q_out.data.max(1)
        out = deterministic_actions.cpu().numpy().astype(np.int32).reshape(-1)
        return out[0]

    def update(self, obs, actions, rewards, next_obs, dones, weights):
        obs = Variable(torch.from_numpy(obs).type(torch.FloatTensor)).view(-1, 4)
        next_obs = Variable(torch.from_numpy(next_obs).type(torch.FloatTensor)).view(-1, 4)
        dones = Variable(torch.from_numpy(dones.astype(float)).type(torch.FloatTensor)).view(-1, 1)
        rewards = Variable(torch.from_numpy(rewards).type(torch.FloatTensor)).view(-1, 1)
        actions = Variable(torch.from_numpy(actions.astype(int)).type(torch.LongTensor)).view(-1, 1)

        # Compute Bellman loss -> DDQN
        q_next = self.target_model(next_obs).detach()
        _, best_actions = self.model(next_obs).detach().max(1)
        q_next_best = q_next.gather(1, best_actions.view(-1, 1))
        q_next_best_rhs = rewards + self.args.gamma * q_next_best * (1 - dones)
        q = self.model(obs)
        q = q.gather(1, actions).squeeze(1)
        loss = self.criterion(q, q_next_best_rhs)

        # Step optimizer
        self.optimizer.zero_grad()
        loss.mean().backward()
        self.optimizer.step()

        return loss.data.numpy().flatten()

    def test_update(self, obs, actions, rewards, obs_next, dones, weights):
        obs = Variable(torch.from_numpy(obs).type(self.dtype)).view(-1, *self.ob_shape)
        obs_next = Variable(torch.from_numpy(obs_next).type(self.dtype)).view(-1, *self.ob_shape)
        weights = Variable(torch.from_numpy(weights).type(self.dtype)).view(-1, 1)
        actions = Variable(torch.from_numpy(actions.astype(int)).type(self.atype)).view(-1, 1)
        rewards = torch.from_numpy(rewards).type(self.dtype).view(-1, 1)
        dones = torch.from_numpy(dones.astype(float)).type(self.dtype).view(-1, 1)

        #
        online_q = self.model(obs)
        online_q_selected = online_q.gather(1, actions)

        # DDQN
        next_online_q = self.model(obs_next)
        _, next_online_action = next_online_q.data.max(1)
        next_target_q = self.target_model(obs_next).data

        next_target_best = next_target_q.gather(1, next_online_action.view(-1, 1))
        targets = rewards + (1.0 - dones) * self.args.gamma * next_target_best

        # Error
        td_error = online_q_selected.data - targets
        errors = F.smooth_l1_loss(online_q_selected, Variable(targets), reduce=False)
        weighted_error = (errors * weights).mean()

        #
        self.optimizer.zero_grad()
        weighted_error.backward()
        nn.utils.clip_grad_norm(self.model.parameters(), self.args.grad_norm_clipping)
        self.optimizer.step()
        return td_error.cpu().numpy().flatten()

def learn(env, args):
    ob = env.reset()
    ob_shape = ob.shape
    num_action = int(env.action_space.n)

    agent = TestAgent(ob_shape, num_action, args)
    replay_buffer = PrioritizedReplayBuffer(args.buffer_size, alpha=args.prioritized_replay_alpha)
    args.prioritized_replay_beta_iters = args.max_timesteps
    beta_schedule = LinearSchedule(args.prioritized_replay_beta_iters,
                                    initial_p=args.prioritized_replay_beta0,
                                    final_p=1.0)

    episode_rewards = [0.0]
    saved_mean_reward = None
    n_step_seq = []

    agent.sample_noise()
    agent.update_target()

    for t in range(args.max_timesteps):
        action = agent.act(ob)
        new_ob, rew, done, _ = env.step(action)
        replay_buffer.add(ob, action, rew, new_ob, float(done))
        ob = new_ob

        episode_rewards[-1] += rew
        if done:
            obs = env.reset()
            episode_rewards.append(0.0)
            reset = True

        if t > args.learning_starts and t % args.replay_period == 0:
            experience = replay_buffer.sample(args.batch_size, beta=beta_schedule.value(t))
            (obs, actions, rewards, obs_next, dones, weights, batch_idxes) = experience
            agent.sample_noise()
            kl_errors = agent.update(obs, actions, rewards, obs_next, dones, weights)
            replay_buffer.update_priorities(batch_idxes, np.abs(kl_errors) + 1e-6)

        if t > args.learning_starts and t % args.target_network_update_freq == 0:
            agent.update_target()

        mean_100ep_reward = round(np.mean(episode_rewards[-101:-1]), 1)
        num_episodes = len(episode_rewards)
        if done and args.print_freq is not None and len(episode_rewards) % args.print_freq == 0:
            print('steps {} episodes {} mean reward {}'.format(t, num_episodes, mean_100ep_reward))