"""
2020.05.06
Andy Revell and Alex Silva
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Purpose:
    Calculate functional correlations between given time series data and channels.
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Logic of code:
    1. Calculate correlations within a given time window: Window data in 1 second
    2. Calculate broadband functional connectivity with echobase broadband_conn
    3. Calculate other band functional connectivity with echobase multiband_conn
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Input:
    inputfile: a pickled list. See get_iEEG_data.py and https://docs.python.org/3/library/pickle.html for more information
        index 0: time series data N x M : row x column : time x channels
        index 1: fs, sampling frequency of time series data
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Output:
    Saves file outputfile as a pickel. For more info on pickeling, see https://docs.python.org/3/library/pickle.html
    Briefly: it is a way to save + compress data. it is useful for saving lists, as in a list of time series data and sampling frequency together along with channel names

    List index 0: ndarray. broadband. C x C x T. C: channels. T: times (1 second intervals). To change, see line 70: endInd = int(((t+1)*fs) - 1)
    List index 1: ndarray. alphatheta. C x C x T
    List index 2: ndarray. beta. C x C x T
    List index 3: ndarray. lowgamma. C x C x T
    List index 4: ndarray. highgamma. C x C x T
    List index 5: ndarray. C x _  Electrode row and column names. Stored here are the corresponding row and column names in the matrices above.
    List index 6: pd.DataFrame. N x 1. order of matrices in pickle file. The order of stored matrices are stored here to aid in transparency.
        Typically, the order in broadband, alphatheta, beta, lowgamma, highgamma

    To open the pickle file, use command:
    with open(outputfile, 'rb') as f: broadband, alphatheta, beta, lowgamma, highgamma, electrode_row_and_column_names, order_of_matrices_in_pickle_file = pickle.load(f)

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Example:

inputfile = '/Users/andyrevell/mount/DATA/Human_Data/BIDS_processed/sub-RID0278/eeg/sub-RID0278_HUP138_phaseII_248432340000_248525740000_EEG.pickle'
outputfile = '/Users/andyrevell/mount/DATA/Human_Data/BIDS_processed/sub-RID0278/connectivity_matrices/functional/sub-RID0278_HUP138_phaseII_248432340000_248525740000_functionalConnectivity.pickle'
get_Functional_connectivity(inputfile,outputfile)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Please use this naming convention if data is from iEEG.org
sub-RIDXXXX_iEEGFILENAME_STARTTIME_STOPTIME_functionalConnectivity.pickle
example: 'sub-RID0278_HUP138_phaseII_248432340000_248525740000_functionalConnectivity.pickle'

"""

import numpy as np
import pickle
import pandas as pd
from scipy.io import loadmat #fromn scipy to read mat files (structural connectivity)
from scipy.stats import spearmanr, pearsonr
import matplotlib.pyplot as plt

def SFC(structure_file_path,function_file_path,electrode_localization_by_atlas_file_path, outputfile):
    """
    :param structure_file_path:
    :param function_file_path:
    :param electrode_localization_by_atlas_file_path:
    :return:
    """

    """

    #Example:
    structure_file_path= '/Users/andyrevell/mount/DATA/Human_Data/BIDS_processed/sub-RID0278/connectivity_matrices/structural/RA_N1000/sub-RID0278_ses-preop3T_dwi-eddyMotionB0Corrected.nii.gz.trk.gz.RA_N1000_Perm0001.count.pass.connectivity.mat'
    electrode_localization_by_atlas_file_path = '/Users/andyrevell/mount/DATA/Human_Data/BIDS_processed/sub-RID0278/electrode_localization/electrode_localization_by_atlas/sub-RID0278_electrode_coordinates_mni_RA_N1000_Perm0001.csv'

    #seizure 3
    #pre-ictal
    function_file_path = '/Users/andyrevell/mount/DATA/Human_Data/BIDS_processed/sub-RID0278/connectivity_matrices/functional/sub-RID0278_HUP138_phaseII_415933490000_416023190000_functionalConnectivity_window0.20.pickle'
    #ictal
    function_file_path = '/Users/andyrevell/mount/DATA/Human_Data/BIDS_processed/sub-RID0278/connectivity_matrices/functional/sub-RID0278_HUP138_phaseII_416023190000_416112890000_functionalConnectivity_window0.20.pickle'
    #postictal
    function_file_path = '/Users/andyrevell/mount/DATA/Human_Data/BIDS_processed/sub-RID0278/connectivity_matrices/functional/sub-RID0278_HUP138_phaseII_416112890000_416292890000_functionalConnectivity_window0.20.pickle'

    #seizure 1
    #pre-ictal
    function_file_path = '/Users/andyrevell/mount/DATA/Human_Data/BIDS_processed/sub-RID0278/connectivity_matrices/functional/sub-RID0278_HUP138_phaseII_248432340000_248525740000_functionalConnectivity.pickle'
    #ictal
    function_file_path = '/Users/andyrevell/mount/DATA/Human_Data/BIDS_processed/sub-RID0278/connectivity_matrices/functional/sub-RID0278_HUP138_phaseII_248525740000_248619140000_functionalConnectivity.pickle'
    #postictal
    function_file_path = '/Users/andyrevell/mount/DATA/Human_Data/BIDS_processed/sub-RID0278/connectivity_matrices/functional/sub-RID0278_HUP138_phaseII_248619140000_248799140000_functionalConnectivity.pickle'

    #Output Files:
    #seizure 3
    #pre-ictal
    outputfile = '/Users/andyrevell/mount/DATA/Human_Data/BIDS_processed/sub-RID0278/connectivity_matrices/structure_function_correlation/RA_N1000/sub-RID0278_HUP138_phaseII_415933490000_416023190000_RA_N1000_Perm0001_correlation_window0.20.pickle'
    #ictal
    outputfile = '/Users/andyrevell/mount/DATA/Human_Data/BIDS_processed/sub-RID0278/connectivity_matrices/structure_function_correlation/RA_N1000/sub-RID0278_HUP138_phaseII_416023190000_416112890000_RA_N1000_Perm0001_correlation_window0.20.pickle'
    #postictal
    outputfile = '/Users/andyrevell/mount/DATA/Human_Data/BIDS_processed/sub-RID0278/connectivity_matrices/structure_function_correlation/RA_N1000/sub-RID0278_HUP138_phaseII_416112890000_416292890000_RA_N1000_Perm0001_correlation_window0.20.pickle'

    """
    #Get functional connecitivty data in pickle file format
    with open(function_file_path, 'rb') as f: broadband, alphatheta, beta, lowgamma, highgamma, electrode_row_and_column_names, order_of_matrices_in_pickle_file = pickle.load(f)
    FC_list = [broadband, alphatheta, beta, lowgamma,highgamma ]

    #Get Structural Connectivity data in mat file format. Output from DSI studio
    structural_connectivity_array = np.array(pd.DataFrame(loadmat(structure_file_path)['connectivity']))

    #Get electrode localization by atlas csv file data. From get_electrode_localization.py
    electrode_localization_by_atlas = pd.read_csv(electrode_localization_by_atlas_file_path)

    # normalizing and log-scaling the structural matrices
    structural_connectivity_array[structural_connectivity_array == 0] = 1;
    structural_connectivity_array = np.log10(structural_connectivity_array)  # log-scaling. Converting 0s to 1 to avoid taking log of zeros
    structural_connectivity_array = structural_connectivity_array / np.max(structural_connectivity_array)  # normalization

    #Remove electrodes in "electrode localization" not found in Functional Connectivity matrices
    electrode_localization_names = np.array(electrode_localization_by_atlas['electrode_name'])
    electrode_localization_by_atlas = electrode_localization_by_atlas[np.in1d(electrode_localization_names,electrode_row_and_column_names )]
    # Remove electrodes in the Functional Connectivity matrices not found in "electrode localization"
    not_in_functional_connectivity = np.in1d(electrode_row_and_column_names, electrode_localization_names)
    for i in range(len(FC_list)):
        FC_list[i] = FC_list[i][not_in_functional_connectivity,:,:]
        FC_list[i] = FC_list[i][:,not_in_functional_connectivity, :]

    #Fisher z-transform of functional connectivity data. This is to take means of correlations and do correlations to the structural connectivity
    #Fisher z transform is just arctanh
    for i in range(len(FC_list)):
        FC_list[i] = np.arctanh(FC_list[i])

    # Remove structural ROIs not in electrode_localization ROIs
    electrode_ROIs = np.unique(np.array( electrode_localization_by_atlas.iloc[:,4]))
    electrode_ROIs = electrode_ROIs[~(electrode_ROIs == 0)] #remove region 0
    structural_index = electrode_ROIs - 1 #subtract 1 because of python's zero indexing
    structural_connectivity_array = structural_connectivity_array[structural_index,:]
    structural_connectivity_array = structural_connectivity_array[:,structural_index]

    #taking average functional connectivity for those electrodes in same atlas regions
    for i in range(len(FC_list)):
        ROIs = np.array( electrode_localization_by_atlas.iloc[:,4])
        for r in range(len(electrode_ROIs)):
            index_logical = (ROIs == electrode_ROIs[r])
            index_first = np.where(index_logical)[0][0]
            index_second_to_end = np.where(index_logical)[0][1:]
            mean = np.mean(FC_list[i][index_logical,:,:], axis=0)
            # Fill in with mean.
            FC_list[i][index_first,:,:] = mean
            FC_list[i][:, index_first, :] = mean
            #delete the other rows and oclumns belonging to same region.
            FC_list[i] = np.delete(FC_list[i], index_second_to_end, axis=0)
            FC_list[i] = np.delete(FC_list[i], index_second_to_end, axis=1)
            #keeping track of which electrode labels correspond to which rows and columns
            ROIs = np.delete(ROIs, index_second_to_end, axis=0)
        #remove electrodes in the ROI labeld as zero
        index_logical = (ROIs == 0)
        index = np.where(index_logical)[0]
        FC_list[i] = np.delete(FC_list[i], index, axis=0)
        FC_list[i] = np.delete(FC_list[i], index, axis=1)
        ROIs = np.delete(ROIs, index, axis=0)

    #order FC matrices by ROIs
    order = np.argsort(ROIs)
    for i in range(len(FC_list)):
        FC_list[i] = FC_list[i][order,:,:]
        FC_list[i] = FC_list[i][:, order, :]

    #un-fisher ztranform
    for i in range(len(FC_list)):
        FC_list[i] = np.tanh(FC_list[i])

    #initialize correlation arrays
    Corrrelation_list = [None] * len(FC_list)
    for i in range(len(FC_list)):
        Corrrelation_list[i] = np.zeros(  [FC_list[0].shape[2]], dtype=float)

    correlation_type = 'spearman'
    #calculate Structure-Function Correlation.
    for i in range(len(FC_list)):
        for t in range(FC_list[i].shape[2]-1):
            #Spearman Rank Correlation: functional connectivity and structural connectivity are non-normally distributed. So we should use spearman
            if correlation_type == 'spearman':
                Corrrelation_list[i][t] = spearmanr(  np.ndarray.flatten(FC_list[i][:,:,t]) , np.ndarray.flatten(structural_connectivity_array)   ).correlation
                #print("spearman")
            # Pearson Correlation: This is calculated bc past studies use Pearson Correlation and we want to see if these results are comparable.
            if correlation_type == 'pearson':
                Corrrelation_list[i][t] = pearsonr(np.ndarray.flatten(FC_list[i][:, :, t]), np.ndarray.flatten(structural_connectivity_array))[0]

    order_of_matrices_in_pickle_file = pd.DataFrame(["broadband", "alphatheta", "beta", "lowgamma" , "highgamma" ], columns=["Order of matrices in pickle file"])
    with open(outputfile, 'wb') as f: pickle.dump([Corrrelation_list[0], Corrrelation_list[1], Corrrelation_list[2], Corrrelation_list[3], Corrrelation_list[4], order_of_matrices_in_pickle_file], f)

"""
#Vizualizing distributions of raw data:
fig, axs = plt.subplots(2,1)
axs[0].hist( np.ndarray.flatten( np.array(structural_connectivity_array)), bins = 40 )
axs[0].title.set_text('Structural Streamline Counts Distribution')
axs[0].set_xlabel('Log normalized Stremline Counts (only ones in electrode regions)')
axs[0].set_ylabel('Frequency')
axs[1].hist(np.ndarray.flatten(np.array(  np.arctanh(broadband)  )), bins=40)
axs[1].title.set_text('BroadBand Cross Correlation Distribution')
axs[1].set_xlabel('Cross Correlation Fisher Z transform')
axs[1].set_ylabel('Frequency')
plt.show()
#Notes: streamline counts seems to have a HEAVY right skew distribution: Okay for correlating Structure and Function distirbutions with just a pearson correlation? Should do spearman rank

#visualize structure-function correlation on a correlational line
x = np.ndarray.flatten(structural_connectivity_array)
i = 0
t=0
y = np.ndarray.flatten(FC_list[i][:,:,t])
fig, axs = plt.subplots(1,1)
axs.scatter(x, y )
axs.axes.set_xlim(-0.1,1)
axs.axes.set_ylim(0, 1)
axs.title.set_text('')
axs.set_xlabel('Structural Correlation')
axs.set_ylabel('Functional Correlation')
plt.show()

# visualize structure-function correlation by time
i = 0
x = range(len(Corrrelation_list[i]))
y = np.ndarray.flatten(Corrrelation_list[i])
fig, axs = plt.subplots(1, 1)
axs.scatter(x, y)
#axs.axes.set_xlim(-0.1, 1)
axs.axes.set_ylim(0.1, 0.6)
axs.title.set_text('')
axs.set_xlabel('')
axs.set_ylabel('')
plt.show()

#save FC_list for vizualization
# seizure 3
# pre-ictal
outputfile_FC_list = '/Users/andyrevell/mount/USERS/arevell/papers/paper001/data_processed/supplemental_data/sub-RID0278_HUP138_phaseII_415933490000_416023190000_RA_N1000_Perm0001_matrices_window0.20.pickle'
# ictal
outputfile_FC_list = '/Users/andyrevell/mount/USERS/arevell/papers/paper001/data_processed/supplemental_data/sub-RID0278_HUP138_phaseII_416023190000_416112890000_RA_N1000_Perm0001_matrices_window0.20.pickle'
# postictal
outputfile_FC_list = '/Users/andyrevell/mount/USERS/arevell/papers/paper001/data_processed/supplemental_data/sub-RID0278_HUP138_phaseII_416112890000_416292890000_RA_N1000_Perm0001_matrices_window0.20.pickle'

order_of_matrices_in_pickle_file = pd.DataFrame(["broadband", "alphatheta", "beta", "lowgamma" , "highgamma", "structural" ], columns=["Order of matrices in pickle file"])
with open(outputfile_FC_list, 'wb') as f: pickle.dump([FC_list[0], FC_list[1], FC_list[2], FC_list[3], FC_list[4],structural_connectivity_array, ROIs, order_of_matrices_in_pickle_file], f)

"""


import tweepy
from tweepy import OAuthHandler
import json
import datetime as dt
import time
import os
import sys
import couchdb
from tweepy.utils import import_simplejson
from CouchDBConnect import CouchDBConnect
import core

'''
Python 3 and tweepy version 3.5.0.
'''

def load_api():
    ''' Function that loads the twitter API after authorizing the user. '''
    args = core.config('twitter','OAuth')

    consumer_key = args['consumer_key']
    consumer_secret = args['consumer_secret']
    access_token = args['access_token']
    access_secret = args['access_secret']
    auth = OAuthHandler(consumer_key, consumer_secret)
    auth.set_access_token(access_token, access_secret)
    # load the twitter API via tweepy
    return tweepy.API(auth)

def tweet_search(api, max_tweets,geocode,since_id,max_id=-1):
    ''' Function that takes in a search string 'query', the maximum
        number of tweets 'max_tweets'.
        It returns a list of tweepy.models.Status objects. '''

    searchedtweets = []
    while len(searchedtweets) < max_tweets:
        remainingtweets = max_tweets - len(searched_tweets)
        try:
            if(max_id== -1) :
                td = dt.datetime.now() - dt.timedelta(days=2)
                tweet_date = '{0}-{1:0>2}-{2:0>2}'.format(td.year, td.month, td.day)
                # get list of up to 100 tweets
                new_tweets = api.search(count=remaining_tweets,geocode=geocode,until=tweet_date)

            else :
                # Search tweets according to since_id and max_id
                new_tweets = api.search(count=remaining_tweets,geocode=geocode,max_id=max_id,since_id=since_id)
            print('found', len(new_tweets), 'tweets')
            if not new_tweets:
                print('no tweets found')
                break
            searchedtweets.extend(new_tweets)
        except tweepy.TweepError:
            print('exception raised, waiting 15 minutes')
            print('(until:', dt.datetime.now() + dt.timedelta(minutes=15), ')')
            time.sleep(15 * 60)
            break  # stop the loop
    return searchedtweets

def write_tweets(tweets):
    ''' Function that appends tweets to couchdb. '''

    json = import_simplejson()
    _db_handle=CouchDBConnect.connect_CouchDB()
    db = _db_handle["tweets_all"]
    dtweet={}
    for tweet in tweets:

            dtweet= json.dumps(tweet._json)
            store_tweet(db,dtweet)

def store_tweet(db,dtweet) :
    """Tweet as Dict"""
    try:
        doc = json.loads(dtweet)
        doc.update({'_id': doc['id_str']})
    except KeyError:
        print(
            "Warning: store_tweet() called, but tweet input "
            + "parameter does not contain an id_str."
        )
        return None
    if not isinstance(doc, dict):
        print(
            "Warning: store_dict() called, but doc input parameter"
            + " is not a dict."
        )
        return None
    # Attempt to save document to CouchDB
    try:
        response = db.save(doc)
    # If the _id already exists
    except couchdb.http.ResourceConflict as e:
        print(
            "Warning: The doc (_id: "
            + doc['_id']
            + ") is already exists "
        )

        print(
            "Overwriting doc (_id: "
            + doc['_id']
            + ")"
        )
def get_tweet_id(api, days_ago,geocode="Australia"):
    ''' Function that gets the ID of a tweet. I trecives api , days_ago
        and region for search as parameters'''
    # return an ID from __ days ago
    td = dt.datetime.now() - dt.timedelta(days=days_ago)
    tweet_date = '{0}-{1:0>2}-{2:0>2}'.format(td.year, td.month, td.day)
    # get list of up to 10 tweets
    tweet = api.search(count=10,until=tweet_date,geocode=geocode)
    # return the id of the first tweet in the list
    return tweet[0].id

def get_replies(db):
    """Retrieves a list of reply-to's that need to be
    downloaded.
    """
    print("get_replies",db.view('replies/replies'))
    try:
    #if(True) :
        for row in db.view('replies/replies',
                                 wrapper=None,
                                 group_level=1):
            if (row.value == 1):
                queue.update({row.key[0]: {}})
        for row in db.view('replies/replies',
                                 wrapper=None,
                                 group='true'):
            if row.key[0] in queue:
                queue[row.key[0]]['reply'] = row.key[1]
                queue[row.key[0]]['reply_user'] = row.key[2]
        print("get_replies1")
    except:
        print("Error")
    return queue
def iterate_replies(db,db2,api):
    """This method downloads tweets that tweets in our database
    have replied to.
    """
    try:
        replies = get_replies(db2)
        for r in replies:
            try:
                tweet = api.get_status(r)
                reply = replies[r]['reply']
                reply_user = replies[r]['reply_user']
                dtweet= json.dumps(tweet._json)
                store_tweet(db,dtweet)
            except:
                print("Exception1")
                pass
    except:
        print("Exception2")
        pass
def main():
    ''' This is a script that continuously searches for tweets
        that were created over a given number of days. The search
        dates and search phrase can be changed below. '''

    _db_handle=CouchDBConnect.connect_CouchDB()
    db2=_db_handle["tweets"]
    db = _db_handle["tweets_all"]

    time_limit = 25  # runtime limit in hours
    max_tweets = 100  # number of tweets per search (will be
    # iterated over) - maximum is 100
    min_days_old, max_days_old = 0, 7  # search limits e.g., from 7 to 8
    # gives current weekday from last week,
    # min_days_old=0 will search from right now

    Australia = '-34.337655,135.697131,2700km'

    api = load_api()
    max_id=-1
    while(True):
        #for search_phrase in search_phrases:
         if(True):
            if(max_id==-1) :
                  since_id=get_tweet_id(api, days_ago=4,geocode=Australia)
            else :
                  since_id=max_id
            json_file="json_file.json"

            ''' tweet gathering loop  '''
            start = dt.datetime.now()
            end = start + dt.timedelta(hours=time_limit)
            count = 0
            exitcount = 0
            while dt.datetime.now() < end:
                count += 1
                print('count =', count)
                # collect tweets
                try:
                    with open(json_file, 'r') as f:
                        lines= f.readlines()
                        max_id = json.loads(lines[-1])['id']

                        print('Searching from the bottom ID in file ', max_id)
                except FileNotFoundError:
                    max_id = -1
                tweets= tweet_search(api, max_tweets,geocode=Australia,since_id=since_id,max_id=max_id)
                # write tweets to couchdb in JSON format

                if tweets:
                    write_tweets(tweets)
                    with open(json_file, 'w') as f:
               # write the tweets to a json file for getting the max_id after each iteration.
                       for tweet in tweets:
                            json.dump(tweet._json, f)
                            f.write('\n')
                    exitcount = 0
                else:
                    exitcount += 1
                    if exitcount == 3:
                           break

if __name__ == "__main__":
    main()


import tensorflow as tf
import numpy as np
import scipy.io
import time
import horovod.tensorflow as hvd

# Hyper parameters
tor = 1e-4
batch_num = 150
lambda_b = 10.
lambda_i = 50.
layers = [4] + 10 * [6 * 50] + [4]

#
N_Skip = batch_num

class ChannelFlow:
    # Initialize the class
    def __init__(self, t, x, y, z,  # inside points
                 data_boundary,  # boundary conditions
                 data_initial,  # initial conditions
                 layers, Xmin, Xmax,   # network architecture
                 P_x, niu):  # flow parameters

        self.P_x = P_x
        self.niu = niu

        # data of boundary conditions
        self.tb = data_boundary[:, 0:1]
        self.xb = data_boundary[:, 1:2]
        self.yb = data_boundary[:, 2:3]
        self.zb = data_boundary[:, 3:4]
        self.ub = data_boundary[:, 4:5]
        self.vb = data_boundary[:, 5:6]
        self.wb = data_boundary[:, 6:7]
        self.pb = data_boundary[:, 7:8]

        # Number of data points for the boundary
        self.Nb = self.tb.shape[0]

        # data of initial conditions
        self.ti = data_initial[:, 0:1]
        self.xi = data_initial[:, 1:2]
        self.yi = data_initial[:, 2:3]
        self.zi = data_initial[:, 3:4]
        self.ui = data_initial[:, 4:5]
        self.vi = data_initial[:, 5:6]
        self.wi = data_initial[:, 6:7]
        self.pi = data_initial[:, 7:8]

        # Number of data points for the initial
        self.Ni = self.ti.shape[0]

        Xb = np.concatenate([self.tb, self.xb, self.yb, self.zb], 1)
        Xi = np.concatenate([self.ti, self.xi, self.yi, self.zi], 1)
        X = np.concatenate([Xb, Xi], 0)
        self.lower_X = Xmin #X.min(0)  # minimum of X
        self.upper_X = Xmax #X.max(0)  # maximum of X

        # data inside the domain
        self.te = t
        self.xe = x
        self.ye = y
        self.ze = z

        # Number of data points for the equation
        self.Ne = self.te.shape[0]

        # layers
        self.layers = layers

        # initialize NN
        self.weights, self.biases = self.initialize_NN(layers)

        # tf placeholders and graph

        config=tf.ConfigProto()
        config.gpu_options.allow_growth=True
        config.gpu_options.visible_device_list=str(hvd.local_rank())
        self.sess = tf.Session(config=config)
        self.saver = tf.train.Saver()
        self.learning_rate = tf.placeholder(tf.float32, shape=[])  # it can change during the training process

        # placeholders for data of boundary
        self.tb_tf = tf.placeholder(tf.float32, shape=[None, 1])
        self.xb_tf = tf.placeholder(tf.float32, shape=[None, 1])
        self.yb_tf = tf.placeholder(tf.float32, shape=[None, 1])
        self.zb_tf = tf.placeholder(tf.float32, shape=[None, 1])
        self.ub_tf = tf.placeholder(tf.float32, shape=[None, 1])
        self.vb_tf = tf.placeholder(tf.float32, shape=[None, 1])
        self.wb_tf = tf.placeholder(tf.float32, shape=[None, 1])

        # placeholders for data of initial
        self.ti_tf = tf.placeholder(tf.float32, shape=[None, 1])
        self.xi_tf = tf.placeholder(tf.float32, shape=[None, 1])
        self.yi_tf = tf.placeholder(tf.float32, shape=[None, 1])
        self.zi_tf = tf.placeholder(tf.float32, shape=[None, 1])
        self.ui_tf = tf.placeholder(tf.float32, shape=[None, 1])
        self.vi_tf = tf.placeholder(tf.float32, shape=[None, 1])
        self.wi_tf = tf.placeholder(tf.float32, shape=[None, 1])

        # placeholders for data inside the domain
        self.te_tf = tf.placeholder(tf.float32, shape=[None, 1])
        self.xe_tf = tf.placeholder(tf.float32, shape=[None, 1])
        self.ye_tf = tf.placeholder(tf.float32, shape=[None, 1])
        self.ze_tf = tf.placeholder(tf.float32, shape=[None, 1])

        # physics informed neural networks (boundary)
        (self.ub_pred, self.vb_pred, self.wb_pred, self.pb_pred,
         self.eq1b_pred, self.eq2b_pred, self.eq3b_pred, self.eq4b_pred) =\
            self.net_NS_AD(self.tb_tf, self.xb_tf, self.yb_tf, self.zb_tf)

        # physics informed neural networks (initial)
        (self.ui_pred, self.vi_pred, self.wi_pred, self.pi_pred,
         self.eq1i_pred, self.eq2i_pred, self.eq3i_pred, self.eq4i_pred) =\
            self.net_NS_AD(self.ti_tf, self.xi_tf, self.yi_tf, self.zi_tf)

        # physics informed neural networks (inside the domain)
        (self.u_pred, self.v_pred, self.w_pred, self.p_pred,
         self.eq1_pred, self.eq2_pred, self.eq3_pred, self.eq4_pred) =\
            self.net_NS_AD(self.te_tf, self.xe_tf, self.ye_tf, self.ze_tf)

        self.loss_b = tf.reduce_mean(tf.square(self.ub_tf - self.ub_pred)) + \
                      tf.reduce_mean(tf.square(self.vb_tf - self.vb_pred)) + \
                      tf.reduce_mean(tf.square(self.wb_tf - self.wb_pred))

        self.loss_i = tf.reduce_mean(tf.square(self.ui_tf - self.ui_pred)) + \
                      tf.reduce_mean(tf.square(self.vi_tf - self.vi_pred)) + \
                      tf.reduce_mean(tf.square(self.wi_tf - self.wi_pred))

        self.loss_eq1 = tf.reduce_mean(tf.square(self.eq1_pred)) + \
                        tf.reduce_mean(tf.square(self.eq1b_pred)) + \
                        tf.reduce_mean(tf.square(self.eq1i_pred))
        self.loss_eq2 = tf.reduce_mean(tf.square(self.eq2_pred)) + \
                        tf.reduce_mean(tf.square(self.eq2b_pred)) + \
                        tf.reduce_mean(tf.square(self.eq2i_pred))
        self.loss_eq3 = tf.reduce_mean(tf.square(self.eq3_pred)) + \
                        tf.reduce_mean(tf.square(self.eq3b_pred)) + \
                        tf.reduce_mean(tf.square(self.eq3i_pred))
        self.loss_eq4 = tf.reduce_mean(tf.square(self.eq4_pred)) + \
                        tf.reduce_mean(tf.square(self.eq4b_pred)) + \
                        tf.reduce_mean(tf.square(self.eq4i_pred))

        # loss
        self.loss = self.loss_b * lambda_b + self.loss_i * lambda_i + \
                    self.loss_eq1 + self.loss_eq2 + self.loss_eq3 + self.loss_eq4

        self.loss1=hvd.allreduce(self.loss)
        # optimizers
        self.optimizer = tf.train.AdamOptimizer(learning_rate=self.learning_rate)
        self.optimizer = hvd.DistributedOptimizer(self.optimizer)
        self.train_op = self.optimizer.minimize(self.loss)
        self.IteNum = 0
        self.epoch = 0
        self.lossCol = []
        init = tf.global_variables_initializer()
        bcast=hvd.broadcast_global_variables(0)
        self.sess.run(init)
        self.sess.run(bcast)
        #print("Weights", self.sess.run(self.weights), "Biases", self.sess.run(self.biases), "rank", hvd.rank() )

    def initialize_NN(self, layers):
        weights = []
        biases = []
        num_layers = len(layers)
        for l in range(0, num_layers - 1):
            W = self.xavier_init(size=[layers[l], layers[l + 1]])
            b = tf.Variable(tf.zeros([1, layers[l + 1]], dtype=tf.float32), dtype=tf.float32)
            weights.append(W)
            biases.append(b)
        return weights, biases  # they are all three dimensional

    def xavier_init(self, size):
        in_dim = size[0]
        out_dim = size[1]
        xavier_stddev = np.sqrt(2 / (in_dim + out_dim))  # stddev decreases with the increasing of in_dim + out_dim
        return tf.Variable(tf.truncated_normal([in_dim, out_dim], stddev=xavier_stddev), dtype=tf.float32)

    def neural_net(self, X, weights, biases):
        num_layers = len(weights) + 1

        H = 2.0 * (X - self.lower_X) / (self.upper_X - self.lower_X) - 1.0  # The input is normalize to [-1, 1].
        for l in range(0, num_layers - 2):
            W = weights[l]
            b = biases[l]
            H = tf.tanh(tf.add(tf.matmul(H, W), b))
        W = weights[-1]
        b = biases[-1]
        Y = tf.add(tf.matmul(H, W), b)
        return Y

    def net_NS_AD(self, t, x, y, z):
        uvwp = self.neural_net(tf.concat([t, x, y, z], 1), self.weights, self.biases)

        u = uvwp[:, 0:1]
        v = uvwp[:, 1:2]
        w = uvwp[:, 2:3]
        p = uvwp[:, 3:4]

        u_t = tf.gradients(u, t)[0]
        v_t = tf.gradients(v, t)[0]
        w_t = tf.gradients(w, t)[0]

        u_x = tf.gradients(u, x)[0]
        v_x = tf.gradients(v, x)[0]
        w_x = tf.gradients(w, x)[0]
        p_x = tf.gradients(p, x)[0]

        u_y = tf.gradients(u, y)[0]
        v_y = tf.gradients(v, y)[0]
        w_y = tf.gradients(w, y)[0]
        p_y = tf.gradients(p, y)[0]

        u_z = tf.gradients(u, z)[0]
        v_z = tf.gradients(v, z)[0]
        w_z = tf.gradients(w, z)[0]
        p_z = tf.gradients(p, z)[0]

        u_xx = tf.gradients(u_x, x)[0]
        v_xx = tf.gradients(v_x, x)[0]
        w_xx = tf.gradients(w_x, x)[0]

        u_yy = tf.gradients(u_y, y)[0]
        v_yy = tf.gradients(v_y, y)[0]
        w_yy = tf.gradients(w_y, y)[0]

        u_zz = tf.gradients(u_z, z)[0]
        v_zz = tf.gradients(v_z, z)[0]
        w_zz = tf.gradients(w_z, z)[0]

        eq1 = u_t + (u * u_x + v * u_y + w * u_z) + (p_x + self.P_x) - self.niu * (u_xx + u_yy + u_zz)
        eq2 = v_t + (u * v_x + v * v_y + w * v_z) + p_y - self.niu * (v_xx + v_yy + v_zz)
        eq3 = w_t + (u * w_x + v * w_y + w * w_z) + p_z - self.niu * (w_xx + w_yy + w_zz)
        eq4 = u_x + v_y + w_z

        return u, v, w, p, eq1, eq2, eq3, eq4

    def train(self, num_epochs, batch_number, learning_rate):
        batch_size_b = self.Nb // batch_number
        batch_size_i = self.Ni // batch_number
        batch_size_e = self.Ne // batch_number
        print("batch_size_b is ", batch_size_b)
        print("batch_size_i is ", batch_size_i)
        print("batch_size_e is ", batch_size_e)

        start_time = time.time()
        loss_value = 1.
        for epoch in range(num_epochs):
            if loss_value > tor:
                self.epoch += 1
                perm_b = np.random.permutation(self.Nb)
                perm_i = np.random.permutation(self.Ni)
                perm_e = np.random.permutation(self.Ne)
                for it in range(0, batch_num):
                    self.IteNum += 1
                    idx_b = perm_b[it*batch_size_b:(it+1)*batch_size_b]
                    idx_i = perm_i[it*batch_size_i:(it+1)*batch_size_i]
                    idx_e = perm_e[it*batch_size_e:(it+1)*batch_size_e]

                    tf_dict = {self.ti_tf: self.ti[idx_i, :], self.xi_tf: self.xi[idx_i, :],
                               self.yi_tf: self.yi[idx_i, :], self.zi_tf: self.zi[idx_i, :],
                               self.ui_tf: self.ui[idx_i, :], self.vi_tf: self.vi[idx_i, :],
                               self.wi_tf: self.wi[idx_i, :],  # initial
                               self.tb_tf: self.tb[idx_b, :], self.xb_tf: self.xb[idx_b, :],
                               self.yb_tf: self.yb[idx_b, :], self.zb_tf: self.zb[idx_b, :],
                               self.ub_tf: self.ub[idx_b, :], self.vb_tf: self.vb[idx_b, :],
                               self.wb_tf: self.wb[idx_b, :],  # boundary
                               self.te_tf: self.te[idx_e, :], self.xe_tf: self.xe[idx_e, :],
                               self.ye_tf: self.ye[idx_e, :], self.ze_tf: self.ze[idx_e, :],
                               self.learning_rate: learning_rate}
                    # print(tf_dict)

                    self.sess.run(self.train_op, tf_dict)
                    # Print
                    if self.IteNum % N_Skip == 0:
                        elapsed = time.time() - start_time
                        (loss_value,
                         loss_b, loss_i,
                         loss_eq1, loss_eq2, loss_eq3, loss_eq4,
                         learning_rate_value) = self.sess.run([self.loss1, self.loss_b, self.loss_i,
                                                               self.loss_eq1, self.loss_eq2,
                                                               self.loss_eq3, self.loss_eq4,
                                                               self.learning_rate], tf_dict)
                        print('***************************************************************')
                        print('Epoch: %d, It: %d, Time: %.2f, Learning Rate: %.1e, Loss: %.3e'
                              % (self.epoch, it, elapsed, learning_rate_value, loss_value))
                        print('LossI: %.3e, LossB: %.3e, LossEqX: %.3e, LossEqY: %.3e, LossEqZ: %.3e, LossEqIncom: %.3e'
                              % (loss_i, loss_b, loss_eq1, loss_eq2, loss_eq3, loss_eq4))
                        print('')
                        start_time = time.time()
                        self.lossCol.append([self.IteNum, loss_value, loss_b,
                                             loss_i, loss_eq1, loss_eq2, loss_eq3, loss_eq4])
        return loss_value

    def predict(self, t_star, x_star, y_star, z_star):

        tf_dict = {self.te_tf: t_star, self.xe_tf: x_star, self.ye_tf: y_star, self.ze_tf: z_star}

        u_star = self.sess.run(self.u_pred, tf_dict)
        v_star = self.sess.run(self.v_pred, tf_dict)
        w_star = self.sess.run(self.w_pred, tf_dict)
        p_star = self.sess.run(self.p_pred, tf_dict)

        return u_star, v_star, w_star, p_star

# -------------------------------------------------------- #
# ------------------------Training------------------------ #
# -------------------------------------------------------- #
# Load Data
data = scipy.io.loadmat('./Data/Channel_1000plus.mat')

t_star = data['t_star']  # T x 1
x_star = data['x_star']  # N x 1
y_star = data['y_star']  # N x 1
z_star = data['z_star']  # N x 1

T = t_star.shape[0]
N = x_star.shape[0]

# Rearrange Data
T_star = np.tile(t_star, (1, N)).T  # N x T
X_star = np.tile(x_star, (1, T))  # N x T
Y_star = np.tile(y_star, (1, T))  # N x T
Z_star = np.tile(z_star, (1, T))  # N x T

t_train = T_star.flatten()[:, None]  # NT x 1
x_train = X_star.flatten()[:, None]  # NT x 1
y_train = Y_star.flatten()[:, None]  # NT x 1
z_train = Z_star.flatten()[:, None]  # NT x 1
t_max=t_train.max(0)
print("t_max is ", t_max)
dataB=data['data_b']
dataI=data['data_i']
print("Ne ", np.shape(t_train), "NI", np.shape(dataI), "Nb", np.shape(dataB))

tb = dataB[:, 0:1]
xb = dataB[:, 1:2]
yb = dataB[:, 2:3]
zb = dataB[:, 3:4]

# data of initial conditions
ti = dataI[:, 0:1]
xi = dataI[:, 1:2]
yi = dataI[:, 2:3]
zi = dataI[:, 3:4]

# Number of data points for the initial
Xb = np.concatenate([tb, xb, yb, zb], 1)
Xi = np.concatenate([ti, xi, yi, zi], 1)
X = np.concatenate([Xb, Xi], 0)
Xmin=X.min(0)
Xmax=X.max(0)

# Training
hvd.init()
#
totalSize=np.shape(t_train)[0]
chunkSize=totalSize // hvd.size()

startId=hvd.rank()*chunkSize
endId=startId + chunkSize

totalBSize=np.shape(dataB)[0]
totalISize=np.shape(dataI)[0]
chunkBSize=totalBSize // hvd.size()
chunkISize=totalISize // hvd.size()
startBId=hvd.rank()*chunkBSize
endBId=startBId + chunkBSize

startIId=hvd.rank()*chunkISize
endIId=startIId + chunkISize

x_loc=x_train[startId:endId, :]
y_loc=y_train[startId:endId, :]
z_loc=z_train[startId:endId, :]
t_loc=t_train[startId:endId :]
dataB_loc=dataB[startBId:endBId, :]
dataI_loc=dataI[startIId:endIId, :]

print("Local dataB id is ", np.shape(dataB_loc), "Local DATAI ", np.shape(dataI_loc), "rank is ", hvd.rank(), "Boundary Points", np.shape(x_loc))

model = ChannelFlow(t_loc, x_loc, y_loc, z_loc,
                    dataB_loc,
                    dataI_loc,
                    layers, Xmin, Xmax,
                    P_x=0.0025, niu=5e-5)

print("End loading")

start_time=time.perf_counter()
start_clock=time.process_time()

#_ = model.train(num_epochs=500, batch_number=batch_num, learning_rate=hvd.size()*1e-3)
# Warm-Up for learning rate

error = model.train(num_epochs=500, batch_number=batch_num, learning_rate=1e-3)
if hvd.rank()==0:
    model.saver.save(model.sess, './model/model.cpkt')
    scipy.io.savemat('Loss.mat', {'Loss': model.lossCol})

error = model.train(num_epochs=500, batch_number=batch_num, learning_rate=1e-4)
if hvd.rank()==0:
    model.saver.save(model.sess, './model/model.cpkt')
    scipy.io.savemat('Loss.mat', {'Loss': model.lossCol})

error = model.train(num_epochs=500, batch_number=batch_num, learning_rate=1e-5)
if hvd.rank()==0:
    model.saver.save(model.sess, './model/model.cpkt')
    scipy.io.savemat('Loss.mat', {'Loss': model.lossCol})

while error > tor:
    error = model.train(num_epochs=200, batch_number=batch_num, learning_rate=1e-5)
    if hvd.rank() == 0:
        model.saver.save(model.sess, './model/model.cpkt')
        scipy.io.savemat('Loss.mat', {'Loss': model.lossCol})

_ = model.train(num_epochs=10, batch_number=batch_num, learning_rate=1e-3)

if hvd.rank()==0:
    model.saver.save(model.sess, './model/model.cpkt')
    scipy.io.savemat('Loss.mat', {'Loss': model.lossCol})
_ = model.train(num_epochs=500, batch_number=batch_num, learning_rate=hvd.size()*1e-3)

if hvd.rank()==0:
    model.saver.save(model.sess, './model/model.cpkt')
    scipy.io.savemat('Loss.mat', {'Loss': model.lossCol})
_ = model.train(num_epochs=500, batch_number=batch_num, learning_rate=hvd.size()*1e-4)

if hvd.rank()==0:
    model.saver.save(model.sess, './model/model.cpkt')
    scipy.io.savemat('Loss.mat', {'Loss': model.lossCol})

error = model.train(num_epochs=500, batch_number=batch_num, learning_rate=hvd.size()*1e-5)
if hvd.rank()==0:
    model.saver.save(model.sess, './model/model.cpkt')
    scipy.io.savemat('Loss.mat', {'Loss': model.lossCol})

while error > tor:
     error = model.train(num_epochs=200, batch_number=batch_num, learning_rate=hvd.size()*1e-5)
     model.saver.save(model.sess, './model/model.cpkt')
     scipy.io.savemat('Loss.mat', {'Loss': model.lossCol})

stop_clock=time.process_time()
stop_time=time.perf_counter()

if hvd.rank()==0:
    print("Process time for training is ", stop_clock - start_clock)
    print("Performance time for training is ", stop_time - start_time)


from __future__ import absolute_import, division, print_function

from NNFlow.session_config.session_config                  import SessionConfig        as NNFlowSessionConfig
from NNFlow.neural_network_training.neural_network_trainer import NeuralNetworkTrainer as NNFlowNeuralNetworkTrainer

def train_neural_network(save_path,
                         model_name,
                         model_id,
                         network_type,
                         hidden_layers,
                         activation_function_name,
                         dropout_keep_probability,
                         l2_regularization_beta,
                         early_stopping_intervall,
                         path_to_training_data_set,
                         path_to_validation_data_set,
                         optimizer,
                         batch_size_training,
                         batch_size_classification   = 200000,
                         session_config              = None
                         ):

    if session_config is None:
        session_config = NNFlowSessionConfig()

    neural_network_trainer = NNFlowNeuralNetworkTrainer()

    neural_network_trainer.train(save_path                   = save_path,
                                 model_name                  = model_name,
                                 model_id                    = model_id,
                                 network_type                = network_type,
                                 hidden_layers               = hidden_layers,
                                 activation_function_name    = activation_function_name,
                                 dropout_keep_probability    = dropout_keep_probability,
                                 l2_regularization_beta      = l2_regularization_beta,
                                 early_stopping_intervall    = early_stopping_intervall,
                                 path_to_training_data_set   = path_to_training_data_set,
                                 path_to_validation_data_set = path_to_validation_data_set,
                                 optimizer                   = optimizer,
                                 batch_size_training         = batch_size_training,
                                 batch_size_classification   = batch_size_classification,
                                 session_config              = session_config
                                 )


import scrapy

class FunSpider(scrapy.Spider):
    name = "blog_scrape"

    start_urls = [
        'https://blog.scrapinghub.com/'
        ]

    def parse(self,response):
        for url in response.css('.read-more .more-link::attr(href)').getall():
            yield scrapy.Request(url, callback=self.parse_blog_data)

        next_link = response.css('.blog-pagination .next-posts-link::attr(href)').get()
        if next_link is not None:
            yield response.follow(next_link, self.parse)

    def parse_blog_data(self,response):
        yield {
                'tite':response.css(".post-header h1 span::text").get(),
                'date':response.css('.post-header .date a::text').get(),
                'author':response.css('.post-header .author a::text').get(),
                'blog_data':response.css('.post-body p::text').getall()

                }


#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright 2018 Snowball Beijing, Inc
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest

from shici.data.shici import ShiCi
from shici.utils import config

class ShiciTest(unittest.TestCase):
    def setUp(self):
        config_info = config.user_config()
        self.shici = ShiCi(config_info)

    def test_get_by_id(self):
        shici_item = self.shici.get(1)
        print(shici_item)
        self.assertFalse(shici_item.empty)

    def test_get_all(self):
        shici_all = self.shici.getall()
        print(shici_all.head(5))
        self.assertFalse(shici_all.empty)


import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt

df=pd.read_csv('df1',index_col=0)
df1=pd.read_csv('df2')
print (df.head())
print (df1)
df.plot.hist() #calling histogram using pandas inbuilt data viz,import seaborn so that image will be gud
plt.show()
df1.plot.area(alpha=0.5) #calling area plot which describes the area of the values.alpha for transparency
plt.show()
df1.plot.bar(stacked=True)  #plots based on the index value,stacks the value one over the other
plt.show()
df['A'].plot.hist(bins=30)
df.plot.line(x=df.index,y='B',figsize=(5,5))  #lineplot accepts x and y value and u can use all the matplotlib func like figsize,lw,ls
df.plot.scatter(x='A',y='B',c='C',cmap='coolwarm')  #two arg as x vs y and passing c as the color value
df.plot.scatter(x='A',y='B',s=df['C']*100)  #passing c values as the size here
df.plot.box()
df2=pd.DataFrame(np.random.randn(1000,2),columns=['A','B'])
print (df2)
df2.plot.hexbin(x='A',y='B',gridsize=20,cmap='coolwarm') #accepts two args
df2.plot.kde()  #kde plot

plt.style.use('ggplot')
df3 = pd.read_csv('df3')
#df3.plot.scatter(x='a',y='b',s=50,c='red')  #setting size and color for the plot
#df3['a'].plot.hist(bins=40,alpha=0.5)
print (df3.iloc[0:30])
#df3.iloc[0:30].plot.area()
#plt.legend(loc=9,bbox_to_anchor=(0.5, -0.1))
plt.show()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"Created on 24 Jun 21:30 2017"
"@author: ratnadeepb"

import pandas as pd
import numpy as np
from scipy.stats import mode
from itertools import product
from sklearn.preprocessing import Imputer
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import OneHotEncoder
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split

#################### Import Data ####################
data = pd.read_csv("/home/ratnadeepb/app/machine_learning/ml_programming/Data/Data.csv")
# data = pd.read_csv("Data/Data.csv")
columns = data.columns  # Reference to feature names

#################### Separate into features and target ####################
X = data[['Country', 'Age', 'Salary']].values
y = data['Purchased'].values

############### Separating Categorical and Numerical Data ############
cat_cls = [0]  # Row numbers in X with categorical data
cls = [cl for cl in range(X.shape[1]) if cl not in cat_cls]

C = np.array([list(elem) for elem in zip(*[X[:, i] for i in cat_cls])])
C = C.reshape(C.shape[0], -1)
N = np.array([list(elem) for elem in zip(*[X[:, i] for i in cls])])
N = N.reshape(N.shape[0], -1)

#################### Handling missing numerical data ####################
# Impute numerical variables
imputer_num = Imputer()
N = imputer_num.fit_transform(N)
c_cls = {}
for cl in cat_cls:
    c_cls[cl] = cat_cls.index(cl)
vals = {cl:mode(C[:,cl])[0][0] for cl in c_cls.values()}

missing_indices = [[rw,cl] for rw, cl in product(range(C.shape[0]), range(C.shape[1]))
                   if pd.isnull(C[rw, cl]) or  C[rw, cl] == 'nan']

for i in missing_indices:
    C[i[0]] = vals[i[1]]

#################### Encoding categorical data ####################
original_tags = []
for cl in list(c_cls.values()):
    original_tags.extend(np.unique(C[:, cl]))
    C_labels = LabelEncoder()
    C = C_labels.fit_transform(C[:, cl])

C = C.reshape(C.shape[0], -1)

categorical = []

for cl in list(c_cls.values()):
    onehotencoder = OneHotEncoder(n_values='auto', categorical_features=[cl])
    temp = onehotencoder.fit_transform(C[:, cl].reshape(-1, 1)).toarray()
    temp = np.delete(temp, -1, 1)  # Dropping the last array
    categorical.extend(temp)

C = np.array(categorical)

#################### Recreate features ####################
X = np.concatenate((C, N), axis=1)

#################### Handling missing response data ####################
response = mode(y)[0][0]

missing_y = [ind for ind in range(y.shape[0])
                   if pd.isnull(y[ind]) or  y[ind] == 'nan']

for i in missing_y:
    y[i] = response

y_labels = LabelEncoder()
y = y_labels.fit_transform(y)

#################### Train -Test Set ####################
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)

#################### Feature Scaling ####################
# Here we are scaling the categorical variables
# The scaling can also be done to the numerical variables only
sc_X = StandardScaler()
X_train = sc_X.fit_transform(X_train)
X_test = sc_X.transform(X_train)


class Verification(object):
    """
    Classe servant à vérifier les dates et la cohérence de toutes les données importées
    """

    def __init__(self):
        """
        initialisation
        """
        self.a_verifier = 2

    def verification_date(self, bilans):
        """
        vérifie les dates de tous les bilans importés
        :param bilans: bilans importés
        :return: 0 si ok, sinon le nombre d'échecs à la vérification
        """
        verif = 0
        for bilan in bilans:
            verif += bilan.verification_date()
        self.a_verifier = 1
        return verif

    def verification_coherence(self, subgeneraux, subedition, subcomptes, submachines, subprestations, bilans,
                               consolidation, force):
        """
        vérifie la cohérence des données importées
        :param subgeneraux: paramètres généraux
        :param subedition: paramètres d'édition
        :param subcomptes: comptes subsides importés
        :param submachines: plafonds machines importées
        :param subprestations: plafonds prestations importées
        :param bilans: bilans importés
        :param consolidation: classe de consolidation des données des bilans
        :param force: comptes forcés importés
        :return: 0 si ok, sinon le nombre d'échecs à la vérification
        """
        verif = 0

        verif += subcomptes.est_coherent(subgeneraux)
        verif += submachines.est_coherent(subcomptes)
        verif += subprestations.est_coherent(subgeneraux, subcomptes)
        verif += consolidation.coherence_bilans(bilans, subcomptes, subgeneraux, subedition, force)
        self.a_verifier = 0
        return verif


# -*- coding: utf-8 -*-
# @Time : 2018/9/12
# @Author : virus
# @File : 1-2.py
# @Desp : 解压可迭代对象赋值给多个变量

record = ('Anna',"Anna@example.com",'773-555-1212','847-555-1212')
# 分解记录
name,email,*phone_numbers = record

print(name)
print(email)
print(phone_numbers)

*trailing,current = [10,8,7,1,9,5,10,3]
# 分解记录

print(trailing)
print(current)

records = [
    ('foo',1,2,3),
    ('bar', 'hello'),
    ('foo',3,4,5)
]

def do_foo(x,y,z):
    print('foo',x,y,z)

def do_bar(s):
    print('bar',s)

for tag, *args in records:
    if tag == 'foo':
        print(args)
        do_foo(*args)
    elif tag == 'bar':
        print(args)
        do_bar(*args)

line = 'nobody:*:-2:-2:Unprivileged User:/var/empty:/usr/bin/false'
# 字符切割

line_arr = line.split(":")

# 分解
uname, *fields, homedir, sh = line_arr

# 输出
print(uname,fields,homedir,sh)

# 解压后丢弃 可以用_或者ign

record = ('ACME',50,123.45,(12,9,2018))

name, *_, (*_, year) = record

# 输出

print(name,year)

items = [1, 10, 7, 4, 5, 9]
head, *tail = items

def sum(items):
    head, *tail = items
    return head + sum(tail) if tail else head

print(sum(items))


# from typing import List, Optional, Union

import cv2
import imutils
import numpy as np

from Object import Bucket, Cube
#from dominant_color import DominantColorDetector

class ObjectsDetector:

    def stupid_detection(self, frame):
        hsv, thresh = self.__prepare_frame(frame, height=self._height, width=self._width)
        cnts = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL,
                                cv2.CHAIN_APPROX_SIMPLE)
        cnts = imutils.grab_contours(cnts)
        for c in cnts:
            if cv2.contourArea(c) < self._min_area:
                continue
            (x, y, w, h) = cv2.boundingRect(c)

            return x + w / 2, y + h / 2

    """
    Constants
    #
    # :param width
    # :param height
    #
    # :param circles_pool_length length of pool with previous circles to prevent fluctuations of contours
    # :param min_area_to_detect min area to detect contours
    #
    # :param circle_factor coefficient to prevent fluctuations of contours with circles
    # :param debug_mode if true call cv2.imshow() to take a look at result
    # :param rotation_factor coefficient for is_rotated. The smaller it is, the more precise would be algorithm.
    # :param daytime preset for day or night light. Should be customize for your environment.
    """

    def __init__(self,
                 # TODO determine this factor
                 rotation_factor=22,
                 width=640,
                 height=360,
                 circles_pool_length=1,
                 min_area_to_detect=400,
                 circle_factor=1.25,
                 debug_mode=False,
                 # TODO change min are
                 min_area_to_compute_mean_colors=100000,
                 daytime="DAY"):

        self._width = width
        self._height = height
        self._pool_size = circles_pool_length
        self._min_area = min_area_to_detect
        self._circle_factor = circle_factor
        self._circles_coords_pool = list()
        self._debug_mode = debug_mode
        self._min_area_for_dominant = min_area_to_compute_mean_colors
        self._daytime = 100 if daytime == "DAY" else 125
        self._rotation_factor = rotation_factor
        #self._dominant_detector = DominantColorDetector(n_clusters=7)

    # crop image
    def _get_subimage_by_pxs(self, image, start, shift):
        cropped_image = []

        for i in range(shift[1]):
            cropped_image.append([])
            for j in range(shift[0]):
                cropped_image[i].append([])
                cropped_image[i][j] = image[start[1] + i][start[0] + j]

        return np.array(cropped_image)

    # Can detect that cube is rotated relatively to the camera
    def is_rotated(self, frame):
        hsv, thresh = self.__prepare_frame(frame, height=self._height, width=self._width)

        cv2.imshow("thresh", thresh)
        # find cube
        cnts = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL,
                                cv2.CHAIN_APPROX_SIMPLE)
        cnts = imutils.grab_contours(cnts)

        for c in cnts:
            if cv2.contourArea(c) < self._min_area:
                continue
            (x, y, w, h) = cv2.boundingRect(c)
            # take a small image of it
            area = self._get_subimage_by_pxs(thresh, start=(x, y), shift=(w, h))
            print(area.shape)
            # cv2.imshow('result', area)
            # compute a factor
            factor = self.__get_difference(area)

            if cv2.waitKey(25) & 0xFF == ord('q'):
                return None

            if factor >= self._rotation_factor:
                return True
            else:
                return False

    # using absdiff to get all pixels that deviate from reference
    def __get_difference(self, frame_thresh):
        reference = cv2.imread('./src/reference.png')
        shape = min(min((frame_thresh.shape, reference.shape[:-1])))
        frame = cv2.resize(reference, (shape, shape))
        hsv, reference_thresh = self.__prepare_frame(reference, width=shape, height=shape)
        frame_thresh = cv2.resize(frame_thresh, (shape, shape))

        frame_delta = cv2.absdiff(frame_thresh, reference_thresh)
        frame_delta = cv2.dilate(frame_delta, None, iterations=2)

        # count pixels
        counter = 0
        for row in frame_delta:
            for pixel in row:
                if pixel == 255:
                    counter += 1
        # devide by shape to get relation with size of the cropped image
        return counter / shape

    # function that resize and make hsv and thresh copy of a frame
    def __prepare_frame(self, frame, width, height):

        hsv = cv2.cvtColor(frame, cv2.COLOR_RGB2HSV)
        blur = cv2.GaussianBlur(hsv, (19, 19), 0)

        gray = cv2.cvtColor(blur, cv2.COLOR_BGR2GRAY)
        ret, thresh = cv2.threshold(gray, self._daytime, 255, 1)
        thresh = cv2.bitwise_not(thresh)
        return hsv, thresh

    # Check if this object is circle and we shouldn't detect it
    # iterate though pool with circles and check a distance between coordinates and circles
    def _circle_check(self, pool, coords):
        if pool:
            for circles in pool:
                for circle in circles:
                    if max(abs(np.array(circle.get_position()) - np.array(coords))) < \
                            self._circle_factor * circle.get_radius():
                        return False

        return True

    """
    # :brief Color detection by hue, check HSV wiki for details. Take a 20x20 px square and count mean H component
    # then translate it to 360 points scale. Then just compare it with table (wiki)
    # :note if a color cannot be detected return 'no_color'
    # :param image just image where needed to detect color

    # :param x coordinate of a dot where to detect color
    # :param y

    """

    def _get_color(self, image, x, y):
        # if contour nearby edge use square with smaller side
        if x == self._width:
            x -= 1
        if y == self._height:
            y -= 1

        area = 0
        for i in range(10):
            if x - i < 0 or x + i >= self._width:
                break
            elif y - i < 0 or y + i >= self._height:
                break
            else:
                area = i

        avg_color = 0
        # Count mean
        for i in range(x - area, x + area):
            for j in range(y - area, y + area):
                avg_color += image[j, i, 0]

        # translate result
        avg_color = avg_color / area ** 2 if area != 0 else image[y, x, 0]
        hsv_color = 255 * avg_color / 360
        # compare
        if 0 < hsv_color <= 13 or 330 <= hsv_color:
            return 'RED'
        elif 13 <= hsv_color < 35:
            return 'BLUE'
        elif 35 <= hsv_color < 70:
            return 'ORANGE'
        elif 70 <= hsv_color < 180:
            return 'GREEN'
        elif 180 <= hsv_color < 300:
            return 'YELLOW'
        else:
            return 'no_color'

    """
    # :brief Detect objects(bucket or cube) in image
    # :param frame take an image (RGB) were need to detect objects(bucket or cube).
    """

    def get_objects(self, frame):
        if frame is None:
            raise ValueError('Frame is none')

        frame = cv2.resize(frame, (self._width, self._height))

        # Font
        font = cv2.FONT_HERSHEY_SIMPLEX
        fontScale = 0.5
        fontColor = (255, 255, 255)

        hsv, thresh = self.__prepare_frame(frame, height=self._height, width=self._width)

        # Store here all objects
        objects_in_frame = []

        """
        # First detect only buckets. It's more simple to detect circle in frame then
        # we iterate each contour and check if it is circle or not.
        # We conclude a contour a circle if it has the center nearby a center of already detected circles.
        # `get_color` - function that make a verification for contour
        """

        #  Pool must not be bigger than max size
        if self._circles_coords_pool is not None and len(self._circles_coords_pool) >= self._pool_size:
            self._circles_coords_pool.pop()

        # Get circles
        circles = cv2.HoughCircles(cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY), cv2.HOUGH_GRADIENT, dp=1.5, minDist=1000,
                                   param1=65, param2=90)
        if circles is not None:
            # convert the (x, y) coordinates and radius of the circles to integers
            circles = np.round(circles[0, :]).astype("int")
            # loop over the (x, y) coordinates and radius of the circles
            for (x, y, r) in circles:
                # draw the circle in the output image, then draw a rectangle
                # corresponding to the center of the circle
                cv2.circle(frame, (x, y), r, (0, 255, 0), 4)
                circles = list()
                bucket = Bucket(position=(x, y), radius=r, color=self._get_color(hsv, x, y))
                cv2.putText(img=frame, text=str(bucket),
                            org=(x, y),
                            fontFace=font,
                            fontScale=fontScale,
                            color=fontColor)
                circles.append(bucket)
                objects_in_frame.append(bucket)
            self._circles_coords_pool.append(circles)

        cv2.imshow('thresh', thresh)
        # Check all contours
        cnts = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL,
                                cv2.CHAIN_APPROX_SIMPLE)
        cnts = imutils.grab_contours(cnts)

        for c in cnts:
            # if the contour is too small, ignore it
            (x, y, w, h) = cv2.boundingRect(c)
            if cv2.contourArea(c) < self._min_area:
                continue
            elif cv2.contourArea(c) > self._min_area_for_dominant:
                # TODO find dominant color
                #self._reveal_clusters(frame, position=(x, y), shift=(w, h))
                pass
                # area = self._get_image_by_px(frame, [x, y], [x + w, y + h])
                # cv2.imshow("cropped", area)

            # area = self._get_image_by_px(frame, [x, y], [x + w, y + h])
            # Validation with circle_check
            if self._circle_check(self._circles_coords_pool, [x, y]):
                cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2)
                center_x = int(x + w / 2)
                center_y = int(y + h / 2)
                cube = Cube(position=(center_x, center_y), color=self._get_color(hsv, x=center_x, y=center_y))
                cv2.putText(img=frame, text=str(cube),
                            org=(x, y),
                            fontFace=font,
                            fontScale=fontScale,
                            color=fontColor)
                objects_in_frame.append(cube)

        cv2.imshow('result', frame)

        if cv2.waitKey(25) & 0xFF == ord('q'):
            return None

        return objects_in_frame, frame


# Copyright (C) 2020 Cancer Care Associates

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import base64
import functools
import os
import pathlib
import subprocess
import sys
from datetime import datetime

from pymedphys._imports import imageio
from pymedphys._imports import numpy as np
from pymedphys._imports import pandas as pd
from pymedphys._imports import plt
from pymedphys._imports import streamlit as st
from pymedphys._imports import streamlit_ace, timeago, tomlkit

import pymedphys
from pymedphys._streamlit.utilities import exceptions as _exceptions
from pymedphys._streamlit.utilities import misc as st_misc

from . import _config, _dicom, _icom, _monaco, _mosaiq, _trf

DATA_OPTION_LABELS = {
    "monaco": "Monaco tel.1 filepath",
    "dicom": "DICOM RTPlan file upload",
    "icom": "iCOM record timestamp",
    "trf": "Linac Backup `.trf` filepath",
    "mosaiq": "Mosaiq SQL query",
}

LEAF_PAIR_WIDTHS = (10,) + (5,) * 78 + (10,)
MAX_LEAF_GAP = 410
GRID_RESOLUTION = 1
GRID = pymedphys.metersetmap.grid(
    max_leaf_gap=MAX_LEAF_GAP,
    grid_resolution=GRID_RESOLUTION,
    leaf_pair_widths=LEAF_PAIR_WIDTHS,
)
COORDS = (GRID["jaw"], GRID["mlc"])

def sidebar_overview():
    overview_placeholder = st.sidebar.empty()

    def set_overview_data(patient_id, patient_name, total_mu):
        overview_placeholder.markdown(
            f"Patient ID: `{patient_id}`\n\n"
            f"Patient Name: `{patient_name}`\n\n"
            f"Total MU: `{total_mu}`"
        )

    return set_overview_data

def get_most_recent_file_and_print(linac_id, filepaths):
    if not isinstance(filepaths, list):
        raise ValueError("Filepaths needs to be a list")

    try:
        latest_filepath = max(filepaths, key=os.path.getmtime)
    except ValueError:
        st.sidebar.markdown(f"{linac_id}: `Never`")
        return

    most_recent = datetime.fromtimestamp(os.path.getmtime(latest_filepath))
    now = datetime.now()

    if most_recent > now:
        most_recent = now

    human_readable = timeago.format(most_recent, now)

    st.sidebar.markdown(f"{linac_id}: `{human_readable}`")

def icom_status(linac_id, icom_directory):
    filepaths = list(pathlib.Path(icom_directory).glob("*.txt"))
    get_most_recent_file_and_print(linac_id, filepaths)

def trf_status(linac_id, backup_directory):
    directory = pathlib.Path(backup_directory).joinpath(linac_id)
    filepaths = list(directory.glob("*.zip"))
    get_most_recent_file_and_print(linac_id, filepaths)

def show_status_indicators(config):
    if st.sidebar.button("Check status of iCOM and backups"):
        try:
            linac_icom_live_stream_directories = (
                _config.get_icom_live_stream_directories(config)
            )
            linac_indexed_backups_directory = _config.get_indexed_backups_directory(
                config
            )
        except KeyError:
            st.sidebar.write(
                _exceptions.ConfigMissing(
                    "iCOM and/or TRF backup configuration is missing. "
                    "Unable to show status."
                )
            )

            return

        linac_ids = list(linac_icom_live_stream_directories.keys())

        st.sidebar.markdown(
            """
            ## Last recorded iCOM stream
            """
        )

        for linac_id, icom_directory in linac_icom_live_stream_directories.items():
            icom_status(linac_id, icom_directory)

        st.sidebar.markdown(
            """
            ## Last indexed backup
            """
        )

        for linac_id in linac_ids:
            trf_status(linac_id, linac_indexed_backups_directory)

def display_deliveries(deliveries):
    if not deliveries:
        return 0

    st.write(
        """
        #### Overview of selected deliveries
        """
    )

    data = []
    for delivery in deliveries:
        num_control_points = len(delivery.mu)

        if num_control_points != 0:
            total_mu = delivery.mu[-1]
        else:
            total_mu = 0

        data.append([total_mu, num_control_points])

    columns = ["MU", "Number of Data Points"]
    df = pd.DataFrame(data=data, columns=columns)
    st.write(df)

    total_mu = round(df["MU"].sum(), 1)

    st.write(f"Total MU: `{total_mu}`")

    return total_mu

def get_input_data_ui(
    overview_updater_map,
    data_method_map,
    default_method,
    key_namespace,
    advanced_mode,
    **previous_results,
):
    if advanced_mode:
        data_method_options = list(data_method_map.keys())
        data_method = st.selectbox(
            "Data Input Method",
            data_method_options,
            index=data_method_options.index(default_method),
        )

    else:
        data_method = default_method

    results = data_method_map[data_method](  # type: ignore
        key_namespace=key_namespace,
        advanced_mode=advanced_mode,
        **previous_results,
    )

    try:
        total_mu = round(display_deliveries(results["deliveries"]), 1)
    except KeyError:
        total_mu = 0

    try:
        patient_id = results["patient_id"]
    except KeyError:
        patient_id = ""

    try:
        patient_name = results["patient_name"]
    except KeyError:
        patient_name = ""

    overview_updater_map[key_namespace](patient_id, patient_name, total_mu)

    return results

@st.cache
def to_tuple(array):
    return tuple(map(tuple, array))

def plot_gamma_hist(gamma, percent, dist):
    valid_gamma = gamma[~np.isnan(gamma)]

    plt.hist(valid_gamma, 50, density=True)
    pass_ratio = np.sum(valid_gamma <= 1) / len(valid_gamma)

    plt.title(
        "Local Gamma ({0}%/{1}mm) | Percent Pass: {2:.2f} % | Mean Gamma: {3:.2f} | Max Gamma: {4:.2f}".format(
            percent, dist, pass_ratio * 100, np.mean(valid_gamma), np.max(valid_gamma)
        )
    )

def plot_and_save_results(
    reference_metersetmap,
    evaluation_metersetmap,
    gamma,
    gamma_options,
    png_record_directory,
    header_text="",
    footer_text="",
):
    reference_filepath = png_record_directory.joinpath("reference.png")
    evaluation_filepath = png_record_directory.joinpath("evaluation.png")
    diff_filepath = png_record_directory.joinpath("diff.png")
    gamma_filepath = png_record_directory.joinpath("gamma.png")

    diff = evaluation_metersetmap - reference_metersetmap

    imageio.imwrite(reference_filepath, reference_metersetmap)
    imageio.imwrite(evaluation_filepath, evaluation_metersetmap)
    imageio.imwrite(diff_filepath, diff)
    imageio.imwrite(gamma_filepath, gamma)

    largest_metersetmap = np.max(
        [np.max(evaluation_metersetmap), np.max(reference_metersetmap)]
    )
    largest_diff = np.max(np.abs(diff))

    widths = [1, 1]
    heights = [0.5, 1, 1, 1, 0.4]
    gs_kw = dict(width_ratios=widths, height_ratios=heights)

    fig, axs = plt.subplots(5, 2, figsize=(10, 16), gridspec_kw=gs_kw)
    gs = axs[0, 0].get_gridspec()

    for ax in axs[0, 0:]:
        ax.remove()

    for ax in axs[1, 0:]:
        ax.remove()

    for ax in axs[4, 0:]:
        ax.remove()

    ax_header = fig.add_subplot(gs[0, :])
    ax_hist = fig.add_subplot(gs[1, :])
    ax_footer = fig.add_subplot(gs[4, :])

    ax_header.axis("off")
    ax_footer.axis("off")

    ax_header.text(0, 0, header_text, ha="left", wrap=True, fontsize=21)
    ax_footer.text(0, 1, footer_text, ha="left", va="top", wrap=True, fontsize=6)

    plt.sca(axs[2, 0])
    pymedphys.metersetmap.display(
        GRID, reference_metersetmap, vmin=0, vmax=largest_metersetmap
    )
    axs[2, 0].set_title("Reference MetersetMap")

    plt.sca(axs[2, 1])
    pymedphys.metersetmap.display(
        GRID, evaluation_metersetmap, vmin=0, vmax=largest_metersetmap
    )
    axs[2, 1].set_title("Evaluation MetersetMap")

    plt.sca(axs[3, 0])
    pymedphys.metersetmap.display(
        GRID, diff, cmap="seismic", vmin=-largest_diff, vmax=largest_diff
    )
    plt.title("Evaluation - Reference")

    plt.sca(axs[3, 1])
    pymedphys.metersetmap.display(GRID, gamma, cmap="coolwarm", vmin=0, vmax=2)
    plt.title(
        "Local Gamma | "
        f"{gamma_options['dose_percent_threshold']}%/"
        f"{gamma_options['distance_mm_threshold']}mm"
    )

    plt.sca(ax_hist)
    plot_gamma_hist(
        gamma,
        gamma_options["dose_percent_threshold"],
        gamma_options["distance_mm_threshold"],
    )

    return fig

@st.cache(hash_funcs={pymedphys.Delivery: hash})
def calculate_metersetmap(delivery):
    return delivery.metersetmap(
        max_leaf_gap=MAX_LEAF_GAP,
        grid_resolution=GRID_RESOLUTION,
        leaf_pair_widths=LEAF_PAIR_WIDTHS,
    )

def calculate_batch_metersetmap(deliveries):
    metersetmap = calculate_metersetmap(deliveries[0])

    for delivery in deliveries[1::]:
        metersetmap = metersetmap + calculate_metersetmap(delivery)

    return metersetmap

@st.cache
def calculate_gamma(reference_metersetmap, evaluation_metersetmap, gamma_options):
    gamma = pymedphys.gamma(
        COORDS,
        to_tuple(reference_metersetmap),
        COORDS,
        to_tuple(evaluation_metersetmap),
        **gamma_options,
    )

    return gamma

def advanced_debugging(config):
    st.sidebar.markdown("# Advanced Debugging")
    if st.sidebar.button("Compare Baseline to Output Directory"):
        st.write(
            """
            ## Comparing Results to Baseline
            """
        )

        baseline_directory = pathlib.Path(
            config["debug"]["baseline_directory"]
        ).resolve()

        png_baseline_directory = baseline_directory.joinpath("png")

        baseline_png_paths = [
            path for path in (png_baseline_directory.rglob("*")) if path.is_file()
        ]

        relative_png_paths = [
            path.relative_to(png_baseline_directory) for path in baseline_png_paths
        ]

        output_dir = pathlib.Path(config["output"]["png_directory"]).resolve()

        evaluation_png_paths = [
            output_dir.joinpath(path) for path in relative_png_paths
        ]

        for baseline, evaluation in zip(baseline_png_paths, evaluation_png_paths):

            st.write(f"### {baseline.parent.name}/{baseline.name}")

            st.write(f"`{baseline}`\n\n**vs**\n\n`{evaluation}`")

            baseline_image = imageio.imread(baseline)

            try:
                evaluation_image = imageio.imread(evaluation)
            except FileNotFoundError as e:
                st.write(
                    """
                    #### File was not found
                    """
                )

                st.write(e)

                st.write(
                    f"""
                    For debugging purposes, here are all the files that
                    were found within {str(output_dir)}
                    """
                )

                st.write(
                    [str(path) for path in output_dir.rglob("*") if path.is_file()]
                )

                return

            agree = np.allclose(baseline_image, evaluation_image)
            st.write(f"Images Agree: `{agree}`")

def run_calculation(
    reference_results,
    evaluation_results,
    gamma_options,
    escan_directory: pathlib.Path,
    png_output_directory: pathlib.Path,
):
    st.write("Calculating Reference MetersetMap...")
    reference_metersetmap = calculate_batch_metersetmap(reference_results["deliveries"])

    st.write("Calculating Evaluation MetersetMap...")
    evaluation_metersetmap = calculate_batch_metersetmap(
        evaluation_results["deliveries"]
    )

    st.write("Calculating Gamma...")
    gamma = calculate_gamma(
        reference_metersetmap, evaluation_metersetmap, gamma_options
    )

    patient_id = reference_results["patient_id"]

    st.write("Creating figure...")
    output_base_filename = (
        f"{patient_id} {reference_results['identifier']} vs "
        f"{evaluation_results['identifier']}"
    )
    pdf_filepath = str(
        escan_directory.joinpath(f"{output_base_filename}.pdf").resolve()
    )
    png_record_directory = png_output_directory.joinpath(output_base_filename)
    png_record_directory.mkdir(exist_ok=True, parents=True)
    png_filepath = str(png_record_directory.joinpath("report.png").resolve())

    try:
        patient_name_text = f"Patient Name: {reference_results['patient_name']}\n"
    except KeyError:
        patient_name_text = ""

    header_text = (
        f"Patient ID: {patient_id}\n"
        f"{patient_name_text}"
        f"Reference: {reference_results['identifier']}\n"
        f"Evaluation: {evaluation_results['identifier']}\n"
    )

    reference_path_strings = "\n    ".join(
        [str(path.resolve()) for path in reference_results["data_paths"]]
    )
    evaluation_path_strings = "\n    ".join(
        [str(path.resolve()) for path in evaluation_results["data_paths"]]
    )

    footer_text = (
        f"reference path(s): {reference_path_strings}\n"
        f"evaluation path(s): {evaluation_path_strings}\n"
        f"png record: {png_filepath}"
    )

    fig = plot_and_save_results(
        reference_metersetmap,
        evaluation_metersetmap,
        gamma,
        gamma_options,
        png_record_directory,
        header_text=header_text,
        footer_text=footer_text,
    )

    fig.tight_layout()

    st.write("## Results")
    st.pyplot(fig)

    st.write("## Saving reports")
    st.write("### PNG")
    st.write("Saving figure as PNG...")
    plt.savefig(png_filepath, dpi=100)
    st.write(f"Saved:\n\n`{png_filepath}`")
    convert_png_to_pdf(png_filepath, pdf_filepath)

def convert_png_to_pdf(png_filepath, pdf_filepath):
    st.write("### PDF")
    st.write("Converting PNG to PDF...")

    try:
        subprocess.check_call(
            f'magick convert "{png_filepath}" "{pdf_filepath}"', shell=True
        )
        success = True
    except subprocess.CalledProcessError:
        try:
            subprocess.check_call(
                f'convert "{png_filepath}" "{pdf_filepath}"', shell=True
            )
            success = True
        except subprocess.CalledProcessError:
            success = False

    if success:
        st.write(f"Created:\n\n`{pdf_filepath}`")

        with open(pdf_filepath, "rb") as f:
            pdf_contents = f.read()

        pdf_filename = pathlib.Path(pdf_filepath).name

        pdf_b64 = base64.b64encode(pdf_contents).decode()
        href = f"""
            <a href="data:file/zip;base64,{pdf_b64}" download='{pdf_filename}'>
                Click to download {pdf_filename}
            </a>
        """
        st.markdown(href, unsafe_allow_html=True)

    else:
        if sys.platform == "win32":
            url_hash_parameter = "#windows"
        else:
            url_hash_parameter = ""

        download_url = (
            f"https://imagemagick.org/script/download.php{url_hash_parameter}"
        )

        st.write(
            _exceptions.UnableToCreatePDF(
                "Please install Image Magick to create PDF reports "
                f"<{download_url}>."
            )
        )

def main():
    st.write(
        """
        Tool to compare the MetersetMap between planned and delivery.
        """
    )

    st.sidebar.markdown(
        """
        # Configuration Choice
        """
    )

    config_options = list(_config.CONFIG_OPTIONS.keys())

    try:
        _config.get_config(config_options[0])
    except FileNotFoundError:
        config_options.pop(0)

    config_mode = st.sidebar.radio("Config Mode", options=config_options)
    config = _config.get_config(config_mode)

    show_config = st.sidebar.checkbox("Show/edit config", False)
    if show_config:
        st.write("## Configuration")
        config = tomlkit.loads(
            streamlit_ace.st_ace(value=tomlkit.dumps(config), language="toml")
        )

    st.sidebar.markdown(
        """
        # MetersetMap Overview
        """
    )

    st.sidebar.markdown(
        """
        ## Reference
        """
    )

    set_reference_overview = sidebar_overview()

    st.sidebar.markdown(
        """
        ## Evaluation
        """
    )

    set_evaluation_overview = sidebar_overview()

    overview_updater_map = {
        "reference": set_reference_overview,
        "evaluation": set_evaluation_overview,
    }

    st.sidebar.markdown(
        """
        # Status indicators
        """
    )

    show_status_indicators(config)

    st.sidebar.markdown(
        """
        # Advanced options

        Enable advanced functionality by ticking the below.
        """
    )
    advanced_mode = st.sidebar.checkbox("Run in Advanced Mode")

    gamma_options = _config.get_gamma_options(config, advanced_mode)

    data_option_functions = {
        "monaco": _monaco.monaco_input_method,
        "dicom": _dicom.dicom_input_method,
        "icom": _icom.icom_input_method,
        "trf": _trf.trf_input_method,
        "mosaiq": _mosaiq.mosaiq_input_method,
    }

    default_reference_id = config["data_methods"]["default_reference"]
    default_evaluation_id = config["data_methods"]["default_evaluation"]
    available_data_methods = config["data_methods"]["available"]

    default_reference = DATA_OPTION_LABELS[default_reference_id]
    default_evaluation = DATA_OPTION_LABELS[default_evaluation_id]

    data_method_map = {}
    for method in available_data_methods:
        data_method_map[DATA_OPTION_LABELS[method]] = functools.partial(
            data_option_functions[method], config=config
        )

    st.write(
        """
        ## Selection of data to compare
        """
    )

    st.write("---")

    ref_col, eval_col = st.beta_columns(2)

    with ref_col:
        st.write(
            """
            ### Reference
            """
        )

        reference_results = get_input_data_ui(
            overview_updater_map,
            data_method_map,
            default_reference,
            "reference",
            advanced_mode,
        )

    with eval_col:
        st.write(
            """
            ### Evaluation
            """
        )

        evaluation_results = get_input_data_ui(
            overview_updater_map,
            data_method_map,
            default_evaluation,
            "evaluation",
            advanced_mode,
            **reference_results,
        )

    st.write("---")

    st.write(
        """
        ## Output Locations
        """
    )

    st.write(
        """
        ### eSCAN Directory

        The location to save the produced pdf report.
        """
    )

    default_site = evaluation_results.get("site", None)
    if default_site is None:
        default_site = reference_results.get("site", None)

    _, escan_directory = st_misc.get_site_and_directory(
        config,
        "eScan Site",
        "escan",
        default=default_site,
        key="escan_export_site_picker",
    )

    escan_directory = pathlib.Path(os.path.expanduser(escan_directory)).resolve()

    if advanced_mode:
        st.write(escan_directory)

    default_png_output_directory = config["output"]["png_directory"]

    if advanced_mode:
        st.write(
            """
            ### Image record

            Path to save the image of the results for posterity
            """
        )

        png_output_directory = pathlib.Path(
            st.text_input("png output directory", default_png_output_directory)
        )
        st.write(png_output_directory.resolve())

    else:
        png_output_directory = pathlib.Path(default_png_output_directory)

    png_output_directory = pathlib.Path(
        os.path.expanduser(png_output_directory)
    ).resolve()

    st.write(
        """
        ## Calculation
        """
    )

    if st.button("Run Calculation"):

        st.write(
            """
            ### MetersetMap usage warning
            """
        )

        st.warning(pymedphys.metersetmap.WARNING_MESSAGE)

        st.write(
            """
            ### Calculation status
            """
        )

        run_calculation(
            reference_results,
            evaluation_results,
            gamma_options,
            escan_directory,
            png_output_directory,
        )

    if advanced_mode:
        advanced_debugging(config)


import pandas as pd
import numpy as np
from scipy.stats import chi2_contingency
from itertools import product, combinations
from statsmodels.stats.multitest import multipletests
from tqdm import tqdm

np.seterr(divide='ignore', invalid='ignore')

def generate_permutations(hb, x_variables):
    """Helper function for crosstab(). Returns list containining permutations of variables in dicts"""
    var_combos = [set for r in range(1, len(x_variables)+1) for set in combinations(x_variables, r)]
    perms = []
    for combo in var_combos:
        d = {col: hb[col].unique() for col in combo}
        perms = perms + [dict(zip(d, v)) for v in product(*d.values())]
    return perms

def return_row_for_permutation(hb, perm, x_variables, y_variable, y_values, date):
    """Helper function for crosstab(). Returns row of crosstab stats for a variable permutation"""
    n_ids = hb[hb.report_date == date].scrambled_id.nunique()
    subset = control = hb[hb.report_date == date]
    row = dict(perm)
    for key, value in row.items():
        if pd.isnull(value):
            subset = subset[subset[key].isnull()]
        else:
            subset = subset[subset[key] == value]
    control = control[~control.scrambled_id.isin(subset.scrambled_id.unique())]
    sub_total_count = subset.scrambled_id.nunique()
    control_total_count = control.scrambled_id.nunique()
    assert n_ids == (sub_total_count + control_total_count), date+str(perm)
    row.update({"permutation": perm, "date": date,
                "sub_total_pop": sub_total_count,
                "control_total_pop": control_total_count})
    chi2_obs = np.empty(shape=(len(y_values), 2))
    for index, val in enumerate(y_values):
        sub_count = subset[subset[y_variable] == val].scrambled_id.nunique()
        control_count = control[control[y_variable] == val].scrambled_id.nunique()
        row.update({"sub_"+y_variable+"_"+str(val): sub_count})
        row.update({"control_"+y_variable+"_"+str(val): control_count})

        row.update({"sub_"+y_variable+"_"+str(val)+"_rate": np.float32(sub_count)/sub_total_count})
        row.update({"control_"+y_variable+"_"+str(val)+"_rate": np.float32(control_count)/control_total_count})

        chi2_obs[index, 0] = sub_count
        chi2_obs[index, 1] = control_count
    p_value = np.NaN if (~chi2_obs.any(axis=0)).any() else chi2_contingency(chi2_obs)[1]
    row.update({"p_value": p_value})
    return row

def crosstab(hb, x_columns, y_variable, dates):
    """Returns crosstab dataframe. Pass list of categorical columns as x_columns
    and list of dates to produces crosstabs for. The y_variable should be categorical.

    Example use: crosstab(hb, ['counter_dhp', 'main_claimant_gender'], 'counter_homeless', ['2012-01-01', '2013-01-01'])
    """
    y_values = hb[y_variable].unique()
    perms = generate_permutations(hb, x_columns)
    result = pd.DataFrame()
    for p in tqdm(product(perms, dates), total=len(perms)*len(dates)):
        row = return_row_for_permutation(hb, p[0], x_columns, y_variable, y_values, p[1])
        result = result.append(row, ignore_index=True)
    result['p_value'] = multipletests(result.p_value, method='bonferroni')[1]
    return result

def simplify_counter_homeless_crosstab(crosstab, min_pop=10, p_value_max=0.05):
    """Function to process crosstab result to pull out most useful sumary results."""
    crosstab = crosstab[(crosstab.control_total_pop >= min_pop) & (crosstab.sub_total_pop >= min_pop)].copy()
    crosstab = crosstab[crosstab.p_value < p_value_max]
    crosstab['sub_homeless_pc'] = round(100*crosstab.sub_counter_homeless_True_rate,2)
    crosstab['control_homeless_pc'] = round(100*crosstab.control_counter_homeless_True_rate,1)
    crosstab = crosstab[['permutation', 'date',
                         'sub_total_pop', 'sub_homeless_pc',
                         'control_total_pop', 'control_homeless_pc',
                         'p_value']]
    return crosstab.sort_values('p_value')


# Example Array Job Program
import time
# To access $PBS_ARRAY_INDEX
import sys

# subjob task
def serial_task(idx):
    time.sleep(0.5)
    filename = f'../../Output/Array/Array.{idx}.txt'
    with open(filename,'w') as f:
    	for i in range(idx+1):
    		f.write(f'{i**2}\n')
    return

def main():
    # Get subjob idx from python arg vars
    program_name, pbs_array_idx = sys.argv
    pbs_array_idx = int(pbs_array_idx)
    # Code performed for each subjob
    start = time.time()
    serial_task(pbs_array_idx)
    runtime = time.time()-start
    print(f'Time: {runtime}s')

if __name__=='__main__':
    main()


# Modules
import socket,time,threading,sys
from optparse import OptionParser

# Usage text
usage_text = """

    Usage: scan.py [kwargs]

    ARGUMENTS (kwargs):

      REQUIRED:
        ------------------------------------------------
        --ip:
          The IP of the 'victim'.
        ------------------------------------------------

      OPTIONAL (One of these is needed):
        ------------------------------------------------
        -p or --portscan [port limit]:
          Just a simple scan up until the given port number.
        ------------------------------------------------
        -w or --wp:
          Scan through all of the well known ports.
          If used in GUI, leave the port input blank.
        ------------------------------------------------
        -s or --specificport [port]:
          Scans a specific port.
        ------------------------------------------------
        -f or --file [filename]:
          Write scan reports to a file.
        ------------------------------------------------

      HELP:
       -h or --help:

      """
# Usage
def usage():
  print(usage_text)
  sys.exit()

#Setting up the options for the terminal
parser = OptionParser()
parser.set_conflict_handler("resolve")
parser.add_option("-h", "--help", dest="help", action="store_true")
parser.add_option("--ip", dest="IP")
parser.add_option("-p", "--portscan", dest="port_scan")
parser.add_option("-w", "--wp", dest="known_ports", action="store_true")
parser.add_option("-s", "--specificport", dest="specific_port")
parser.add_option("-f","--file", dest="fileName")
(options, args) = parser.parse_args()

# Main class
class Scanner():
  def __init__(self):
    self.options = options.__dict__
    self.IP = str(options.IP)
    self.openPorts = []
    self.methodCount = []
    self.trues = []
    self.methods = ["port_scan", "known_ports", "specific_port"]
    self.wellKnownports = [20, 21, 22, 23, 25, 53, 67, 68, 80, 88, 101, 110, 111, 113, 115, 119, 135, 137, 138, 139, 143, 161, 194,
    443, 445, 464, 512, 513, 531, 548, 626, 660, 687, 749, 751, 752, 873, 989, 990, 992, 993, 995, 1080, 1243, 1433, 1434,
    1723, 1985, 2432, 2336, 3306, 3307, 3283, 3389, 5900, 8080, 9050, 9051, 9010, 33568, 40421, 60008]
    self.outputText = []

  # Updating info
  def update_screen(self):
    if __name__ == "__main__":
      print(self.outputText[-1])
    else:
      pass

  # Making threads
  def make_thread(self,target,**kwargs):
    thread = threading.Thread(target=target, args=(kwargs["ip"], kwargs["port"],))
    thread.daemon = True
    thread.start()
    time.sleep(0.01)

  # Method for executing functions
  def LoopAndThread(self, ip):
    # Check args
    for i in self.options:
      if (i in self.methods) and (self.options[i] is not None):
        self.trues.append(i)
      if len(self.trues) > 1:
        print("Illegal amount of arguments")
        usage()

    # Loop and execute
    self.startTime = time.time()
    self.outputText.append(f"\nStart time of scan: {time.ctime()}\nHost: {ip}\n\nPORT STATE\n")
    self.update_screen()

    for i in self.options:
      # Port scanning
      if (i == "port_scan") and (self.options["port_scan"] is not None):
        self.options[i] = int(self.options[i])
        self.options[i] += 1
        for port in range(int(self.options[i])):
          self.make_thread(self.portScan,ip=ip,port=port)
        self.openPorts = sorted(set(self.openPorts))
        for i in self.openPorts:
          self.outputText.append(f"{i}  open")
          self.update_screen()
        self.outputText.append(f"\nScan is done: {ip} scanned in  {(time.time() - self.startTime):.3} seconds")
        self.update_screen()
      # Well known port scanner
      elif (i == "known_ports") and (self.options["known_ports"] is True):
        for port in self.wellKnownports:
          self.make_thread(self.wellKnownPortScan, ip=ip, port=port)
        self.openPorts = sorted(set(self.openPorts))
        for i in self.openPorts:
          self.outputText.append(f"{i}  open")
          self.update_screen()
        self.outputText.append(f"\nScan is done: {ip} scanned in  {(time.time() - self.startTime):.3} seconds")
        self.update_screen()
      # Specific port scan
      elif (i == "specific_port") and (self.options["specific_port"] is not None):
        for l in range(3):
          self.make_thread(self.SpecificPortScan, ip=ip,port=int(self.options[i]),)
        self.openPorts = sorted(set(self.openPorts))
        for i in self.openPorts:
          self.outputText.append(f"{i}  open")
          self.update_screen()
        if len(self.openPorts) == 0:
          self.outputText.append(f"{self.options[i]} closed")
          self.update_screen()
        self.outputText.append(f"\nScan is done: {ip} scanned in  {(time.time() - self.startTime):.3} seconds")
        self.update_screen()

    # Writing to file if needed
    if options.fileName:
      f = open(str(options.fileName), "a")
      f.write(self.outputText)

  # Simple port scan method
  def portScan(self, ip ,port):
    self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    self.result = self.sock.connect_ex((ip, port))
    if self.result == 0:
      if port in self.openPorts:
        pass
      else:
        self.openPorts.append(port)
    else:
      pass

  # Well known port scan
  def wellKnownPortScan(self, ip, port):
    self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    self.result = self.sock.connect_ex((ip, port))
    if self.result == 0:
      self.openPorts.append(port)
    else:
      pass

  #Specific Port scan
  def SpecificPortScan(self, ip, port):
    self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    self.result = self.sock.connect_ex((ip, port))
    if self.result == 0:
      self.openPorts.append(port)

# On-start
if __name__ ==  "__main__":
  #Checking for needed arguments
  if options.help:
    usage()
  if options.IP is None:
    usage()
  if (options.port_scan is None) and (options.known_ports is None) and (options.specific_port is None):
    usage()
  # Starting class
  Scanning = Scanner()
  Scanning.LoopAndThread(Scanning.IP)


import random

scores_list = [1000]

def high_score():
    return min(scores_list)

def random_number():
    return random.randint(1, 10)

def begin_game():
    if len(scores_list) >1:
        scores_list.sort
        print('-' * 40)
        print("\nThe current HIGHSCORE is {}.".format(high_score()))
    else:
        print('*' * 40)
        print("  Welcome to the Number Guessing Game!")
        print('*' * 40)
        print("\nThe rules are simple...\n\n"
              "Choose a number 1 - 10 (inclusive) until\n"
              "you guess the number. The less attempts\n"
              "you take the better. Good luck!")
        print("\nNo HIGHSCORE has been set. Go get it!")

def guess_the_number():
    answer = random_number()
    attempts = 1
    guess = 0
    highscore = high_score()

    while guess != answer:
        try:
            guess = int(input("\nPick a number between 1 - 10:    "))
            if guess < 1 or guess > 10:
                raise ValueError
                continue
            elif guess < answer:
                print("\nIt's higher!")
                attempts += 1
                continue
            elif guess > answer:
                print("\nIt's lower!")
                attempts += 1
                continue
            else:
                print('-' * 40)
                print("\nGot it!")
                if len(scores_list) == 1:
                    print("\nYou set the score to beat... {} attemps.".format(attempts))
                elif attempts >= highscore:
                    print("\nYou took {} attempts.\n\nNo new HIGHSCORE :(".format(attempts))
                elif attempts < highscore:
                    print("\nYou only took {} attempts.\n\n*** Congratulations! ***\n\nThat's a new HIGHSCORE!!!".format(attempts))
                scores_list.append(attempts)

        except ValueError:
            print("\nOops that's not a valid entry.\nOnly numbers 1 - 10 (inclusive) please.\nTry again...")
            continue

def replay_or_end():
    while len(scores_list) > 1:
        try:
            print('-' * 40)
            play_again = input("\nWould you like to play again?  Y/N    ")
            if play_again.lower() == "n":
                print('-' * 40)
                print("\nThe game is ending... Play again soon :)\n")
                break
            if play_again.lower() == "y":
                start_game()
                break
            else:
                raise ValueError
                continue
        except ValueError:
            print("\nOops, didnt understand your answer. Please enter 'Y' or 'N' only...")
            continue

def start_game():

    begin_game()
    guess_the_number()
    replay_or_end()

if __name__ == '__main__':
    start_game()