import tensorflow as tf
import numpy as np
from scipy.integrate import odeint
import matplotlib.pyplot as plt
from plotting import newfig, savefig
import matplotlib.gridspec as gridspec
import seaborn as sns
import time

from utilities import neural_net, fwd_gradients, heaviside, \
                      tf_session, mean_squared_error, relative_error

class HiddenPathways:
    # Initialize the class
    def __init__(self, t_data, S_data, t_eqns, layers, meal_tq):

        self.D = S_data.shape[1]

        self.t_min = t_data.min(0)
        self.t_max = t_data.max(0)

#        self.S_scale = tf.Variable(np.array(self.D*[1.0]), dtype=tf.float32, trainable=False)
        self.S_scale = S_data.std(0)

        # data on all the species (only some are used as input)
        self.t_data, self.S_data = t_data, S_data
        self.t_eqns = t_eqns

        # layers
        self.layers = layers

        self.mt = 2.0*(meal_tq[0] - self.t_min)/(self.t_max - self.t_min) - 1.0
        self.mq = meal_tq[1]

#        self.k = tf.Variable(1.0/120.0, dtype=tf.float32, trainable=False)
        self.Rm = tf.Variable(209.0/100.0, dtype=tf.float32, trainable=False)
        self.Vg = tf.Variable(10.0, dtype=tf.float32, trainable=False)
        self.C1 = tf.Variable(300.0/100.0, dtype=tf.float32, trainable=False)
        self.a1 = tf.Variable(6.6, dtype=tf.float32, trainable=False)
#        self.Ub = tf.Variable(72.0/100.0, dtype=tf.float32, trainable=False)
#        self.C2 = tf.Variable(144.0/100.0, dtype=tf.float32, trainable=False)
#        self.U0 = tf.Variable(4.0/100.0, dtype=tf.float32, trainable=False)
#        self.Um = tf.Variable(90.0/100.0, dtype=tf.float32, trainable=False)
#        self.C3 = tf.Variable(100.0/100.0, dtype=tf.float32, trainable=False)
#        self.C4 = tf.Variable(80.0/100.0, dtype=tf.float32, trainable=False)
        self.Vi = tf.Variable(11.0, dtype=tf.float32, trainable=False)
        self.E = tf.Variable(0.2, dtype=tf.float32, trainable=False)
        self.ti = tf.Variable(100.0, dtype=tf.float32, trainable=False)
#        self.beta = tf.Variable(1.772, dtype=tf.float32, trainable=False)
#        self.Rg = tf.Variable(180.0/100.0, dtype=tf.float32, trainable=False)
#        self.alpha = tf.Variable(7.5, dtype=tf.float32, trainable=False)
        self.Vp = tf.Variable(3.0, dtype=tf.float32, trainable=False)
#        self.C5 = tf.Variable(26.0/100.0, dtype=tf.float32, trainable=False)
        self.tp = tf.Variable(6.0, dtype=tf.float32, trainable=False)
#        self.td = tf.Variable(12.0, dtype=tf.float32, trainable=False)

        self.logk = tf.Variable(-6.0, dtype=tf.float32, trainable=True)
#        self.logRm = tf.Variable(0.0, dtype=tf.float32, trainable=True)
#        self.logVg = tf.Variable(0.0, dtype=tf.float32, trainable=True)
#        self.logC1 = tf.Variable(0.0, dtype=tf.float32, trainable=True)
#        self.loga1 = tf.Variable(0.0, dtype=tf.float32, trainable=True)
        self.logUb = tf.Variable(0.0, dtype=tf.float32, trainable=True)
        self.logC2 = tf.Variable(0.0, dtype=tf.float32, trainable=True)
        self.logU0 = tf.Variable(0.0, dtype=tf.float32, trainable=True)
        self.logUm = tf.Variable(0.0, dtype=tf.float32, trainable=True)
        self.logC3 = tf.Variable(0.0, dtype=tf.float32, trainable=True)
        self.logC4 = tf.Variable(0.0, dtype=tf.float32, trainable=True)
#        self.logVi = tf.Variable(0.0, dtype=tf.float32, trainable=True)
#        self.logE = tf.Variable(0.0, dtype=tf.float32, trainable=True)
#        self.logti = tf.Variable(0.0, dtype=tf.float32, trainable=True)
        self.logbeta = tf.Variable(0.0, dtype=tf.float32, trainable=True)
        self.logRg = tf.Variable(0.0, dtype=tf.float32, trainable=True)
        self.logalpha = tf.Variable(0.0, dtype=tf.float32, trainable=True)
#        self.logVp = tf.Variable(0.0, dtype=tf.float32, trainable=True)
        self.logC5 = tf.Variable(0.0, dtype=tf.float32, trainable=True)
#        self.logtp = tf.Variable(0.0, dtype=tf.float32, trainable=True)
        self.logtd = tf.Variable(0.0, dtype=tf.float32, trainable=True)

        self.var_list_eqns = [self.logk, self.logUb,
                              self.logC2, self.logU0, self.logUm, self.logC3, self.logC4,
                              self.logbeta, self.logRg, self.logalpha, self.logC5,
                              self.logtd]

        self.k = tf.exp(self.logk)
#        self.Rm = tf.exp(self.logRm)
#        self.Vg = tf.exp(self.logVg)
#        self.C1 = tf.exp(self.logC1)
#        self.a1 = tf.exp(self.loga1)
        self.Ub = tf.exp(self.logUb)
        self.C2 = tf.exp(self.logC2)
        self.U0 = tf.exp(self.logU0)
        self.Um = tf.exp(self.logUm)
        self.C3 = tf.exp(self.logC3)
        self.C4 = tf.exp(self.logC4)
#        self.Vi = tf.exp(self.logVi)
#        self.E = tf.exp(self.logE)
#        self.ti = tf.exp(self.logti)
        self.beta = tf.exp(self.logbeta)
        self.Rg = tf.exp(self.logRg)
        self.alpha = tf.exp(self.logalpha)
#        self.Vp = tf.exp(self.logVp)
        self.C5 = tf.exp(self.logC5)
#        self.tp = tf.exp(self.logtp)
        self.td = tf.exp(self.logtd)

        # placeholders for data
        self.t_data_tf = tf.placeholder(tf.float32, shape=[None, 1])
        self.S_data_tf = tf.placeholder(tf.float32, shape=[None, self.D])
        self.t_eqns_tf = tf.placeholder(tf.float32, shape=[None, 1])
        self.mt_tf = tf.placeholder(tf.float32, shape=[None, self.mt.shape[1]])
        self.mq_tf = tf.placeholder(tf.float32, shape=[None, self.mq.shape[1]])
        self.learning_rate = tf.placeholder(tf.float32, shape=[])

        # physics uninformed neural networks
        self.net_sysbio = neural_net(layers=self.layers)

        self.H_data = 2.0*(self.t_data_tf - self.t_min)/(self.t_max - self.t_min) - 1.0
        self.S_data_pred = self.S_data[0,:] + self.S_scale*(self.H_data+1.0)*self.net_sysbio(self.H_data)

        # physics informed neural networks
        self.H_eqns = 2.0*(self.t_eqns_tf - self.t_min)/(self.t_max - self.t_min) - 1.0
        self.S_eqns_pred = self.S_data[0,:] + self.S_scale*(self.H_eqns+1.0)*self.net_sysbio(self.H_eqns)

        self.E_eqns_pred, self.IG = self.SysODE(self.S_eqns_pred, self.t_eqns_tf,
                                                self.H_eqns, self.mt_tf, self.mq_tf)

        # Adaptive S_scale
#        self.S_scale = 0.9*self.S_scale + 0.1*tf.math.reduce_std(self.S_eqns_pred, 0)
#        scale_list = tf.unstack(self.S_scale)
#        scale_list[2] = self.S_data.std(0)[2]
#        self.S_scale = tf.stack(scale_list)

        # loss
        self.loss_data = mean_squared_error(self.S_data_tf[:,2:3]/self.S_scale[2:3], self.S_data_pred[:,2:3]/self.S_scale[2:3])
        self.loss_eqns = mean_squared_error(0.0, self.E_eqns_pred/self.S_scale)
        self.loss_auxl = mean_squared_error(self.S_data_tf[-1,:]/self.S_scale, self.S_data_pred[-1,:]/self.S_scale)
        self.loss = 0.99*self.loss_data + 0.01*self.loss_eqns + 0.01*self.loss_auxl

        # optimizers
        self.optimizer = tf.train.AdamOptimizer(learning_rate=self.learning_rate)
        self.optimizer_para = tf.train.AdamOptimizer(learning_rate=0.001)

        self.train_op = self.optimizer.minimize(self.loss,
                                                var_list=[self.net_sysbio.weights,
                                                          self.net_sysbio.biases,
                                                          self.net_sysbio.gammas])
        self.trainpara_op = self.optimizer_para.minimize(self.loss,
                                                         var_list=self.var_list_eqns)
        self.sess = tf_session()

    def SysODE(self, S, t, H, mt, mq):
        intake = self.k * mq * heaviside(H-mt) * tf.exp(self.k*(mt-H)*(self.t_max-self.t_min)/2.0)
        IG = tf.reduce_sum(intake, axis=1, keepdims=True)
        kappa = 1.0/self.Vi + 1.0/(self.E*self.ti)
        f1 = self.Rm * tf.sigmoid(S[:,2:3]/(self.Vg*self.C1) - self.a1)
        f2 = self.Ub * (1.0 - tf.exp(-S[:,2:3]/(self.Vg*self.C2)))
        safe_log = tf.where(S[:,1:2] <= 0.0, tf.ones_like(S[:,1:2]), S[:,1:2])
        f3 = (self.U0 + self.Um*tf.sigmoid(self.beta*tf.log(kappa*safe_log/self.C4))) / (self.Vg*self.C3)
        f4 = self.Rg * tf.sigmoid(-self.alpha*(S[:,5:6]/(self.Vp*self.C5)-1.0))

        F0 = f1 - self.E*(S[:,0:1]/self.Vp-S[:,1:2]/self.Vi) - S[:,0:1]/self.tp
        F1 = self.E*(S[:,0:1]/self.Vp-S[:,1:2]/self.Vi) - S[:,1:2]/self.ti
        F2 = f4 + IG - f2 - f3*S[:,2:3]
        F3 = (S[:,0:1] - S[:,3:4]) / self.td
        F4 = (S[:,3:4] - S[:,4:5]) / self.td
        F5 = (S[:,4:5] - S[:,5:6]) / self.td

        F = tf.concat([F0, F1, F2, F3, F4, F5], 1)

        S_t = fwd_gradients(S, t)

        E = S_t - F
        return E, IG

    def train(self, num_epochs, batch_size, learning_rate):

        N_data = self.t_data.shape[0]
        N_eqns = self.t_eqns.shape[0]

        for epoch in range(num_epochs):
            start_time = time.time()
            for it in range(N_eqns//batch_size):
                idx_data = np.concatenate([np.array([0]),
                                          np.random.choice(np.arange(1, N_data-1), min(batch_size, N_data)-2),
                                          np.array([N_data-1])])
                idx_eqns = np.random.choice(N_eqns, batch_size)

                t_data_batch, S_data_batch = self.t_data[idx_data,:], self.S_data[idx_data,:]
                t_eqns_batch = self.t_eqns[idx_eqns,:]
                mt_batch, mq_batch = self.mt[idx_eqns,:], self.mq[idx_eqns,:]

                tf_dict = {self.t_data_tf: t_data_batch,
                           self.S_data_tf: S_data_batch,
                           self.t_eqns_tf: t_eqns_batch,
                           self.mt_tf: mt_batch, self.mq_tf: mq_batch,
                           self.learning_rate: learning_rate}

                self.sess.run([self.train_op, self.trainpara_op], tf_dict)

                # Print
                if it % 10 == 0:
                    elapsed = time.time() - start_time
                    [loss_data_value,
                     loss_eqns_value,
                     loss_auxl_value,
                     learning_rate_value] = self.sess.run([self.loss_data,
                                                           self.loss_eqns,
                                                           self.loss_auxl,
                                                           self.learning_rate], tf_dict)
                    print('Epoch: %d, It: %d, Loss Data: %.3e, Loss Eqns: %.3e, Loss Aux: %.3e, Time: %.3f, Learning Rate: %.1e'
                          %(epoch, it, loss_data_value, loss_eqns_value, loss_auxl_value, elapsed, learning_rate_value))
                    start_time = time.time()

    def predict(self, t_star, meal_tq):

        meal_tq[0] = 2.0*(meal_tq[0] - self.t_min)/(self.t_max - self.t_min) - 1.0
        tf_dict = {self.t_eqns_tf: t_star,
                   self.mt_tf: meal_tq[0], self.mq_tf: meal_tq[1]}

        S_star, IG = self.sess.run([self.S_eqns_pred, self.IG], tf_dict)
        S_star = np.append(S_star[:,:], IG[:], axis=1)
        return S_star

if __name__ == "__main__":

    layers = [1] + 6*[6*30] + [6]

    meal_t = [300., 650., 1100., 2000.]
    meal_q = [60e3, 40e3, 50e3, 100e3]

    def intake(tn, k):
        def s(mjtj):
            return k*mjtj[1]*np.heaviside(tn-mjtj[0], 0.5)*np.exp(k*(mjtj[0]-tn))
        IG = np.array([s(mjtj) for mjtj in list(zip(meal_t, meal_q))]).sum()
        return IG

    # function that returns dx/dt
    def f(x, t): # x is 6 x 1
        k = 1./120.
        Rm = 209.
        Vg = 10.
        C1 = 300.
        a1 = 6.6
        Ub = 72.
        C2 = 144.
        U0 = 4.
        Um = 90.
        C3 = 100.
        C4 = 80.
        Vi = 11.
        E = 0.2
        ti = 100.
        beta = 1.772
        Rg = 180.
        alpha = 7.5
        Vp = 3.
        C5 = 26.
        tp = 6.
        td = 12.

        kappa = 1.0/Vi + 1.0/E/ti
        f1 = Rm / (1.0 + np.exp(-x[2]/Vg/C1 + a1))
        f2 = Ub * (1.0 - np.exp(-x[2]/Vg/C2))
        f3 = (U0 + Um/(1.0+np.exp(-beta*np.log(kappa*x[1]/C4)))) / Vg / C3
        f4 = Rg / (1.0 + np.exp(alpha*(x[5]/Vp/C5-1.0)))

        x0 = f1 - E*(x[0]/Vp-x[1]/Vi) - x[0]/tp
        x1 = E*(x[0]/Vp-x[1]/Vi) - x[1]/ti
        x2 = f4 + intake(t, k) - f2 - f3*x[2]
        x3 = (x[0] - x[3]) / td
        x4 = (x[3] - x[4]) / td
        x5 = (x[4] - x[5]) / td

        X = np.array([x0, x1, x2, x3, x4, x5])
        return X

    # function that returns dx/dt
    def f_pred(x, t): # x is 6 x 1
        k =  0.007751
        Rm = 73.858517
        Vg = 10.000000
        C1 = 319.160032
        a1 = 6.253946
        Ub = 86.824888
        C2 = 152.637362
        U0 = 19.412358
        Um = 141.051173
        C3 = 235.955381
        C4 = 251.580667
        Vi = 2.689281
        E = 0.147199
        ti = 36.766254
        beta = 2.475349
        Rg = 212.777472
        alpha = 7.182466
        Vp = 0.707807
        C5 = 101.811242
        tp = 139.384628
        td = 7.417875

        kappa = 1.0/Vi + 1.0/E/ti
        f1 = Rm / (1.0 + np.exp(-x[2]/Vg/C1 + a1))
        f2 = Ub * (1.0 - np.exp(-x[2]/Vg/C2))
        f3 = (U0 + Um/(1.0+np.exp(-beta*np.log(kappa*x[1]/C4)))) / Vg / C3
        f4 = Rg / (1.0 + np.exp(alpha*(x[5]/Vp/C5-1.0)))

        x0 = f1 - E*(x[0]/Vp-x[1]/Vi) - x[0]/tp
        x1 = E*(x[0]/Vp-x[1]/Vi) - x[1]/ti
        x2 = f4 + intake(t, k) - f2 - f3*x[2]
        x3 = (x[0] - x[3]) / td
        x4 = (x[3] - x[4]) / td
        x5 = (x[4] - x[5]) / td

        X = np.array([x0, x1, x2, x3, x4, x5])
        return X

    def plotting(t_star, S_star, S_pred, perm, Vg2, forecast=False):
        sns.set()

        fig, ax = newfig(2.0, 0.7)
        gs0 = gridspec.GridSpec(1, 1)
        gs0.update(top=0.9, bottom=0.15, left=0.1, right=0.95, hspace=0.3, wspace=0.3)
        ax = plt.subplot(gs0[0:1, 0:1])
        ax.plot(t_star,S_star[:,2],'C1',linewidth=2,label='input data')
        ax.scatter(t_star[perm],S_star[perm,2],marker='o',s=40,label='sampled input')
        ax.set_xlabel('$t\ (min)$', fontsize=18)
        ax.set_ylabel('$G\ (mg/dl) $', fontsize=18)
        ax.legend(fontsize='large')

        ####################################
        fig, ax = newfig(1.8, 0.75)
        gs1 = gridspec.GridSpec(1, 2)
        gs1.update(top=0.85, bottom=0.15, left=0.1, right=0.95, hspace=0.3, wspace=0.3)
        ax = plt.subplot(gs1[0:1, 0:1])
        ax.plot(t_star,S_star[:,0]*Vg2,'C1',linewidth=2,label='exact')
        ax.plot(t_star,S_pred[:,0]*Vg2,'g-.',linewidth=3,label='learned')
        ax.set_xlabel('$t\ (min)$', fontsize=18)
        ax.set_ylabel('$I_p\ (\mu U/ml)$', fontsize=18)
        ax.legend(fontsize='large')

        ax = plt.subplot(gs1[0:1, 1:2])
        ax.plot(t_star,S_star[:,1]*Vg2,'C1',linewidth=2)
        ax.plot(t_star,S_pred[:,1]*Vg2,'g-.',linewidth=3)
        ax.set_xlabel('$t\ (min)$', fontsize=18)
        ax.set_ylabel('$I_i\ (\mu U/ml)$', fontsize=18)

        fig, ax = newfig(1.8, 0.75)
        gs2 = gridspec.GridSpec(1, 2)
        gs2.update(top=0.85, bottom=0.15, left=0.1, right=0.95, hspace=0.3, wspace=0.3)
        ax = plt.subplot(gs2[0:1, 0:1])
        if not forecast:
            ax.scatter(t_star[perm],S_star[perm,2],marker='o',c='C1',s=30)
        else:
            ax.plot(t_star,S_star[:,2],'C1',linewidth=2)
        ax.plot(t_star,S_pred[:,2],'g-.',linewidth=3)
        ax.set_xlabel('$t\ (min)$', fontsize=18)
        ax.set_ylabel('$G\ (mg/dl)$', fontsize=18)

        ax = plt.subplot(gs2[0:1, 1:2])
        ax.plot(t_star,IG_star*Vg2,'C1',linewidth=2)
        ax.plot(t_star,S_pred[:,6]*Vg2,'g-.',linewidth=3)
        ax.set_xlabel('$t\ (min)$', fontsize=18)
        ax.set_ylabel('$I_G\ (mg/min)$', fontsize=18)

    # time points
    t_star = np.arange(0, 3000, 1.0)
    N = t_star.shape[0]
    N_eqns = N
    N_data = N // 5

    k = 1./120.
    Vp = 3.0
    Vi = 11.0
    Vg2 = 10.0*10.0
    S0 = 12.0*Vp
    S1 = 4.0*Vi
    S2 = 110.0*Vg2
    S3 = 0.0
    S4 = 0.0
    S5 = 0.0

    # initial condition
    x0 = np.array([S0, S1, S2, S3, S4, S5]).flatten()

    # solve ODE
    S_star = odeint(f, x0, t_star)
    S_star /= Vg2 # scaling by Vg^2
    IG_star = np.array([intake(t, k) for t in t_star]) / Vg2

    t_train = t_star[:,None]
    S_train = S_star

    # two-point data must be given for all the species
    # 1st: initial at t=0; 2nd: any point between (0,T]
    N0 = 0
    N1 = N - 1
    idx_data = np.concatenate([np.array([N0]),
                               np.random.choice(np.arange(1, N-1), size=N_data, replace=False),
                               np.array([N-1]),
                               np.array([N1])])
    idx_eqns = np.concatenate([np.array([N0]),
                               np.random.choice(np.arange(1, N-1), size=N_eqns-2, replace=False),
                               np.array([N-1])])
    meal_tq = [np.array([N_eqns*[x] for x in meal_t]).T,
               np.array([N_eqns*[x/Vg2] for x in meal_q]).T]

    model = HiddenPathways(t_train[idx_data],
                           S_train[idx_data,:],
                           t_train[idx_eqns],
                           layers,
                           meal_tq)

    model.train(num_epochs=25000, batch_size=N_eqns, learning_rate=1e-3)
    model.train(num_epochs=25000, batch_size=N_eqns, learning_rate=1e-4)
    model.train(num_epochs=10000, batch_size=N_eqns, learning_rate=1e-5)

    # NN prediction
    meal_tq = [np.array([N*[x] for x in meal_t]).T,
               np.array([N*[x/Vg2] for x in meal_q]).T]

    S_pred = model.predict(t_star[:,None], meal_tq)
    plotting(t_star, S_star, S_pred, idx_data, Vg2)

    print('k =  %.6f' % ( model.sess.run(model.k) ) )
    print('Rm = %.6f' % ( model.sess.run(model.Rm)*Vg2 ) )
    print('Vg = %.6f' % ( model.sess.run(model.Vg) ) )
    print('C1 = %.6f' % ( model.sess.run(model.C1)*Vg2 ) )
    print('a1 = %.6f' % ( model.sess.run(model.a1) ) )
    print('Ub = %.6f' % ( model.sess.run(model.Ub)*Vg2 ) )
    print('C2 = %.6f' % ( model.sess.run(model.C2)*Vg2 ) )
    print('U0 = %.6f' % ( model.sess.run(model.U0)*Vg2 ) )
    print('Um = %.6f' % ( model.sess.run(model.Um)*Vg2 ) )
    print('C3 = %.6f' % ( model.sess.run(model.C3)*Vg2 ) )
    print('C4 = %.6f' % ( model.sess.run(model.C4)*Vg2 ) )
    print('Vi = %.6f' % ( model.sess.run(model.Vi) ) )
    print('E = %.6f' % ( model.sess.run(model.E) ) )
    print('ti = %.6f' % ( model.sess.run(model.ti) ) )
    print('beta = %.6f' % ( model.sess.run(model.beta) ) )
    print('Rg = %.6f' % ( model.sess.run(model.Rg)*Vg2 ) )
    print('alpha = %.6f' % ( model.sess.run(model.alpha) ) )
    print('Vp = %.6f' % ( model.sess.run(model.Vp) ) )
    print('C5 = %.6f' % ( model.sess.run(model.C5)*Vg2 ) )
    print('tp = %.6f' % ( model.sess.run(model.tp) ) )
    print('td = %.6f' % ( model.sess.run(model.td) ) )

    # Prediction based on inferred parameters
#    k =  0.007751
#    Vp = 0.707807
#    Vi = 2.689281
#    S0 = 12.0*Vp
#    S1 = 4.0*Vi
#    S2 = 110.0*Vg2
#    S3 = 0.0
#    S4 = 0.0
#    S5 = 0.0
#    x0 = np.array([S0, S1, S2, S3, S4, S5]).flatten()
#    S_pred = odeint(f_pred, x0, t_star)
#    S_pred /= Vg2
#    IG_pred = np.array([intake(t, k) for t in t_star]) / Vg2
#    S_pred = np.append(S_pred[:,:], IG_pred[:,None], axis=1)
#    plotting(t_star, S_star, S_pred, idx_data, Vg2, forecast=True)

    # savefig('./figures/Glycolytic', crop = False)


"""Tests for the raster module"""
import pytest
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
import matplotlib
from matplotcheck.raster import RasterTester

@pytest.fixture
def np_ar():
    """Create numpy array for image plot testing"""
    return np.random.rand(100, 100)

@pytest.fixture
def np_ar_rgb():
    """Create rgb numpy array for image plot testing"""
    return np.random.randint(0, 255, size=(100, 100, 3))

@pytest.fixture
def np_ar_discrete():
    """Create numpy array with discrete values for image plot testing"""
    return np.random.choice(4, (10, 10))

@pytest.fixture
def raster_plt(np_ar):
    """Create simple raster plot for testing"""
    fig, ax = plt.subplots()
    im = ax.imshow(np_ar)
    ax.set_title("My Plot Title", fontsize=30)
    fig.colorbar(im)

    axis = plt.gca()
    return RasterTester(axis)

@pytest.fixture
def raster_plt_rgb(np_ar_rgb):
    """Create 3-band raster RGB plot for testing"""
    fig, ax = plt.subplots()
    ax.imshow(np_ar_rgb)

    axis = plt.gca()
    return RasterTester(axis)

@pytest.fixture
def raster_plt_blank():
    """Blank axis for testing. No image is plotted"""
    fig, ax = plt.subplots()

    axis = plt.gca()
    return RasterTester(axis)

@pytest.fixture
def raster_plt_class(np_ar_discrete):
    """Classified/discrete raster plot for testing"""
    values = np.sort(np.unique(np_ar_discrete))

    fig, ax = plt.subplots()
    im = ax.imshow(
        np_ar_discrete, interpolation="none", cmap=plt.get_cmap("tab10")
    )
    ax.set_title("My Plot Title", fontsize=30)

    # Create legend
    colors = [im.cmap(im.norm(val)) for val in values]
    patches = [
        mpatches.Patch(
            color=colors[i], label="Level {lev}".format(lev=values[i])
        )
        for i in range(values.shape[0])
    ]
    plt.legend(handles=patches)

    axis = plt.gca()

    return RasterTester(axis)

""" COLORBAR TESTS """

def test_raster_get_colorbars_length(raster_plt):
    """Check that get_colorbars correctly retrieves 1 colorbar from
    raster_plt1"""
    # Should only be 1 object, and should be a colorbar object
    cb = raster_plt.get_colorbars()
    assert len(cb) == 1
    plt.close()

def test_raster_get_colorbars_type(raster_plt):
    """Check that get_colorbars retrieves a colorbar object"""
    cb = raster_plt.get_colorbars()
    assert isinstance(cb[0], matplotlib.colorbar.Colorbar)
    plt.close()

def test_raster_assert_colorbar_range(raster_plt, np_ar):
    """Colorbar range checker, should be set to min and max of input array"""
    # Colorbar range should be min and max
    raster_plt.assert_colorbar_range([np_ar.min(), np_ar.max()])
    plt.close()

def test_raster_assert_colorbar_range_wrongmin(raster_plt, np_ar):
    """Colorbar range checker should fail with the wrong max"""
    # Should fail with min value
    with pytest.raises(
        AssertionError,
        match="Colorbar minimum is not expected value:{}".format(
            np_ar.min() - 1
        ),
    ):
        raster_plt.assert_colorbar_range([np_ar.min() - 1, np_ar.max()])
    plt.close()

def test_raster_assert_colorbar_range_wrongmax(raster_plt, np_ar):
    """Colorbar range checker should fail with the wrong min"""
    # Should fail with wrong max value
    with pytest.raises(
        AssertionError,
        match="Colorbar maximum is not expected value:{}".format(
            np_ar.max() + 0.1
        ),
    ):
        raster_plt.assert_colorbar_range([np_ar.min(), np_ar.max() + 0.1])
    plt.close()

def test_raster_assert_colorbar_range_multiple(raster_plt, np_ar):
    """Colorbar range checker should fail when there are multiple colorbars"""
    # Add a second colorbar
    raster_plt.ax.imshow(np_ar)
    raster_plt.ax.get_figure().colorbar(raster_plt.ax.images[1], raster_plt.ax)
    with pytest.raises(
        AssertionError, match="Exactly one colorbar should be displayed"
    ):
        raster_plt.assert_colorbar_range([np_ar.min(), np_ar.max()])
    plt.close()

def test_raster_assert_colorbar_range_blank(raster_plt_blank, np_ar):
    """Colorbar range checker should fail if no image has been added to axes"""
    # Should fail with no image on axis
    with pytest.raises(AssertionError, match="No image found on axes"):
        raster_plt_blank.assert_colorbar_range([np_ar.min(), np_ar.max()])
    plt.close()

""" LEGEND TESTS """

def test_raster_assert_legend_accuracy(raster_plt_class, np_ar_discrete):
    """Checks that legend matches image, checking both the labels and color
    patches"""
    values = np.sort(np.unique(np_ar_discrete))
    label_options = [[str(i)] for i in values]

    raster_plt_class.assert_legend_accuracy_classified_image(
        np_ar_discrete, label_options
    )
    plt.close()

def test_raster_assert_legend_accuracy_badlabel(
    raster_plt_class, np_ar_discrete
):
    """Checks that legend matches image, should fail with bad labels"""
    values = np.sort(np.unique(np_ar_discrete))

    # Should fail with bad label
    bad_label_options = [["foo"] * values.shape[0]]
    with pytest.raises(AssertionError, match="Incorrect legend labels"):
        raster_plt_class.assert_legend_accuracy_classified_image(
            np_ar_discrete, bad_label_options
        )
    plt.close()

def test_raster_assert_legend_accuracy_badvalues(
    raster_plt_class, np_ar_discrete
):
    """Checks that legend matches image, should fail if you swap image
    values"""
    values = np.sort(np.unique(np_ar_discrete))
    label_options = [[str(i)] for i in values]

    # Swap image values without updating plot colors
    bad_image = np_ar_discrete
    bad_image[bad_image == 0] = 1

    # Should fail with bad image
    with pytest.raises(
        AssertionError, match="Incorrect legend to data relation"
    ):
        raster_plt_class.assert_legend_accuracy_classified_image(
            bad_image, label_options
        )
    plt.close()

def test_raster_assert_legend_accuracy_nolegend(raster_plt, np_ar_discrete):
    """Check that assert_legend_accuracy fails if there is no legend"""
    values = np.sort(np.unique(np_ar_discrete))
    label_options = [[str(i)] for i in values]

    # Fails without legend
    with pytest.raises(AssertionError, match="No legend displayed"):
        raster_plt.assert_legend_accuracy_classified_image(
            np_ar_discrete, label_options
        )
    plt.close()

def test_raster_assert_legend_accuracy_noimage(
    raster_plt_blank, np_ar_discrete
):
    """Check that assert_legend_accuracy fails if there is no image"""
    values = np.sort(np.unique(np_ar_discrete))
    label_options = [[str(i)] for i in values]

    # Fails when no image displayed
    with pytest.raises(AssertionError, match="No Image Displayed"):
        raster_plt_blank.assert_legend_accuracy_classified_image(
            np_ar_discrete, label_options
        )
    plt.close()

""" IMAGE TESTS """

def test_raster_assert_image(raster_plt, np_ar):
    """Checks that assert_image passes only when plot data matches array"""
    raster_plt.assert_image(np_ar)
    plt.close()

def test_raster_assert_image_baddata(raster_plt, np_ar):
    """assert_image should fail when we change the array values"""
    # Should fail with wrong array
    bad_ar = np_ar + 1
    with pytest.raises(AssertionError, match="Arrays are not equal"):
        raster_plt.assert_image(bad_ar)
    plt.close()

def test_raster_assert_image_blank(raster_plt_blank, np_ar):
    """"assert_image should fail with blank image"""
    with pytest.raises(AssertionError, match="No Image Displayed"):
        raster_plt_blank.assert_image(np_ar)
    plt.close()

def test_raster_assert_image_rgb(raster_plt_rgb, np_ar_rgb):
    """Check assert_image for a 3-band RGB image"""
    raster_plt_rgb.assert_image(np_ar_rgb)
    plt.close()

def test_raster_assert_image_rgb_baddata(raster_plt_rgb, np_ar_rgb):
    """Check assert_image fails with bad data for rgb plot"""
    bad_ar = np_ar_rgb + 1
    with pytest.raises(AssertionError, match="Arrays are not equal"):
        raster_plt_rgb.assert_image(bad_ar)
    plt.close()

def test_raster_assert_image_class(raster_plt_class, np_ar_discrete):
    """Check assert_image for a discrete, classified image"""
    raster_plt_class.assert_image(np_ar_discrete, im_classified=True)
    plt.close()

def test_raster_assert_image_class_baddata(raster_plt_class, np_ar_discrete):
    """Check that assert_image with bad data fails for a discrete, classified
    image"""
    bad_ar = np_ar_discrete + 1
    with pytest.raises(AssertionError, match="Arrays are not equal"):
        raster_plt_class.assert_image(bad_ar)
    plt.close()

def test_raster_assert_image_fullscreen(raster_plt):
    """Checks that the first image on axis takes up full axis"""
    raster_plt.assert_image_full_screen()
    plt.close()

def test_raster_assert_image_fullscreen_fail_xlims(raster_plt):
    """assert fullscreen should fail if we modify the x-axis limits"""
    cur_xlim = raster_plt.ax.get_xlim()
    raster_plt.ax.get_ylim()
    raster_plt.ax.set_xlim([cur_xlim[0], cur_xlim[1] + 5])
    with pytest.raises(
        AssertionError, match="Image is stretched inaccurately"
    ):
        raster_plt.assert_image_full_screen()
    plt.close()

def test_raster_assert_image_fullscreen_fail_ylims(raster_plt):
    """assert fullscreen should fail if we modify the x-axis limits"""
    cur_xlim, cur_ylim = raster_plt.ax.get_xlim(), raster_plt.ax.get_ylim()
    raster_plt.ax.set_xlim([cur_xlim[0], cur_xlim[1]])
    raster_plt.ax.set_ylim([cur_ylim[0], cur_ylim[1] - 1])
    with pytest.raises(
        AssertionError, match="Image is stretched inaccurately"
    ):
        raster_plt.assert_image_full_screen()
    plt.close()

def test_raster_assert_image_fullscreen_blank(raster_plt_blank):
    """assert_image_fullscreen should fail with blank image"""
    with pytest.raises(AssertionError, match="No image found on axes"):
        raster_plt_blank.assert_image_full_screen()
    plt.close()

def test_get_plot_images(raster_plt_rgb):
    """get_plot_image should get correct image from ax object"""
    ax_im = raster_plt_rgb.get_plot_image()
    raster_plt_rgb.assert_image(ax_im)


import os
import gym
import pickle
import numpy as np
import matplotlib.pyplot as plt
import imageio

# path
ROOT_DIR = os.path.dirname(__file__)
Q_TABLE_PATH = os.path.realpath(os.path.join(ROOT_DIR, 'qtable.npy'))
GIF_PATH = os.path.realpath(os.path.join(ROOT_DIR, 'test.gif'))
STATISTIC_PATH = os.path.realpath(os.path.join(ROOT_DIR, 'Statistics.png'))

# constants
LEARNING_RATE = 0.4
DISCOUNT = 0.95
EPISODES = 20000
epsilon = 1
ACTION_SPACE_SIZE = 17
OBSERVATION_SPACE_SIZE = [21, 21, 65]

env = gym.make("Pendulum-v0")
ep_rewards = []
revards_log = {'ep': [], 'avg': [], 'max': [], 'min': []}
discrete_os_win_size = (env.observation_space.high -
                        env.observation_space.low) / [i-1 for i in OBSERVATION_SPACE_SIZE]

def get_input():
    global EPISODES, epsilon
    learn = input("LEARN? (y/n): ")
    if learn == 'y':
        episode_count = input("Enter number of episodes: ")
        EPISODES = int(episode_count)
        return 'learn'
    else:
        epsilon = 0
        return 'test'

def make_action_space():
    high = env.action_space.high
    low = env.action_space.low
    make_action_space = (high - low) / (ACTION_SPACE_SIZE - 1)
    action_space = {}
    for i in range(ACTION_SPACE_SIZE):
        action_space[i] = [low[0] + (i * make_action_space[0])]
    return action_space

def create_qtable():
    if not os.path.exists(Q_TABLE_PATH):
        q_table = np.zeros(OBSERVATION_SPACE_SIZE + [ACTION_SPACE_SIZE])
        np.save(Q_TABLE_PATH, q_table)
    else:
        q_table = np.load(Q_TABLE_PATH)
    return q_table

def get_discrete_state(state):
    low = env.observation_space.low
    ds = (state - low) / discrete_os_win_size
    return tuple(ds.astype(np.int32))

def get_action(q_table, state, epsilon):
    if np.random.random() > epsilon:
        action = np.argmax(q_table[state])
    else:
        action = np.random.randint(0, ACTION_SPACE_SIZE)
    return action

def update_qtable(q_table, state, action, new_state, reward):
    current_q = q_table[state + (action, )]
    max_future_q = np.max(q_table[new_state])
    new_q = (1 - LEARNING_RATE) * current_q + LEARNING_RATE * \
        (reward + DISCOUNT * max_future_q)
    q_table[state + (action, )] = new_q
    return q_table

def q_learning():
    global epsilon
    q_table = create_qtable()
    action_space = make_action_space()

    for episode in range(EPISODES):
        episode_reward = 0
        state = get_discrete_state(env.reset())
        done = False
        while not done:
            action = get_action(q_table, state, epsilon)
            torque = action_space[action]
            new_state, reward, done, _ = env.step(torque)
            episode_reward += reward
            new_state = get_discrete_state(new_state)
            q_table = update_qtable(
                q_table, state, action, new_state, reward)
            state = new_state
        epsilon *= 0.999
        log(episode, episode_reward)

    np.save(Q_TABLE_PATH, q_table)
    return q_table

def log(episode, episode_reward, test=False):
    ep_rewards.append(episode_reward)
    if not episode % 100 or test:
        average_reward = sum(ep_rewards[-100:]) / 100
        revards_log['ep'].append(episode)
        revards_log['avg'].append(average_reward)
        revards_log['max'].append(max(ep_rewards[-100:]))
        revards_log['min'].append(min(ep_rewards[-100:]))
        print(
            f"Episode: {episode:>5d}, average reward: {average_reward:>4.1f}, current epsilon: {epsilon:>1.3f}")

def test_and_make_gif():
    q_table = create_qtable()
    action_space = make_action_space()
    chart_data_time = []
    chart_data_reward = []
    chart_data_action = []
    images = []
    for episode in range(1):
        episode_reward = 0
        state = get_discrete_state(env.reset())
        done = False
        while not done:
            action = get_action(q_table, state, 0)
            torque = action_space[action]
            new_state, reward, done, _ = env.step(torque)
            episode_reward += reward
            new_state = get_discrete_state(new_state)
            state = new_state
            images.append(env.render(mode='rgb_array'))
            chart_data_time.append(action)
            chart_data_reward.append(reward)
            chart_data_action.append(torque)
        log(episode, episode_reward, True)

    env.close()
    print("Making gif...")
    imageio.mimsave(GIF_PATH, images)
    draw_3D_chart(chart_data_time, chart_data_reward, chart_data_action)

def show_statistics():
    ep = revards_log['ep']
    avg = revards_log['avg']
    max = revards_log['max']
    min = revards_log['min']
    with open("statistics", 'wb') as filehandler:
        pickle.dump(revards_log, filehandler)
        plt.plot(ep, avg, label="average rewards")
        plt.plot(ep, max, label="max rewards")
        plt.plot(ep, min, label="min rewards")
        plt.legend(loc=4)
        plt.savefig(STATISTIC_PATH)
        plt.show()

def draw_3D_chart(chart_data_time, chart_data_reward, chart_data_action):
    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')
    ax.scatter(chart_data_time, chart_data_reward,
               chart_data_action, c='r', marker='o')
    ax.set_xlabel('Time')
    ax.set_ylabel('Reward')
    ax.set_zlabel('Action')
    plt.show()
    # save it in a image file
    ax.figure.savefig('QLearning_chart.png')

def main():
    learn_or_test = get_input()
    if learn_or_test == 'learn':
        q_learning()
        show_statistics()
    else:
        test_and_make_gif()

if __name__ == "__main__":
    main()


def list_of_sheets_in_GD(json_keyfile, doc_url):
# json_keyfile - path to json_file
# doc_url - URL of GoogleDoc
    import gspread
    from oauth2client.service_account import ServiceAccountCredentials

    scope = ['https://spreadsheets.google.com/feeds',
             'https://www.googleapis.com/auth/drive']
    credentials = ServiceAccountCredentials.from_json_keyfile_name(json_keyfile, scope)

    client = gspread.authorize(credentials)

    worksheet_list = client.open_by_url(doc_url).worksheets()

    sheet_names = [worksheet.title for worksheet in worksheet_list]

    return sheet_names


#!/usr/bin/env python
# This scripts converts a list of cidr notation IP ranges to /32 ip list.
# Written by: unitato
#
# $cat ip-range.list
# 192.168.0.0/24
# 10.0.0.1
#
# usage: ./cidrtoip.py -f ip-range.list
#

import re
import sys
import os.path
import argparse
from netaddr import IPNetwork

parser = argparse.ArgumentParser()
parser.add_argument('--version', action='version', version='%(prog)s 0.1')
parser.add_argument('-f', action='store')
args = parser.parse_args()
fname = ''
if not args.f:
    print("File is required...")
    exit()
else:
    #check if file exists
    if os.path.isfile(args.f):
        print("file exists... reading file...")
        fname = args.f
    else:
        print("File '%s' does not exists" % args.f )
        exit()

#read the input file and remove spaces
with open(fname) as f:
    content = f.readlines()
# you may also want to remove whitespace characters like `\n` at the end of each line
content = [x.strip() for x in content]

for line in content:
    #print("Checking %s..." % line)
    try:
        for ip in IPNetwork(line):
            print("%s" % ip)
    except: # catch *all* exceptions
        e = sys.exc_info()[0]
        print ("Parsing %s" % ip)
        print( "
Error: %s
" % e )
        continue


#!/usr/bin/env python

"""
    A dcmtk style storescp application.

    Used as a SCP for sending DICOM objects to
"""

import argparse
import logging
import os
import socket
import sys

from pydicom.dataset import Dataset, FileDataset
from pydicom.filewriter import write_file
from pydicom.uid import ExplicitVRLittleEndian, ImplicitVRLittleEndian, \
    ExplicitVRBigEndian, DeflatedExplicitVRLittleEndian

from pynetdicom3 import AE, StorageSOPClassList, VerificationSOPClass
from pynetdicom3 import pynetdicom_uid_prefix

def setup_logger():
    """Setup the logger"""
    logger = logging.Logger('storescp')
    stream_logger = logging.StreamHandler()
    formatter = logging.Formatter('%(levelname).1s: %(message)s')
    stream_logger.setFormatter(formatter)
    logger.addHandler(stream_logger)
    logger.setLevel(logging.ERROR)

    return logger

LOGGER = setup_logger()

def _setup_argparser():
    """Setup the command line arguments"""
    # Description
    parser = argparse.ArgumentParser(
        description="The storescp application implements a Service Class "
                    "Provider (SCP) for the Storage SOP Class. It listens "
                    "for a DICOM C-STORE message from a Service Class User "
                    "(SCU) and stores the resulting DICOM dataset.",
        usage="storescp [options] port")

    # Parameters
    req_opts = parser.add_argument_group('Parameters')
    req_opts.add_argument("port",
                          help="TCP/IP port number to listen on",
                          type=int)

    # General Options
    gen_opts = parser.add_argument_group('General Options')
    gen_opts.add_argument("--version",
                          help="print version information and exit",
                          action="store_true")
    gen_opts.add_argument("-q", "--quiet",
                          help="quiet mode, print no warnings and errors",
                          action="store_true")
    gen_opts.add_argument("-v", "--verbose",
                          help="verbose mode, print processing details",
                          action="store_true")
    gen_opts.add_argument("-d", "--debug",
                          help="debug mode, print debug information",
                          action="store_true")

    # Network Options
    net_opts = parser.add_argument_group('Network Options')
    net_opts.add_argument("-aet", "--aetitle", metavar='[a]etitle',
                          help="set my AE title (default: STORESCP)",
                          type=str,
                          default='STORESCP')
    net_opts.add_argument("-to", "--timeout", metavar='[s]econds',
                          help="timeout for connection requests",
                          type=int,
                          default=None)
    net_opts.add_argument("-ta", "--acse-timeout", metavar='[s]econds',
                          help="timeout for ACSE messages",
                          type=int,
                          default=60)
    net_opts.add_argument("-td", "--dimse-timeout", metavar='[s]econds',
                          help="timeout for DIMSE messages",
                          type=int,
                          default=None)
    net_opts.add_argument("-pdu", "--max-pdu", metavar='[n]umber of bytes',
                          help="set max receive pdu to n bytes (4096..131072)",
                          type=int,
                          default=16384)

    # Transfer Syntaxes
    ts_opts = parser.add_argument_group('Preferred Transfer Syntaxes')
    ts_opts.add_argument("-x=", "--prefer-uncompr",
                         help="prefer explicit VR local byte order (default)",
                         action="store_true")
    ts_opts.add_argument("-xe", "--prefer-little",
                         help="prefer explicit VR little endian TS",
                         action="store_true")
    ts_opts.add_argument("-xb", "--prefer-big",
                         help="prefer explicit VR big endian TS",
                         action="store_true")
    ts_opts.add_argument("-xi", "--implicit",
                         help="accept implicit VR little endian TS only",
                         action="store_true")

    # Output Options
    out_opts = parser.add_argument_group('Output Options')
    out_opts.add_argument('-od', "--output-directory", metavar="[d]irectory",
                          help="write received objects to existing directory d",
                          type=str)
    """
    out_opts.add_argument('-su', "--sort-on-study-uid",
                          help="sort studies into subdirectories using "
                                "Study Instance UID",
                          action="store_true")
    out_opts.add_argument('-su', "--sort-on-patient-id",
                          help="sort studies into subdirectories using "
                                "Patient ID and a timestamp",
                          action="store_true")
    out_opts.add_argument('-uf', "--default-filenames",
                          help="generate filenames from instance UID",
                          action="store_true",
                          default=True)
    """
    """
    # Event Options
    event_opts = parser.add_argument_group('Event Options')
    event_opts.add_argument('-xcr', "--exec-on-reception",
                            metavar="[c]ommand",
                            help="execute command c after receiving and "
                                "processing one C-STORE-RQ message",
                            type=str)
    """

    # Miscellaneous
    misc_opts = parser.add_argument_group('Miscellaneous')
    misc_opts.add_argument('--ignore',
                           help="receive data but don't store it",
                           action="store_true")

    return parser.parse_args()

args = _setup_argparser()

if args.verbose:
    LOGGER.setLevel(logging.INFO)

if args.debug:
    LOGGER.setLevel(logging.DEBUG)
    pynetdicom_logger = logging.getLogger('pynetdicom3')
    pynetdicom_logger.setLevel(logging.DEBUG)

LOGGER.debug('$storescp.py v{0!s} {1!s} $'.format('0.2.0', '2016-03-23'))
LOGGER.debug('')

# Validate port
if isinstance(args.port, int):
    test_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    test_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    try:
        test_socket.bind((os.popen('hostname').read()[:-1], args.port))
    except socket.error:
        LOGGER.error("Cannot listen on port {0:d}, insufficient priveleges".format(args.port))
        sys.exit()

# Set Transfer Syntax options
transfer_syntax = [ImplicitVRLittleEndian,
                   ExplicitVRLittleEndian,
                   DeflatedExplicitVRLittleEndian,
                   ExplicitVRBigEndian]

if args.prefer_uncompr and ImplicitVRLittleEndian in transfer_syntax:
        transfer_syntax.remove(ImplicitVRLittleEndian)
        transfer_syntax.append(ImplicitVRLittleEndian)

if args.implicit:
    transfer_syntax = [ImplicitVRLittleEndian]

if args.prefer_little and ExplicitVRLittleEndian in transfer_syntax:
        transfer_syntax.remove(ExplicitVRLittleEndian)
        transfer_syntax.insert(0, ExplicitVRLittleEndian)

if args.prefer_big and ExplicitVRBigEndian in transfer_syntax:
        transfer_syntax.remove(ExplicitVRBigEndian)
        transfer_syntax.insert(0, ExplicitVRBigEndian)

def on_c_store(dataset):
    """
    Write `dataset` to file as little endian implicit VR

    Parameters
    ----------
    dataset : pydicom.dataset.Dataset
        The DICOM dataset sent via the C-STORE

    Returns
    -------
    status : pydicom.dataset.Dataset
        A Dataset containing a Status element with a value valid for the
        Storage Service Class (see PS3.4 annex B.2.3). The dataset may also
        contain optional elements related to the Status (see PS3.7 Annex C).
    """
    mode_prefix = 'UN'
    mode_prefixes = {'CT Image Storage' : 'CT',
                     'Enhanced CT Image Storage' : 'CTE',
                     'MR Image Storage' : 'MR',
                     'Enhanced MR Image Storage' : 'MRE',
                     'Positron Emission Tomography Image Storage' : 'PT',
                     'Enhanced PET Image Storage' : 'PTE',
                     'RT Image Storage' : 'RI',
                     'RT Dose Storage' : 'RD',
                     'RT Plan Storage' : 'RP',
                     'RT Structure Set Storage' : 'RS',
                     'Computed Radiography Image Storage' : 'CR',
                     'Ultrasound Image Storage' : 'US',
                     'Enhanced Ultrasound Image Storage' : 'USE',
                     'X-Ray Angiographic Image Storage' : 'XA',
                     'Enhanced XA Image Storage' : 'XAE',
                     'Nuclear Medicine Image Storage' : 'NM',
                     'Secondary Capture Image Storage' : 'SC'}

    try:
        mode_prefix = mode_prefixes[dataset.SOPClassUID.__str__()]
    except:
        pass

    filename = '{0!s}.{1!s}'.format(mode_prefix, dataset.SOPInstanceUID)
    LOGGER.info('Storing DICOM file: {0!s}'.format(filename))

    if os.path.exists(filename):
        LOGGER.warning('DICOM file already exists, overwriting')

    meta = Dataset()
    meta.MediaStorageSOPClassUID = dataset.SOPClassUID
    meta.MediaStorageSOPInstanceUID = dataset.SOPInstanceUID
    meta.ImplementationClassUID = pynetdicom_uid_prefix

    ds = FileDataset(filename, {}, file_meta=meta, preamble=b"\0" * 128)
    ds.update(dataset)

    ds.is_little_endian = True
    ds.is_implicit_VR = True

    status_ds = Dataset()

    if not args.ignore:
        # Try to save to output-directory
        if args.output_directory is not None:
            filename = os.path.join(args.output_directory, filename)

        try:
            ds.save_as(filename)
            status_ds.Status = 0x0000 # Success
        except IOError:
            LOGGER.error('Could not write file to specified directory:')
            LOGGER.error("    {0!s}".format(os.path.dirname(filename)))
            LOGGER.error('Directory may not exist or you may not have write '
                    'permission')
            # Failed - Out of Resources - IOError
            status_ds.Status = 0xA700
        except:
            LOGGER.error('Could not write file to specified directory:')
            LOGGER.error("    {0!s}".format(os.path.dirname(filename)))
            # Failed - Out of Resources - Miscellaneous error
            status_ds.Status = 0xA701

    return status_ds

# Test output-directory
if args.output_directory is not None:
    if not os.access(args.output_directory, os.W_OK|os.X_OK):
        LOGGER.error("No write permissions or the output directory may not exist:")
        LOGGER.error("    {0!s}".format(args.output_directory))
        sys.exit()

scp_classes = [x for x in StorageSOPClassList]
scp_classes.append(VerificationSOPClass)

# Create application entity
ae = AE(ae_title=args.aetitle,
        port=args.port,
        scu_sop_class=[],
        scp_sop_class=scp_classes,
        transfer_syntax=transfer_syntax)

ae.maximum_pdu_size = args.max_pdu

# Set timeouts
ae.network_timeout = args.timeout
ae.acse_timeout = args.acse_timeout
ae.dimse_timeout = args.dimse_timeout

ae.on_c_store = on_c_store

ae.start()


print ('<=====CONVERTOR=====>')

num = int(input('Digite um número inteiro: '))
print('''Escolha uma daas bases para a conversão:
[1] Converter para Binário
[2] Converter para Octal
[3] Converter para Hexadecimal''')

opção = int(input('Sua opção: '))

if opção == 1:
    print('{} convertido em binário se torna {}'.format(opção, bin(num)[2:]))

elif opção == 2:
    print ('{} convertido em octal se torna {}'.format(opção, oct(num)[2:]))

elif opção == 3:
    print('{} convertido em hexadecimal se torna {}'.format(opção, hex(num)[2:]))

else:
    print('OPÇÃO INVÁLIDA. TENTE NOVAMENTE.')

Programas/Outros Programas/python desafios/desafio 37.py


import time as t
class Mytime:
    def __init__(self):
        self.cou=''
        self.last=[]
        self.begin=0
        self.end=0
        self.biao=['年','月','日','时','分','秒']

    def __str__(self):
        return self.cou

    __repr__=__str__

    def __add__(self,other):
        cou='一共运行了'
        for i in range(6):
            result=self.last[i]+other.last[i]
            if result:
                cou+=str(result)+self.biao[i]
        return cou

    def start(self):
        self.begin=t.localtime()
        self.cou='请使用stop()方法'
        print('开始计时')

    def stop(self):
        if self.cou=='':
            print('请使用start()方法')
        else:
            self.end=t.localtime()
            print('计时结束')
            self._count()
            print('时间为'+self.cou)

    def _count(self):
        for i in range(6):
            self.last.append(self.end[i]-self.begin[i])
            if self.last[i]:
                self.cou=str(self.last[i])+self.biao[i]
        self.begin=0
        self.end=0


import datetime
import json
import math
import os
import sys
import time
import warnings
from functools import partial
from pathlib import Path
from typing import Dict, Iterable, List, Optional

import numpy as np
import torch
import torch.backends.cudnn as cudnn
import torch.distributed as dist
from einops import rearrange
from torch import Tensor, nn
from torch.nn.parallel import DistributedDataParallel
from torch.utils.data import DataLoader, DistributedSampler, RandomSampler
from tqdm import tqdm

import utils
from demo.app import log_inference
from multimae import MultiMAE
from multimae.criterion import MaskedCrossEntropyLoss, MaskedL1Loss, MaskedMSELoss
from multimae.input_adapters import PatchedInputAdapter, SemSegInputAdapter
from multimae.output_adapters import SpatialOutputAdapter
from pretrain_argparser import PretrainArgparser, get_args
from utils import NativeScalerWithGradNormCount as NativeScaler
from utils import create_model
from utils.data_constants import COCO_SEMSEG_NUM_CLASSES
from utils.datasets import (
    build_multimae_pretraining_dev_dataset_v2,
    build_multimae_pretraining_train_dataset_v2,
)
from utils.optim_factory import create_optimizer

CPU_DEVICE = torch.device("cpu")

DOMAIN_CONF = {
    "rgb": {
        "channels": 3,
        "stride_level": 1,
        "input_adapter": partial(PatchedInputAdapter, num_channels=3),
        "output_adapter": partial(SpatialOutputAdapter, num_channels=3),
        "loss": MaskedMSELoss,
    },
    "depth": {
        "channels": 1,
        "stride_level": 1,
        "input_adapter": partial(PatchedInputAdapter, num_channels=1),
        "output_adapter": partial(SpatialOutputAdapter, num_channels=1),
        "loss": MaskedL1Loss,
    },
    "semseg": {
        "num_classes": 133,
        "stride_level": 4,
        "input_adapter": partial(
            SemSegInputAdapter,
            num_classes=COCO_SEMSEG_NUM_CLASSES,
            dim_class_emb=64,
            interpolate_class_emb=False,
        ),
        "output_adapter": partial(
            SpatialOutputAdapter, num_channels=COCO_SEMSEG_NUM_CLASSES
        ),
        "loss": partial(MaskedCrossEntropyLoss, label_smoothing=0.0),
    },
}

def get_model(args: PretrainArgparser) -> MultiMAE:
    """Creates and returns model from arguments"""
    print(
        f"Creating model: {args.model} for inputs {args.in_domains} and outputs {args.out_domains}"
    )

    input_adapters = {
        domain: DOMAIN_CONF[domain]["input_adapter"](
            stride_level=DOMAIN_CONF[domain]["stride_level"],
            patch_size_full=args.patch_size,
        )
        for domain in args.in_domains
    }

    output_adapters = {
        domain: DOMAIN_CONF[domain]["output_adapter"](
            stride_level=DOMAIN_CONF[domain]["stride_level"],
            patch_size_full=args.patch_size,
            dim_tokens=args.decoder_dim,
            depth=args.decoder_depth,
            num_heads=args.decoder_num_heads,
            use_task_queries=args.decoder_use_task_queries,
            task=domain,
            context_tasks=list(args.in_domains),
            use_xattn=args.decoder_use_xattn,
        )
        for domain in args.out_domains
    }

    # Add normalized pixel output adapter if specified
    if args.extra_norm_pix_loss:
        output_adapters["norm_rgb"] = DOMAIN_CONF["rgb"]["output_adapter"](
            stride_level=DOMAIN_CONF["rgb"]["stride_level"],
            patch_size_full=args.patch_size,
            dim_tokens=args.decoder_dim,
            depth=args.decoder_depth,
            num_heads=args.decoder_num_heads,
            use_task_queries=args.decoder_use_task_queries,
            task="rgb",
            context_tasks=list(args.in_domains),
            use_xattn=args.decoder_use_xattn,
        )

    model = create_model(
        args.model,
        input_adapters=input_adapters,
        output_adapters=output_adapters,
        num_global_tokens=args.num_global_tokens,
        drop_path_rate=args.drop_path,
    )

    return model

def main(args: PretrainArgparser):
    utils.init_distributed_mode(args)
    device = torch.device(args.device)

    # Fix the seed for reproducibility
    seed = args.seed + utils.get_rank()
    torch.manual_seed(seed)
    np.random.seed(seed)
    # random.seed(seed)

    cudnn.benchmark = True

    if not args.show_user_warnings:
        warnings.filterwarnings("ignore", category=UserWarning)

    args.in_domains = args.in_domains.split("-")
    args.out_domains = args.out_domains.split("-")
    args.all_domains = list(set(args.in_domains) | set(args.out_domains))

    model = get_model(args)

    tasks_loss_fn = {
        domain: DOMAIN_CONF[domain]["loss"](
            patch_size=args.patch_size, stride=DOMAIN_CONF[domain]["stride_level"]
        )
        for domain in args.out_domains
    }

    # Add normalized pixel loss if specified
    if args.extra_norm_pix_loss:
        tasks_loss_fn["norm_rgb"] = DOMAIN_CONF["rgb"]["loss"](
            patch_size=args.patch_size,
            stride=DOMAIN_CONF["rgb"]["stride_level"],
            norm_pix=True,
        )

    # Get dataset
    dataset_train = build_multimae_pretraining_train_dataset_v2(args)
    dataset_dev = build_multimae_pretraining_dev_dataset_v2(args)

    if True:  # args.distributed:
        world_size = utils.get_world_size()
        global_rank = utils.get_rank()
        print("global_rank", global_rank)
        sampler_rank = global_rank
        num_training_steps_per_epoch = (
            len(dataset_train) // args.batch_size // world_size
        )

        sampler_train = DistributedSampler(
            dataset_train,
            num_replicas=world_size,
            rank=sampler_rank,
            shuffle=True,
            drop_last=True,
        )
        # sampler_dev = DistributedSampler(
        #     dataset_dev, num_replicas=num_tasks,
        #     rank=sampler_rank, shuffle=False, drop_last=False,
        # )
        print("Sampler_train = %s" % str(sampler_train))
    else:
        sampler_train = RandomSampler(dataset_train)

    print(args)

    data_loader_train = DataLoader(
        dataset_train,
        sampler=sampler_train,
        batch_size=args.batch_size,
        num_workers=args.num_workers,
        pin_memory=args.pin_mem,
        drop_last=True,
    )
    # data_loader_dev = DataLoader(
    #     dataset_dev, sampler=sampler_dev,
    #     batch_size=args.batch_size,
    #     num_workers=args.num_workers,
    #     pin_memory=args.pin_mem,
    #     drop_last=False,
    # )

    model.to(device)
    model_without_ddp = model
    n_parameters = sum(p.numel() for p in model.parameters() if p.requires_grad)

    # print(f"Model = %s" % str(model_without_ddp))
    print(f"Number of params: {n_parameters / 1e6} M")

    total_batch_size = args.batch_size * utils.get_world_size()
    args.lr = args.blr * total_batch_size / 256

    print("LR = %.8f" % args.lr)
    print("Batch size = %d" % total_batch_size)
    print("Number of training steps = %d" % num_training_steps_per_epoch)
    print(
        "Number of training examples per epoch = %d"
        % (total_batch_size * num_training_steps_per_epoch)
    )

    if args.distributed:
        model = DistributedDataParallel(model, device_ids=[args.gpu])
        model_without_ddp = model.module

    optimizer = create_optimizer(args, {"model": model_without_ddp})
    loss_scaler = NativeScaler()

    print("Use step level LR & WD scheduler!")
    lr_schedule_values = utils.cosine_scheduler(
        args.lr,
        args.min_lr,
        args.epochs,
        num_training_steps_per_epoch,
        warmup_epochs=args.warmup_epochs,
        warmup_steps=args.warmup_steps,
    )
    if args.weight_decay_end is None:
        args.weight_decay_end = args.weight_decay
    wd_schedule_values = utils.cosine_scheduler(
        args.weight_decay,
        args.weight_decay_end,
        args.epochs,
        num_training_steps_per_epoch,
    )
    print(
        "Max WD = %.7f, Min WD = %.7f"
        % (max(wd_schedule_values), min(wd_schedule_values))
    )

    utils.auto_load_model(
        args=args,
        model=model,
        model_without_ddp=model_without_ddp,
        optimizer=optimizer,
        loss_scaler=loss_scaler,
    )

    if global_rank == 0 and args.log_wandb:
        log_writer = utils.WandbLogger(args)
        log_writer.set_step(0)
    else:
        log_writer = None

    if global_rank == 0:
        log_inference(
            model,
            seed,
            dataset_dev,
            log_writer,
            epoch=args.start_epoch,
            num_samples=10,
        )

    if log_writer is not None:
        log_writer.finish()

if __name__ == "__main__":
    raise Exception("Deprecated!")
    opts = get_args()
    if opts.output_dir:
        Path(opts.output_dir).mkdir(parents=True, exist_ok=True)
    main(opts)


"""Generic platform functions."""
# Standard Library
from collections import defaultdict

# Lutris Modules
from lutris import runners

# gets populated by _init_platforms()
__all__ = defaultdict(list)

def _init_platforms():
    for runner_name in runners.__all__:
        runner = runners.import_runner(runner_name)()
        for platform in runner.platforms:
            __all__[platform].append(runner_name)

_init_platforms()

LONG_PLATFORM_OVERRIDES = {
    "Odyssey": "Magnavox Odyssey",
    "Intellivision": "Mattel Intellivision",
    "PC-8000 / PC-8800 series": "NEC PC-88",
    "Vectrex": "GCE Vectrex",
    "VIC20": "Commodore VIC20",
    "CPC": "Amstrad CPC",
    "MSX": "Microsoft MSX",
    "Nintendo Entertainment System": "Nintendo NES",
    "Master System": "Sega Master System",
    "PC-98": "NEC PC-98",
    "ZX81": "Sinclair ZX81",
    "ZX Spectrum (various)": "Sinclair ZX Spectrum",
    "Game Boy/Game Boy Color": "Nintendo Game Boy/Game Boy Color",
    "Game Gear": "Sega Game Gear",
    "Genesis": "Sega Genesis",
    "Lynx": "Atari Lynx",
    "Neo Geo": "SNK Neo Geo",
    "32X": "Sega 32X",
    "X68000": "Sharp X68000",
    "Super Nintendo Entertainment System": "Nintendo SNES",
    "Jaguar": "Atari Jaguar",
    "PC-FX": "NEC PC-FX",
    "PlayStation": "Sony PlayStation",
    "Saturn": "Sega Saturn",
    "Virtual Boy": "Nintendo Virtual Boy",
    "WonderSwan/Color": "Bandai WonderSwan/WonderSwan Color",
    "Dreamcast": "Sega Dreamcast",
    "Game Boy Advance": "Nintendo Game Boy Advance",
    "Gamecube": "Nintendo Gamecube",
    "Neo Geo Pcket (Color)": "SNK Neo Geo Pocket/Neo Geo Pocket Color",
    "PlayStation 2": "Sony PlayStation 2",
    "Xbox": "Microsoft Xbox",
    "DS": "Nintendo DS",
    "PlayStation 3": "Sony PlayStation 3",
    "PlayStation Portable": "Sony PlayStation Portable",
    "Wii": "Nintendo Wii",
    "Xbox 360": "Microsoft Xbox 360",
    "3DS": "Nintendo 3DS",
    "PlayStation Vita": "Sony PlayStation Vita",
    "Wii U": "Nintendo Wii U",
    "Switch": "Nintendo Switch",
    "PlayStation 4": "Sony PlayStation 4",
    "Stadia": "Google Stadia",
    "Xbox One": "Microsoft Xbox One"
}


import os
import openai
import asyncio
import requests
from settings import settings as _st

openai.api_key = _st.OPEN_AI_KEY  # os.environ.get("OPENAI_API_KEY")

class ChatPGT:

    def __init__(self, phrase: str) -> None:

        self.phrase = phrase
        self._response = {}

    def get_answer(self) -> str:

        result = ""

        try:

            self._response = openai.Completion.create(
                        model="text-davinci-003",
                        prompt=self.phrase,
                        max_tokens=600,
                        temperature=0.8)

            for answer in self._response.get("choices", []):
                result += answer["text"]

        except openai.error.OpenAIError as e:
            print(e.http_status)
            print(e.error)
            print(e.args)

        return result

def main():

    phrase = "I have already told everyone no hookahs in our house and other smoking.Well, " \
             "like let them do what they want, but if not with us)"

    print(ChatPGT(phrase).get_answer())

if __name__ == "__main__":
    main()


import json
from flask import Flask, jsonify, request, abort, redirect
import requests

app = Flask(__name__)

# Configuration needed by SQLAlchemy
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///ips_data.db'
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False

# Instead of using db = SQLAlchemy(app)
db.init_app(app)

# PROCESS VARIABLES

@app.route('/process_variables/<run_id>', methods=['POST'])
def create_process_variables(run_id):

    if not request.json:
        abort(400)

    data = request.json

    for rec in data:
        new_pv = ProcessVariables(RUN_ID=run_id,
                                  PV_NAME=rec['PV_NAME'],
                                  PV_CONTENT=rec['PV_CONTENT'],
                                  PV_REASON=rec['PV_REASON'])
        db.session.add(new_pv)
    db.session.commit()

    return "", 201

# Old method copying from tamplate

# @app.route('/process_variables/<run_id>/<template_id>', methods=['POST'])
# def create_process_variables(run_id, template_id):
#
#     # Get the process variables records associated with the template id
#     data = ProcessVariables.query.filter_by(RUN_ID=template_id).all()
#
#     # Quit if no date is returned (the template id has no associated process variables)
#     if not data:
#         abort(400)
#
#     # Loop through and copy the template into the new pv set using the run id specified.
#     for rec in data:
#         new_pv = ProcessVariables(RUN_ID=run_id,
#                                   PV_NAME=rec.PV_NAME,
#                                   PV_CONTENT=rec.PV_CONTENT,
#                                   PV_REASON=rec.PV_REASON)
#         db.session.add(new_pv)
#
#     db.session.commit()
#
#     return "", 201

@app.route('/process_variables', methods=['GET'])
@app.route('/process_variables/<run_id>', methods=['GET'])
def get_process_variables(run_id=None):
    column_names = ['RUN_ID', 'PV_NAME', 'PV_CONTENT', 'PV_REASON']
    # if a run id is provided search for this specific run
    if run_id:
        # Get records relating to that pv
        data = ProcessVariables.query.filter_by(RUN_ID=run_id).all()
    else:
        # Get all records
        data = ProcessVariables.query.all()

    if not data:
        abort(400)

    output = []
    for rec in data:
        output_record = {}
        for name in column_names:
            output_record[name] = getattr(rec, name)
        output.append(output_record)

    return jsonify(output)

@app.route('/process_variables', methods=['PUT'])
def update_process_variable():

    # the request should be json and an id must be present
    if not request.json:
        abort(400)

    updated_data = request.json

    # Get records relating to that pv
    data = ProcessVariables.query.filter_by(RUN_ID=updated_data['RUN_ID']).all()

    if not data:
        abort(400)

    for rec in data:
        if rec.PV_NAME in updated_data['PV_NAME']:
            rec.PV_CONTENT = updated_data['PV_CONTENT']
            rec.PV_REASON = updated_data['PV_REASON']

    db.session.commit()

    return "", 200

# SHIFT DATA

@app.route('/SHIFT_DATA/<run_id>', methods=['POST'])
@app.route('/shift_data/<run_id>', methods=['POST'])
def import_shift_data(run_id):

    if not request.json:
        abort(400)

    data = request.json

    for rec in data:
        run = ShiftData(RUN_ID=run_id,
                        YEAR=rec['YEAR'],
                        MONTH=rec['MONTH'],
                        DATASOURCE=rec['DATASOURCE'],
                        PORTROUTE=rec['PORTROUTE'],
                        WEEKDAY=rec['WEEKDAY'],
                        ARRIVEDEPART=rec['ARRIVEDEPART'],
                        TOTAL=rec['TOTAL'],
                        AM_PM_NIGHT=rec['AM_PM_NIGHT'])
        db.session.add(run)
    db.session.commit()

    return "", 200

@app.route('/SHIFT_DATA', methods=['GET'])
@app.route('/shift_data', methods=['GET'])
@app.route('/SHIFT_DATA/<run_id>', methods=['GET'])
@app.route('/shift_data/<run_id>', methods=['GET'])
@app.route('/SHIFT_DATA/<run_id>/<data_source>', methods=['GET'])
@app.route('/shift_data/<run_id>/<data_source>', methods=['GET'])
def get_shift_data(run_id=None, data_source='0'):
    column_names = ['RUN_ID', 'YEAR', 'MONTH', 'DATASOURCE', 'PORTROUTE', 'WEEKDAY', 'ARRIVEDEPART', 'TOTAL',
                    'AM_PM_NIGHT']

    data = ShiftData.query.all()

    if not data:
        abort(400)

    output = []
    for rec in data:
        output_record = {}
        for name in column_names:
            output_record[name] = getattr(rec, name)
        output.append(output_record)

    if run_id:
        run_filtered = []

        for rec in output:
            if rec['RUN_ID'] == run_id:
                run_filtered.append(rec)

        if len(run_filtered) == 0:
            abort(400)

        output = run_filtered

    if data_source != '0':
        ds_filtered = []

        for rec in output:
            if rec['DATASOURCE'] == data_source:
                ds_filtered.append(rec)

        if len(ds_filtered) == 0:
            abort(400)

        output = ds_filtered

    return jsonify(output)

@app.route('/SHIFT_DATA/<run_id>', methods=['DELETE'])
@app.route('/shift_data/<run_id>', methods=['DELETE'])
def delete_shift_data(run_id=None):
    data = ShiftData.query.filter_by(RUN_ID=run_id).all()
    for rec in data:
        db.session.delete(rec)

    db.session.commit()

    return "", 200

# NON RESPONSE DATA

@app.route('/NON_RESPONSE_DATA/<run_id>', methods=['POST'])
@app.route('/non_response_data/<run_id>', methods=['POST'])
def import_non_response_data(run_id):

    if not request.json:
        abort(400)

    data = request.json

    for rec in data:
        run = NonResponseData(RUN_ID=run_id,
                              YEAR=rec['YEAR'],
                              MONTH=rec['MONTH'],
                              DATASOURCE=rec['DATASOURCE'],
                              PORTROUTE=rec['PORTROUTE'],
                              WEEKDAY=rec['WEEKDAY'],
                              ARRIVEDEPART=rec['ARRIVEDEPART'],
                              AM_PM_NIGHT=rec['AM_PM_NIGHT'],
                              SAMPINTERVAL=rec['SAMPINTERVAL'],
                              MIGTOTAL=rec['MIGTOTAL'],
                              ORDTOTAL=rec['ORDTOTAL'])
        db.session.add(run)
    db.session.commit()

    return "", 200

@app.route('/NON_RESPONSE_DATA', methods=['GET'])
@app.route('/non_response_data', methods=['GET'])
@app.route('/NON_RESPONSE_DATA/<run_id>', methods=['GET'])
@app.route('/non_response_data/<run_id>', methods=['GET'])
@app.route('/NON_RESPONSE_DATA/<run_id>/<data_source>', methods=['GET'])
@app.route('/non_response_data/<run_id>/<data_source>', methods=['GET'])
def get_non_response_data(run_id=None, data_source='0'):
    column_names = ['RUN_ID', 'YEAR', 'MONTH', 'DATASOURCE', 'PORTROUTE', 'WEEKDAY', 'ARRIVEDEPART', 'AM_PM_NIGHT',
                    'SAMPINTERVAL', 'MIGTOTAL', 'ORDTOTAL']

    data = NonResponseData.query.all()

    if not data:
        abort(400)

    output = []
    for rec in data:
        output_record = {}
        for name in column_names:
            output_record[name] = getattr(rec, name)
        output.append(output_record)

    if run_id:
        run_filtered = []

        for rec in output:
            if rec['RUN_ID'] == run_id:
                run_filtered.append(rec)

        if len(run_filtered) == 0:
            abort(400)

        output = run_filtered

    if data_source != '0':
        ds_filtered = []

        for rec in output:
            if rec['DATASOURCE'] == data_source:
                ds_filtered.append(rec)

        if len(ds_filtered) == 0:
            abort(400)

        output = ds_filtered

    return jsonify(output)

@app.route('/NON_RESPONSE_DATA/<run_id>', methods=['DELETE'])
@app.route('/non_response_data/<run_id>', methods=['DELETE'])
def delete_non_response_data(run_id=None):
    data = NonResponseData.query.filter_by(RUN_ID=run_id).all()
    for rec in data:
        db.session.delete(rec)

    db.session.commit()

    return "", 200

# TRAFFIC DATA

@app.route('/TRAFFIC_DATA/<run_id>', methods=['POST'])
@app.route('/traffic_data/<run_id>', methods=['POST'])
def import_traffic_data(run_id):

    if not request.json:
        abort(400)

    data = request.json

    for rec in data:
        run = TrafficData(RUN_ID=run_id,
                          YEAR=rec['YEAR'],
                          MONTH=rec['MONTH'],
                          DATASOURCE=rec['DATASOURCE'],
                          PORTROUTE=rec['PORTROUTE'],
                          ARRIVEDEPART=rec['ARRIVEDEPART'],
                          TRAFFICTOTAL=rec['TRAFFICTOTAL'],
                          PERIODSTART=rec['PERIODSTART'],
                          PERIODEND=rec['PERIODEND'],
                          AM_PM_NIGHT=rec['AM_PM_NIGHT'],
                          HAUL=rec['HAUL'],
                          VEHICLE='')
        db.session.add(run)
    db.session.commit()

    return "", 200

@app.route('/TRAFFIC_DATA', methods=['GET'])
@app.route('/traffic_data', methods=['GET'])
@app.route('/TRAFFIC_DATA/<run_id>', methods=['GET'])
@app.route('/traffic_data/<run_id>', methods=['GET'])
@app.route('/TRAFFIC_DATA/<run_id>/<data_source>', methods=['GET'])
@app.route('/traffic_data/<run_id>/<data_source>', methods=['GET'])
def get_traffic_data(run_id=None, data_source='0'):
    column_names = ['RUN_ID', 'YEAR', 'MONTH', 'DATASOURCE', 'PORTROUTE', 'ARRIVEDEPART', 'TRAFFICTOTAL',
                    'PERIODSTART', 'PERIODEND', 'AM_PM_NIGHT', 'HAUL', 'VEHICLE']

    data = TrafficData.query.all()

    if not data:
        abort(400)

    output = []
    for rec in data:
        output_record = {}
        for name in column_names:
            output_record[name] = getattr(rec, name)
        output.append(output_record)

    if run_id:
        run_filtered = []

        for rec in output:
            if rec['RUN_ID'] == run_id:
                run_filtered.append(rec)

        if len(run_filtered) == 0:
            abort(400)

        output = run_filtered

    if data_source != '0':
        ds_filtered = []

        for rec in output:
            if rec['DATASOURCE'] == data_source:
                ds_filtered.append(rec)

        if len(ds_filtered) == 0:
            abort(400)

        output = ds_filtered

    return jsonify(output)

@app.route('/TRAFFIC_DATA/<run_id>', methods=['DELETE'])
@app.route('/traffic_data/<run_id>', methods=['DELETE'])
def delete_traffic_data(run_id=None):
    data = TrafficData.query.filter_by(RUN_ID=run_id).all()
    for rec in data:
        db.session.delete(rec)

    db.session.commit()

    return "", 200

# UNSAMPLED OOH DATA

@app.route('/UNSAMPLED_OOH_DATA/<run_id>', methods=['POST'])
@app.route('/unsampled_ooh_data/<run_id>', methods=['POST'])
def import_unsampled_data(run_id):

    if not request.json:
        abort(400)

    data = request.json

    for rec in data:
        run = UnsampledOOHData(RUN_ID=run_id,
                               YEAR=rec['YEAR'],
                               MONTH=rec['MONTH'],
                               DATASOURCE=rec['DATASOURCE'],
                               PORTROUTE=rec['PORTROUTE'],
                               REGION=rec['REGION'],
                               ARRIVEDEPART=rec['ARRIVEDEPART'],
                               UNSAMP_TOTAL=rec['UNSAMP_TOTAL'])
        db.session.add(run)

    db.session.commit()

    return "", 200

@app.route('/UNSAMPLED_OOH_DATA', methods=['GET'])
@app.route('/unsampled_ooh_data', methods=['GET'])
@app.route('/UNSAMPLED_OOH_DATA/<run_id>', methods=['GET'])
@app.route('/unsampled_ooh_data/<run_id>', methods=['GET'])
@app.route('/UNSAMPLED_OOH_DATA/<run_id>/<data_source>', methods=['GET'])
@app.route('/unsampled_ooh_data/<run_id>/<data_source>', methods=['GET'])
def get_unsampled_data(run_id=None, data_source='0'):
    column_names = ['RUN_ID', 'YEAR', 'MONTH', 'DATASOURCE', 'PORTROUTE', 'REGION', 'ARRIVEDEPART', 'UNSAMP_TOTAL']

    data = UnsampledOOHData.query.all()

    if not data:
        abort(400)

    output = []
    for rec in data:
        output_record = {}
        for name in column_names:
            output_record[name] = getattr(rec, name)
        output.append(output_record)

    if run_id:
        run_filtered = []

        for rec in output:
            if rec['RUN_ID'] == run_id:
                run_filtered.append(rec)

        if len(run_filtered) == 0:
            abort(400)

        output = run_filtered

    if data_source != '0':
        ds_filtered = []

        for rec in output:
            if rec['DATASOURCE'] == data_source:
                ds_filtered.append(rec)

        if len(ds_filtered) == 0:
            abort(400)

        output = ds_filtered

    return jsonify(output)

@app.route('/UNSAMPLED_OOH_DATA/<run_id>', methods=['DELETE'])
@app.route('/unsampled_ooh_data/<run_id>', methods=['DELETE'])
def delete_unsampled_data(run_id=None):
    data = UnsampledOOHData.query.filter_by(RUN_ID=run_id).all()
    for rec in data:
        db.session.delete(rec)

    db.session.commit()

    return "", 200

# IMBALANCE WEIGHT

@app.route('/IMBALANCE_WEIGHT', methods=['GET'])
@app.route('/imbalance_weight', methods=['GET'])
@app.route('/IMBALANCE_WEIGHT/<run_id>', methods=['GET'])
@app.route('/imbalance_weight/<run_id>', methods=['GET'])
def get_imbalance_weight(run_id=None):
    column_names = ['RUN_ID', 'FLOW', 'SUM_PRIOR_WT', 'SUM_IMBAL_WT']

    data = ImbalanceWeight.query.all()

    if not data:
        abort(400)

    output = []
    for rec in data:
        output_record = {}
        for name in column_names:
            output_record[name] = getattr(rec, name)
        output.append(output_record)

    if run_id:
        run_filtered = []

        for rec in output:
            if rec['RUN_ID'] == run_id:
                run_filtered.append(rec)

        if len(run_filtered) == 0:
            abort(400)

        output = run_filtered

    return jsonify(output)

# JACK - EXPORT DATA

@app.route('/EXPORT_DATA_DOWNLOAD', methods=['POST'])
@app.route('/export_data_download', methods=['POST'])
def create_export_data_download():
    # function_name = {"Survey Subsample": None,
    #                  "Final Weight Summary": None,
    #                  "Shift": get_shift_data,
    #                  "Non-Response": get_non_response_data,
    #                  "Shift Weight Summary": None,
    #                  "Non Response Weight Summary": None,
    #                  "Minimum Weight Summary": None,
    #                  "Traffic Weight Summary": get_traffic_data,
    #                  "Unsampled Traffic Weight Summary": get_unsampled_data,
    #                  "Imbalance Weight Summary": get_imbalance_weight}

    # the request should be json
    if not request.data:
        abort(400)

    # Get json dictionary and values
    json_data = json.loads(request.data)
    # table_name = json_data["SOURCE_TABLE"]

    table = requests.get('http://ips-db.apps.cf1.ons.statistics.gov.uk/IMBALANCE_WEIGHT/' + json_data['RUN_ID'])

    table_data = table.text
    # Convert json import to string

    new_rec = ExportDataDownload(RUN_ID=json_data['RUN_ID'],
                                 DOWNLOADABLE_DATA=table_data,
                                 FILENAME=json_data['FILENAME'],
                                 SOURCE_TABLE=json_data['SOURCE_TABLE'],
                                 DATE_CREATED=json_data['DATE_CREATED'])
    db.session.add(new_rec)
    db.session.commit()

    return "", 201

@app.route('/EXPORT_DATA_DOWNLOAD/<run_id>', methods=['GET'])
@app.route('/export_data_download/<run_id>', methods=['GET'])
@app.route('/EXPORT_DATA_DOWNLOAD', methods=['GET'])
@app.route('/export_data_download', methods=['GET'])
@app.route('/EXPORT_DATA_DOWNLOAD/<run_id>/<file_name>/<source_table>', methods=['GET'])
@app.route('/export_data_download/<run_id>/<file_name>/<source_table>', methods=['GET'])
def get_export_data_download(run_id=None, file_name=None, source_table=None):
    column_names = ['RUN_ID', 'DOWNLOADABLE_DATA', 'FILENAME', 'SOURCE_TABLE', 'DATE_CREATED']

    data = ExportDataDownload.query.all()

    if not data:
        abort(400)

    output = []
    for rec in data:
        output_record = {}
        for name in column_names:
            output_record[name] = getattr(rec, name)
        output.append(output_record)

    # Filter by run_id if provided
    if run_id:
        run_filtered = []

        for rec in output:
            if rec['RUN_ID'] == run_id:
                run_filtered.append(rec)

        if len(run_filtered) == 0:
            abort(400)

        output = run_filtered

    # Filter by run_id, file_name and source table if provided
    if run_id and file_name:
        run_filtered = ""

        for rec in output:
            if rec['RUN_ID'] == run_id and rec['FILENAME'] == file_name and rec['SOURCE_TABLE'] == source_table:
                run_filtered = rec

        if len(run_filtered) == 0:
            abort(400)

        output = run_filtered

    json_output = jsonify(output)
    return json_output

@app.route('/test_json', methods=['GET'])
def get_json():
    output = []
    json_data = {'RUN_ID': 'Automated-Run_ID', 'YEAR': 'YEAR', 'MONTH': 'MONTH',
                 'DATASOURCE': 'DATASOURCE', 'PORTROUTE': 'PORTROUTE', 'WEEKDAY': 'WEEKDAY',
                 'ARRIVEDEPART': 'ARRIVEDEPART', 'TOTAL': 'TOTAL', 'AM_PM_NIGHT': 'AM_PM_NIGHT'}
    json_data2 = {'RUN_ID': 'Automated-Run_ID', 'YEAR': 'YEAR-2', 'MONTH': 'MONTH-2',
                 'DATASOURCE': 'DATASOURCE-2', 'PORTROUTE': 'PORTROUTE-2', 'WEEKDAY': 'WEEKDAY-2',
                 'ARRIVEDEPART': 'ARRIVEDEPART-2', 'TOTAL': 'TOTAL-2', 'AM_PM_NIGHT': 'AM_PM_NIGHT-2'}

    output.append(json_data)
    output.append(json_data2)

    return jsonify(output)

# SURVEY SUBSAMPLE

@app.route('/SURVEY_SUBSAMPLE', methods=['GET'])
@app.route('/survey_subsample', methods=['GET'])
@app.route('/SURVEY_SUBSAMPLE/<run_id>', methods=['GET'])
@app.route('/survey_subsample/<run_id>', methods=['GET'])
def get_survey_subsample_data(run_id=None):
    column_names = ['RUN_ID', 'SERIAL', 'AGE', 'AM_PM_NIGHT', 'ANYUNDER16', 'APORTLATDEG', 'APORTLATMIN', 'APORTLATSEC',
                    'APORTLATNS', 'APORTLONDEG', 'APORTLONMIN', 'APORTLONSEC', 'APORTLONEW', 'ARRIVEDEPART', 'BABYFARE',
                    'BEFAF', 'CHANGECODE', 'CHILDFARE', 'COUNTRYVISIT', 'CPORTLATDEG', 'CPORTLATMIN', 'CPORTLATSEC',
                    'CPORTLATNS', 'CPORTLONDEG', 'CPORTLONMIN', 'CPORTLONSEC', 'CPORTLONEW', 'INTDATE', 'DAYTYPE',
                    'DIRECTLEG', 'DVEXPEND', 'DVFARE', 'DVLINECODE', 'DVPACKAGE', 'DVPACKCOST', 'DVPERSONS',
                    'DVPORTCODE', 'EXPENDCODE', 'EXPENDITURE', 'FARE', 'FAREK', 'FLOW', 'HAULKEY', 'INTENDLOS',
                    'KIDAGE', 'LOSKEY', 'MAINCONTRA', 'MIGSI', 'INTMONTH', 'NATIONALITY', 'NATIONNAME', 'NIGHTS1',
                    'NIGHTS2', 'NIGHTS3', 'NIGHTS4', 'NIGHTS5', 'NIGHTS6', 'NIGHTS7', 'NIGHTS8', 'NUMADULTS', 'NUMDAYS',
                    'NUMNIGHTS', 'NUMPEOPLE', 'PACKAGEHOL', 'PACKAGEHOLUK', 'PERSONS', 'PORTROUTE', 'PACKAGE',
                    'PROUTELATDEG', 'PROUTELATMIN', 'PROUTELATSEC', 'PROUTELATNS', 'PROUTELONDEG', 'PROUTELONMIN',
                    'PROUTELONSEC', 'PROUTELONEW', 'PURPOSE', 'QUARTER', 'RESIDENCE', 'RESPNSE', 'SEX', 'SHIFTNO',
                    'SHUTTLE', 'SINGLERETURN', 'TANDTSI', 'TICKETCOST', 'TOWNCODE1', 'TOWNCODE2', ' TOWNCODE3',
                    'TOWNCODE4', 'TOWNCODE5', 'TOWNCODE6', 'TOWNCODE7', 'TOWNCODE8', 'TRANSFER', 'UKFOREIGN', 'VEHICLE',
                    'VISITBEGAN', 'WELSHNIGHTS', 'WELSHTOWN', 'AM_PM_NIGHT_PV', 'APD_PV', 'ARRIVEDEPART_PV',
                    'CROSSINGS_FLAG_PV', 'STAYIMPCTRYLEVEL1_PV', 'STAYIMPCTRYLEVEL2_PV', 'STAYIMPCTRYLEVEL3_PV',
                    'STAYIMPCTRYLEVEL4_PV', 'DAY_PV', 'DISCNT_F1_PV', 'DISCNT_F2_PV', 'DISCNT_PACKAGE_COST_PV',
                    'DUR1_PV', 'DUR2_PV', 'DUTY_FREE_PV', 'FAGE_PV', 'FARES_IMP_ELIGIBLE_PV', 'FARES_IMP_FLAG_PV',
                    'FLOW_PV', 'FOOT_OR_VEHICLE_PV', 'HAUL_PV', 'IMBAL_CTRY_FACT_PV', 'IMBAL_CTRY_GRP_PV',
                    'IMBAL_ELIGIBLE_PV', 'IMBAL_PORT_FACT_PV', 'IMBAL_PORT_GRP_PV', 'IMBAL_PORT_SUBGRP_PV', 'LOS_PV',
                    'LOSDAYS_PV', 'MIG_FLAG_PV', 'MINS_CTRY_GRP_PV', 'MINS_CTRY_PORT_GRP_PV', 'MINS_FLAG_PV',
                    'MINS_NAT_GRP_PV', 'MINS_PORT_GRP_PV', 'MINS_QUALITY_PV', 'NR_FLAG_PV', 'NR_PORT_GRP_PV',
                    'OPERA_PV', 'OSPORT1_PV', 'OSPORT2_PV', 'OSPORT3_PV', 'OSPORT4_PV', 'PUR1_PV', 'PUR2_PV', 'PUR3_PV',
                    'PURPOSE_PV', 'QMFARE_PV', 'RAIL_CNTRY_GRP_PV', 'RAIL_EXERCISE_PV', 'RAIL_IMP_ELIGIBLE_PV',
                    'REG_IMP_ELIGIBLE_PV', 'SAMP_PORT_GRP_PV', 'SHIFT_FLAG_PV', 'SHIFT_PORT_GRP_PV',
                    'SPEND_IMP_FLAG_PV', 'SPEND_IMP_ELIGIBLE_PV', 'STAY_IMP_ELIGIBLE_PV', 'STAY_IMP_FLAG_PV',
                    'STAY_PURPOSE_GRP_PV', 'TOWNCODE_PV', 'TOWN_IMP_ELIGIBLE_PV', 'TYPE_PV', 'UK_OS_PV', 'UKPORT1_PV',
                    'UKPORT2_PV', 'UKPORT3_PV', 'UKPORT4_PV', 'UNSAMP_PORT_GRP_PV', 'UNSAMP_REGION_GRP_PV',
                    'WEEKDAY_END_PV', 'DIRECT', 'EXPENDITURE_WT', 'EXPENDITURE_WTK', 'FAREKEY', 'OVLEG', 'SPEND',
                    'SPEND1', 'SPEND2', 'SPEND3', 'SPEND4', 'SPEND5', 'SPEND6', 'SPEND7', 'SPEND8', 'SPEND9',
                    'SPENDIMPREASON', 'SPENDK', 'STAY', 'STAYK', 'STAY1K', 'STAY2K', 'STAY3K', 'STAY4K', 'STAY5K',
                    'STAY6K', 'STAY7K', 'STAY8K', 'STAY9K', 'STAYTLY', 'STAY_WT', 'STAY_WTK', 'TYPEINTERVIEW', 'UKLEG',
                    'VISIT_WT', 'VISIT_WTK', 'SHIFT_WT', 'NON_RESPONSE_WT', 'MINS_WT', 'TRAFFIC_WT',
                    'UNSAMP_TRAFFIC_WT', 'IMBAL_WT', 'FINAL_WT']

    data = SurveySubsample.query.all()

    if not data:
        abort(400)

    output = []
    for rec in data:
        output_record = {}
        for name in column_names:
            output_record[name] = getattr(rec, name)
        output.append(output_record)

    if run_id:
        run_filtered = []

        for rec in output:
            if rec['RUN_ID'] == run_id:
                run_filtered.append(rec)

        if len(run_filtered) == 0:
            abort(400)

        output = run_filtered

    return jsonify(output)


import numpy as np
from tqdm import tqdm

import mediapy as mp

# 生日快乐歌
birthday = """
5 5 6 5 1^ 7
5 5 6 5 2^ 1^
5 5 5^ 3^ 1^ 7 6
4^ 4^ 3^ 1^ 2^ 1^ 1^
"""
rate, a = mp.nmn.get_float_array(birthday, rate=10000)

def sfft(a, rate: int, window: int, freq_bei: float, duration_bei: float, top: int = 4):
    b = []
    phi = 0
    for i in tqdm(range(0, len(a), window)):
        f = np.fft.fft(a[i:i + window])
        enegy = np.abs(f)
        ind = np.argmax(enegy[:len(f) // 2])
        freq = ind / len(f) * rate
        freq = freq * freq_bei
        duration = window / rate * duration_bei
        part = mp.nmn.freq2wave(freq, rate, duration=duration, phi=phi, freq_wav_type=mp.nmn.FreqWaveType.constant)
        b.append(part)
        # 对于短时傅里叶变换，维持原相位效果最好
        phi += freq * np.pi * 2 * duration
    return mp.nmn.concatenate_clips(b)

high = sfft(a, rate=rate, window=500, freq_bei=2, duration_bei=1, top=1)
low = sfft(a, rate=rate, window=500, freq_bei=0.5, duration_bei=1, top=1)
for i in (high, low, a):
    print(i.shape, i.dtype)
sz = max(len(high), len(low), len(a))
b = np.zeros(sz)
weight = [0.5, 1, 0.5]
b[:len(high)] += high * weight[0]
b[:len(low)] += low * weight[-1]
b[:len(a)] += a * weight[1]
b /= np.sum(weight)
mp.write("a.wav", b, rate=rate)
# mp.play("a.wav")
meta, c = mp.read("a.wav")
# 变调协奏曲再变调，将会失败，因为以上sfft只能检测出强度最大的音调并变调
d = sfft(c, rate=rate, window=300, freq_bei=1, duration_bei=1)
mp.write('b.wav', d, rate)
mp.play('b.wav')


import os
import time
import gym
import pybulletgym
from pathlib2 import Path
from os.path import isfile
import json

def read_json(json_dir):
  """
  Read JSON file from directory.

  :param json_dir: String
    File path.
  :return: Dict
    Data from JSON file as python dict.
  """
  if isfile(json_dir):
    with open(json_dir) as jfile:
        data = json.load(jfile)
        return data
  else:
    print("Error: This is not a JSON file!")

def run_agent(env_name, agent, steps):
  """
  Run an agent on an environment for _ steps.

  :param env_name: String
    Name of the environment.
  :param agent: BaseAgent
    A trained agent.
  :param steps: Int
    Number of steps to run the agent.
  """
  env = gym.make(env_name)
  env.render()
  obs = env.reset()
  action = agent.run(state=obs)
  obs, rewards, done, _ = env.step(action=action)
  for i in range(steps):
    action = agent.run(state=obs)
    obs, rewards, done, _ = env.step(action)
    time.sleep(0.01)

if __name__ == '__main__':
    p = Path(os.getcwd()).parent / "experiments/configs/exp01.json"
    data = read_json(p)
    print(data)


def notas_dp(notas):
    dp = []
    for aluno in notas:
        media = (aluno['PF']+aluno['PI'])/2
        if(media < 5): dp.append(aluno['matricula'])

    return dp

notas = [
    {'matricula' : 123, 'PI' : 7, 'PF': 3},

    {'matricula' : 456, 'PI': 4, 'PF': 8},

    {'matricula' : 789, 'PI': 5, 'PF': 1}
]

print(notas_dp(notas))


import time, os, pathlib, re
filePath = os.path.join(pathlib.Path(__file__).parent.absolute(), 'input.txt')

begin = time.time()

def definePattern(length):
	pattern = r''

	for i in range(0, length):
		pattern += r'(\w)'

		if i != 0:
			pattern += r'(?<!'
			for j in range(1,i+1):
				pattern += r'\%i'%j
				if j < i:
					pattern += r'|'

			pattern += r')'

	return pattern

with open(filePath, 'r') as file:
	for line in file:
		x = re.search(definePattern(4), line)
		y = re.search(definePattern(14), line)

		print('Result 1 : %i - Result 2 : %i' % (x.span()[1], y.span()[1]))

print('Execution time: ' + '{:.3f}'.format((time.time() - begin) * 1000) + ' ms')


import glob
import cv2
import matplotlib.pyplot as plt
from keras.models import Sequential, load_model
import numpy as np

conv = 'abcdefghijklmnopqrstuvwxyz'

model = load_model('trained_model.h5')
file_list = glob.glob(f'training_data/capture/original_images/*.png')
for file in file_list:
	print(file)
	img = cv2.imread(file)
	frame = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
	downscale = cv2.resize(frame, (28, 28), interpolation=cv2.INTER_LINEAR)
	x_data = [downscale]
	x_data = np.asarray(x_data)
	x_data = x_data.reshape((x_data.shape[0], 28, 28, 1)).astype('float32')
	print([conv[np.argmax(prediction)] for prediction in model.predict(x_data)])
	cv2.imshow('image', downscale)
	cv2.waitKey(0)
	#
	#plt.subplot(221)
	#plt.imshow(img, cmap=plt.get_cmap('gray'))


import re

def between(x, mn, mx):
    if isinstance(x, str):
        try:
            x = int(x)
        except:
            x = float(x)
    return x >= mn and mx >= x

int_re = re.compile("\d+")
word_re = re.compile("\w+")
def force_int(x):
    if not x:
        return None
    elif isinstance(x, int) or isinstance(x, float):
        x = str(x)
    m = int_re.match(x).group(0)
    if m:
        return int(m)
    else:
        return None

def sigil(x):
    return word_re.match(x).groups()