import os
import scipy.misc
import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf

from model import DCGAN
from utils import pp, visualize, to_json, show_all_variables, inverse_transform
from time import gmtime, strftime

flags = tf.app.flags
flags.DEFINE_integer("epoch", 50, "Epoch to train [25]")
flags.DEFINE_float("learning_rate", 0.0002, "Learning rate of for adam [0.0002]")
flags.DEFINE_float("beta1", 0.5, "Momentum term of adam [0.5]")
flags.DEFINE_integer("train_size", 1000000000, "The size of train images [np.inf]")
flags.DEFINE_integer("batch_size", 64, "The size of batch images [64]")
flags.DEFINE_integer("input_height", 108, "The size of image to use (will be center cropped). [108]")
flags.DEFINE_integer("input_width", None, "The size of image to use (will be center cropped). If None, same value as input_height [None]")
flags.DEFINE_integer("output_height", 64, "The size of the output images to produce [64]")
flags.DEFINE_integer("output_width", None, "The size of the output images to produce. If None, same value as output_height [None]")
flags.DEFINE_string("data_dir", "./../data", "The name of dataset [celebA, mnist, lsun]")
flags.DEFINE_string("dataset", "celebA", "The name of dataset [celebA, mnist, lsun]")
flags.DEFINE_string("data_list", None, "The file list (if used, input_fname_pattern is ignored). Pathes should be relative to data_dir")
flags.DEFINE_string("input_fname_pattern", "*.png", "Glob pattern of filename of input images [*]")
flags.DEFINE_string("checkpoint_dir", "checkpoint", "Directory name to save the checkpoints [checkpoint]")
flags.DEFINE_string("sample_dir", "samples", "Directory name to save the image samples [samples]")
flags.DEFINE_boolean("train", False, "True for training, False for testing [False]")
flags.DEFINE_boolean("crop", False, "True for training, False for testing [False]")
flags.DEFINE_boolean("test", False, "True for visualizing, False for nothing [False]")
flags.DEFINE_boolean("generate", False, "True for generating set of images, False for training [False]")
flags.DEFINE_integer("generate_num", 10, "Number of images to generate [10]")
flags.DEFINE_float("generate_min_D_prob", None, "Min. discriminator output score for generated images [None]")
flags.DEFINE_string("generate_name_postfix", "" , "Postfix of generated images")
flags.DEFINE_string("generate_dir", "./../generated" , "Path to generated images")
flags.DEFINE_integer("test_mode", 7, "Visualizing mode: 0 - ; 1 - ; 2 - ;3 - ;4 - ;5 - ;6 - ;7 -")
flags.DEFINE_integer("z_dim", 100, "Latent space size")
flags.DEFINE_integer("y_dim", None, " space size")
flags.DEFINE_boolean("use_double_G_train", True, "use_double_G_train")
flags.DEFINE_boolean("add_dense", False, "Adding dense layer on top of generator [False]")
flags.DEFINE_boolean("generate_known_z", False, "Generate using predefined z values")
flags.DEFINE_string("known_z_path", "./../data", "File with mean Z values to use for generation")

FLAGS = flags.FLAGS

def main(_):
  pp(flags.FLAGS.__flags)

  if FLAGS.input_width is None:
    FLAGS.input_width = FLAGS.input_height
  if FLAGS.output_width is None:
    FLAGS.output_width = FLAGS.output_height

  if not os.path.exists(FLAGS.checkpoint_dir):
    os.makedirs(FLAGS.checkpoint_dir)
  if not os.path.exists(FLAGS.sample_dir):
    os.makedirs(FLAGS.sample_dir)

  #gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.333)
  run_config = tf.ConfigProto()
  run_config.gpu_options.allow_growth=True

  with tf.Session(config=run_config) as sess:
    dcgan = DCGAN(
          sess,
          input_width=FLAGS.input_width,
          input_height=FLAGS.input_height,
          output_width=FLAGS.output_width,
          output_height=FLAGS.output_height,
          batch_size=FLAGS.batch_size,
          sample_num=FLAGS.batch_size,
          y_dim=FLAGS.y_dim,
          z_dim=FLAGS.z_dim,
          dataset_name=FLAGS.dataset,
          data_dir=FLAGS.data_dir,
          data_list=FLAGS.data_list,
          input_fname_pattern=FLAGS.input_fname_pattern,
          crop=FLAGS.crop,
          checkpoint_dir=FLAGS.checkpoint_dir,
          sample_dir=FLAGS.sample_dir,
          use_double_G_train=FLAGS.use_double_G_train,
          add_deconv=FLAGS.add_dense)

    show_all_variables()

    if FLAGS.train:
        train_dcgan(dcgan)

    if not FLAGS.train:
        if not dcgan.load(FLAGS.checkpoint_dir)[0]:
            raise Exception("[!] Train a model first, then run test mode")

    if FLAGS.test:
        test_dcgan(dcgan, sess)

    if FLAGS.generate:
        generated_file_pathes = generate_images(dcgan, sess)

    if FLAGS.generate_known_z:
        z_mean = np.loadtxt(FLAGS.known_z_path)
        generated_file_pathes = generate_images(dcgan, sess, z_mean)

def train_dcgan(dcgan):
    dcgan.train(FLAGS)

def test_dcgan(dcgan, sess):
    visualize(sess, dcgan, FLAGS, FLAGS.visualize_mode)

def generate_images(dcgan, sess, predefined_mean_z=None, z_sigma=0.05):
    print(FLAGS.generate_name_postfix)
    strLog = ""
    generation_data = []
    dir = FLAGS.generate_dir
    if not os.path.exists(dir):
        os.makedirs(dir)

    imgInd = 0
    while (imgInd < FLAGS.generate_num):
        if predefined_mean_z is None:
            z_sample = np.random.uniform(-1, 1, size=(FLAGS.batch_size, FLAGS.z_dim))
        else:
            z_sample = [[np.random.normal(mu, z_sigma) for mu in z_sigma] for _ in range(FLAGS.batch_size)]
            z_sample = np.array(z_sample)

        samples = sess.run(dcgan.sampler, feed_dict={dcgan.z: z_sample})

        ps = sess.run(dcgan.D, feed_dict={dcgan.inputs: samples})
        probs = ps[:, 0]

        images = inverse_transform(samples)
        for im, z, d_prob in zip(images, z_sample, probs):
            if imgInd >= FLAGS.generate_num:
                break

            if FLAGS.generate_min_D_prob is None or d_prob >= FLAGS.generate_min_D_prob:
                filename = 'gen_%d_dPr_%.2f.png' % (imgInd, d_prob)
                scipy.misc.imsave(os.path.join(dir, filename), np.squeeze(im))
                strLog += filename + ',' + FLAGS.generate_name_postfix + ',' + strftime("%Y%m%d%H%M%S", gmtime()) + '\n'
                generation_data.append((filename, str(d_prob),','.join([str(_) for _ in z])))
                imgInd += 1

            if imgInd % 50 == 0:
                print(imgInd)

    f = open(dir + "/generated_list.txt", "a")
    f.write(strLog)
    f.close()
    with open(dir + "/generated_info.csv", 'w') as ff:
        ff.write('file,D_prob,' + ','.join(['z' + str(_) for _ in range(0, FLAGS.z_dim)]) + '\n')
        for g_d in generation_data:
            ff.write(','.join(g_d) + '\n')
    # plt.hist(probs)
    # plt.show()
    return [os.path.join(dir, gd[0]) for gd in generation_data]

if __name__ == '__main__':
  tf.app.run()


from typing import List

class Solution:
    data = [1, 2, 3, 4, 5, 6, 7, 8, 9]

    def combinationSum3(self, k: int, n: int) -> List[List[int]]:
        res = []
        self.back_trace(0, 9, res, [], k, n)
        return res

    def back_trace(self, start, total, res, path, k, n):
        # 递归跳出条件
        if len(path) == k:
            res.append(path)
            return

        for i in range(start, total):
            # 跳过
            if len(path) == k - 1:
                if sum(path) + self.data[i] != n:
                    continue
            self.back_trace(i + 1, total, res, path + [self.data[i]], k, n)

print(Solution().combinationSum3(3, 7))
print(Solution().combinationSum3(3, 9))


from nets.resnet import ResNet50
from tensorflow.keras.utils import plot_model
from utils import load_image
from tensorflow.keras.callbacks import TensorBoard, ModelCheckpoint, ReduceLROnPlateau, EarlyStopping
from tensorflow.keras import optimizers

import tensorflow as tf

if __name__ == "__main__":
    gpus = tf.config.list_physical_devices(device_type="GPU")
    for gpu in gpus:
        tf.config.experimental.set_memory_growth(device=gpu, enable=True)

    train_dir = "datasets/train"
    val_dir = "datasets/val"
    batch_size = 2048
    image_size = 224
    model = ResNet50(input_shape=(image_size, image_size, 3), classes=2)
    # model.load_weights("ResNet50.h5")
    model.summary()
    opt = optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
    model.compile(optimizer=opt,
                  loss=tf.keras.losses.sparse_categorical_crossentropy,
                  metrics=[tf.keras.metrics.sparse_categorical_accuracy])

    plot_model(model, to_file='models/ResNet50.png',show_shapes=True, show_layer_names=True)

    logging = TensorBoard(log_dir="logs")
    checkpoint = ModelCheckpoint("models" + "/ResNet_{epoch:04d}-{val_loss:.4f}.h5", monitor="val_loss", mode="min",
                                 verbose=1, save_weights_only=True, save_best_only=True, period=3)

    early_stopping = EarlyStopping(monitor='val_loss', min_delta=0, patience=15, verbose=1)
    reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=10, verbose=1)

    train_dataset = load_image(train_dir, batch_size, image_size)
    val_dataset = load_image(val_dir, batch_size, image_size)
    model.fit(
        train_dataset,
        epochs=2,
        callbacks=[reduce_lr, logging, checkpoint, early_stopping],
        validation_data=val_dataset,
        initial_epoch=0,
        max_queue_size=20,
        workers=4)

    # 保存keras特有的h5文件模型，这个可以直接加载
    model.save("models" + "/ResNet-model.h5")
    # 保存keras的权重文件，这个需要先搭建模型然后加载权重文件
    model.save_weights("models" + "/ResNet-weights.h5")
    # 保存pb文件，用于tf直接加载模型
    tf.saved_model.save(model, "models/pb")


import os
from casacore.tables import table
import numpy as np
import pylab
from numpy import ma
import sys
import warnings
import time
import math
import argparse

class CovWeights:
    def __init__(self,MSName,ntsol=1,SaveDataProducts=0,uvcut=[0,2000],gainfile=None,phaseonly=False,nfreqsol=1,norm=False):
        if MSName[-1]=="/":
            self.MSName=MSName[0:-1]
        else:
            self.MSName=MSName
        self.MaxCorrTime=0
        self.SaveDataProducts=SaveDataProducts
        self.ntSol=ntsol
        self.nfreqsol=nfreqsol
        self.uvcut=uvcut
        self.gainfile=gainfile
        self.phaseonly=phaseonly
        self.normalise=norm

    def FindWeights(self,tcorr=0,colname=""):
        ms=table(self.MSName,readonly=False)
        # open antennas
        ants=table(ms.getkeyword("ANTENNA"))
        # open antenna tables
        antnames=ants.getcol("NAME")
        nAnt=len(antnames)
        # open uvw to perform uvcut - TEST
        u,v,_=ms.getcol("UVW").T
        # load ant indices
        A0=ms.getcol("ANTENNA1")
        A1=ms.getcol("ANTENNA2")
        tarray=ms.getcol("TIME")
        nbl=np.where(tarray==tarray[0])[0].size
        warnings.filterwarnings("ignore")
        warnings.filterwarnings("default")
        if "RESIDUAL_DATA" not in ms.colnames():
            print("RESIDUAL_DATA column not in measurement set: creating it and filling with RESIDUAL_DATA")
            desc=ms.getcoldesc("CORRECTED_DATA")
            desc["name"]="RESIDUAL_DATA"
            desc['comment']=desc['comment'].replace(" ","_")
            ms.addcols(desc)
            ms.putcol("RESIDUAL_DATA",ms.getcol("CORRECTED_DATA"))
        flags=ms.getcol("FLAG")
        print("Please ensure that RESIDUAL_DATA or CORRECTED_DATA contains residual visibilities from complete skymodel subtraction.")
        residualdata=ms.getcol("RESIDUAL_DATA")
        flags=ms.getcol("FLAG")
        # apply uvcut
        uvlen=np.sqrt(u**2+v**2)
        flags[uvlen>self.uvcut[1]]=1
        flags[uvlen<self.uvcut[0]]=1
        # apply flags to data
        residualdata[flags==1]=0
        # exit files gracefully
        ants.close()
        # initialise
        nChan=residualdata.shape[1]
        nPola=residualdata.shape[2]
        nt=int(residualdata.shape[0]/nbl)
        # reshape antennas and data columns
        residualdata=residualdata.reshape((nt,nbl,nChan,nPola))
        # average residual data within calibration cells
        ### TODO: the averaging should be done later; try using a filter instead of this ###
#        times=np.array(list(sorted(set(tarray))))
        A0=A0.reshape((nt,nbl))
        A1=A1.reshape((nt,nbl))
#        ant1=np.arange(nAnt)
#        CoeffArray=np.zeros((nt,nAnt))
#        print("Begin calculating antenna-based coefficients")
#        for i,t_i in enumerate(times):
#            indexmax=min(len(times)-1,i+self.ntSol)
#            indexmin=max(0,i-self.ntSol)
#            tmin=times[indexmin]
#            tmax=times[indexmax]
#            tmask=(tarray>tmin)*(tarray<tmax)
#            for ant in ant1:
#                # set of vis for baselines ant-ant_i
#                set1=(A0==ant)
#                # set of vis for baselines ant_i-ant
#                set2=(A1==ant)
#                resmask=tmask*(set1+set2)
#                rarray=residualdata[resmask]
#                CoeffArray[i,ant] = np.mean(np.abs(rarray*rarray.conj()))
#            PrintProgress(i,nt)

### TODO figure out if below is actually useful. Bugged at present
#        if self.ntSol>1:
#            tspill=nt%self.ntSol
#            nt1=nt+self.ntSol-tspill
#            for i in range(int(nt1/self.ntSol)):
#                for j in range(self.nfreqsol):
#                    print(residualdata[i*self.ntSol:(i+1)*self.ntSol,:,self.nfreqsol*j:(j+1)*nfreqsol,:].shape)
#                    print(np.mean(residualdata[i*self.ntSol:(i+1)*self.ntSol,:,:,:],axis=0).shape)
#                    residualdata[i*self.ntSol:(i+1)*self.ntSol,:,self.nfreqsol*j:(j+1)*nfreqsol,:]=np.mean(residualdata[i*self.ntSol:(i+1)*self.ntSol,:,:,:],axis=0)
        A0=A0.reshape((nt,nbl))
        A1=A1.reshape((nt,nbl))
        ant1=np.arange(nAnt)
        # make rms array
        darray=ms.getcol("RESIDUAL_DATA").reshape((nt,nbl,nChan,nPola))
        ms.close()
        rmsarray=np.zeros((nt,nbl,nChan,2),dtype=np.complex64)
        residuals=np.zeros_like(rmsarray,dtype=np.complex64)
        rmsarray[:,:,:,0]=darray[:,:,:,1]
        rmsarray[:,:,:,1]=darray[:,:,:,2]
        # make proper residual array
        residuals[:,:,:,0]=darray[:,:,:,0]
        residuals[:,:,:,1]=darray[:,:,:,3]
        # antenna coefficient array
        CoeffArray=np.zeros((nt,nAnt,2))
        # start calculating the weights
        print("Begin calculating antenna-based coefficients")
        warnings.filterwarnings("ignore")
        print("Find variance-only weights")
        for t_i in range(nt):
            # build weights for each antenna at time t_i
            for ant in ant1:
                # set of vis for baselines ant-ant_i
                set1=np.where(A0[t_i]==ant1)[0]
                # set of vis for baselines ant_i-ant
                set2=np.where(A1[t_i]==ant)[0]
                CoeffArray[t_i,ant,0] = (np.mean(np.append(residuals[t_i,set1,:,:],residuals[t_i,set2,:,:])*np.append(residuals[t_i,set1,:,:],residuals[t_i,set2,:,:]).conj())\
                                              - np.std( (np.append(rmsarray[t_i,set1,:,:], rmsarray[t_i,set2,:,:]))) )
                CoeffArray[t_i,ant,1] = np.mean(np.append(residuals[t_i,set1,:,:],residuals[t_i,set2,:,:]))*(np.mean(np.append(residuals[t_i,set1,:,:],residuals[t_i,set2,:,:]))).conj()
            PrintProgress(t_i,nt)
        warnings.filterwarnings("default")
        for i in range(nAnt):
            # get rid of NaN
            CoeffArray[np.isnan(CoeffArray)]=np.inf
            tempars=CoeffArray[:,i,0]
            gtempars=CoeffArray[:,i,1]
            thres=0.25*np.median(tempars)
            gthres=0.25*np.median(gtempars)
            CoeffArray[:,i,0][tempars<thres]=thres
            CoeffArray[:,i,0][gtempars<gthres]=gthres
            # normalise per antenna
            if self.normalise==True:
                CoeffArray[:,i,0]=CoeffArray[:,i,0]/CoeffArray[:,i,1]**2
#            CoeffArray[:,i,1]=CoeffArray[:,i,1]/np.mean(CoeffArray[:,i,1])
        if colname=="":
            coeffFilename=self.MSName+"/CoeffArray.ntsol%i.npy"%(ntsol)
        else:
            coeffFilename=self.MSName+"/CoeffArray.%s.ntsol%i.npy"%(colname,ntsol)
        print("Save coefficient array as %s."%coeffFilename)
        np.save(coeffFilename,CoeffArray)
        return CoeffArray

    def SaveWeights(self,CoeffArray,colname=None,AverageOverChannels=True,tcorr=0):
        print("Begin saving the data")
        ms=table(self.MSName,readonly=False)
        # open antennas
        ants=table(ms.getkeyword("ANTENNA"))
        # open antenna tables
        antnames=ants.getcol("NAME")
        nAnt=len(antnames)
        tarray=ms.getcol("TIME")
        darray=ms.getcol("DATA")
        tvalues=np.array(sorted(list(set(tarray))))
        nt=tvalues.shape[0]
        nbl=int(tarray.shape[0]/nt)
        nchan=darray.shape[1]
        A0=np.array(ms.getcol("ANTENNA1").reshape((nt,nbl)))
        A1=np.array(ms.getcol("ANTENNA2").reshape((nt,nbl)))
        if colname in ms.colnames():
            print("%s column already present; will overwrite"%colname)
        else:
            W=np.ones((nt*nbl,nchan))
            desc=ms.getcoldesc("IMAGING_WEIGHT")
            desc["name"]=colname
            desc['comment']=desc['comment'].replace(" ","_")
            ms.addcols(desc)
            ms.putcol(colname,W)
        # create weight array
        w=np.zeros((nt,nbl,nchan))
        ant1=np.arange(nAnt)
        print("Fill weights array")
        A0ind=A0[0,:]
        A1ind=A1[0,:]
        warnings.filterwarnings("ignore")

        # do gains stuff
        ant1gainarray,ant2gainarray=readGainFile(self.gainfile, ms, nt, nchan, nbl,tarray,nAnt,self.MSName,self.phaseonly)
        for i in range(nbl):
            for j in range(nchan):
                #w[:,i,j]=1./(CoeffArray[:,A0ind[i]]*ant2gainarray[i,j]+CoeffArray[:,A1ind[i]]*ant1gainarray[i,j]+CoeffArray[:,A0ind[i]]*CoeffArray[:,A1ind[i]] + 0.1)
                if self.normalise==True:
                    ga=CoeffArray[:,A0ind[i],1]**2
                    gb=CoeffArray[:,A1ind[i],1]**2
                    w[:,i,j]=1./( ga*gb*(CoeffArray[:,A0ind[i],0]+CoeffArray[:,A1ind[i],0]+CoeffArray[:,A0ind[i],0]*CoeffArray[:,A1ind[i],0]) + 0.1)
                else:
                    w[:,i,j]=1./(CoeffArray[:,A0ind[i],0]*CoeffArray[:,A1ind[i],1]+CoeffArray[:,A1ind[i],0]*CoeffArray[:,A0ind[i],1]+CoeffArray[:,A0ind[i],0]*CoeffArray[:,A1ind[i],0] + 0.1)
            PrintProgress(i,nbl)
        warnings.filterwarnings("default")
        w=w.reshape(nt*nbl,nchan)
        w[np.isnan(w)]=0
        w[np.isinf(w)]=0
        # normalise
        w=w/np.mean(w)
        # check shape of column we are writing to
        if "WEIGHT_SPECTRUM" in ms.colnames():
            if ms.getcol(colname).shape==ms.getcol("WEIGHT_SPECTRUM").shape:
                w1=np.zeros_like(ms.getcol("WEIGHT_SPECTRUM"))
                for i in range(4):
                    w1[:,:,i]=w
        w=w1

        if ms.getcol(colname).shape[-1]==4:

            # We are writing to a weight col of shape (nbl*nt, nchan, npol) i.e. WEIGHT_SPECTRUM style
            # and not of shape (nbl*nt, nchan) i.e. IMAGING_WEIGHT style
            print()
        # save in weights column
        if colname!=None:
            ms.putcol(colname,w)
        else: print("No colname given, so weights not saved in MS.")
        ants.close()
        ms.close()

def readGainFile(gainfile,ms,nt,nchan,nbl,tarray,nAnt,msname,phaseonly):
    if phaseonly==True or gainfile=="":
        print("Assume amplitude gain values of 1 everywhere")
        ant1gainarray1=np.ones((nt*nbl,nchan))
        ant2gainarray1=np.ones((nt*nbl,nchan))
    else:
        if gainfile[-4:]==".npz":
            print("Assume reading a kMS sols file")
            gainsnpz=np.load(gainfile)
            gains=gainsnpz["Sols"]
            ant1gainarray=np.ones((nt*nbl,nchan))
            ant2gainarray=np.ones((nt*nbl,nchan))
            A0arr=ms.getcol("ANTENNA1")
            A1arr=ms.getcol("ANTENNA2")
            print("Build squared gain array")
            for i in range(len(gains)):
                timemask=(tarray>gains[i][0])*(tarray<gains[i][1])
                for j in range(nAnt):
                    mask1=timemask*(A0arr==j)
                    mask2=timemask*(A1arr==j)
                    for k in range(nchan):
                        ant1gainarray[mask1,:]=np.abs(np.nanmean(gains[i][2][0,j,0]))#np.abs(np.nanmean(gains[i][3][0,j]))
                        ant2gainarray[mask2,:]=np.abs(np.nanmean(gains[i][2][0,j,0]))#np.abs(np.nanmean(gains[i][3][0,j]))
                PrintProgress(i,len(gains))
            np.save(msname+"/ant1gainarray",ant1gainarray)
            np.save(msname+"/ant2gainarray",ant2gainarray)
            ant1gainarray=np.load(msname+"/ant1gainarray.npy")
            ant2gainarray=np.load(msname+"/ant2gainarray.npy")
            #        ant1gainarray1=np.ones((nt,nbl,nchan))
            #        ant2gainarray1=np.ones((nt,nbl,nchan))
            #        for i in range(nchan):
            #            ant1gainarray1[:,:,i]=ant1gainarray**2
            #            ant2gainarray1[:,:,i]=ant2gainarray**2
            ant1gainarray1=ant1gainarray**2#1.reshape((nt*nbl,nchan))
            ant2gainarray1=ant2gainarray**2#1.reshape((nt*nbl,nchan))
            if gainfile[-3:]==".h5":
                print("Assume reading losoto h5parm file")
                import losoto
                solsetName="sol000"
                soltabName="amp000"
                try:
                    gfile=losoto.h5parm.openSoltab(gainfile,solsetName=solsetName,soltabName=soltabName)
                except:
                    print("Could not find amplitude gains in h5parm. Assuming gains of 1 everywhere.")
                    ant1gainarray1=np.ones((nt*nbl,nchan))
                    ant2gainarray1=np.ones((nt*nbl,nchan))
                    return ant1gainarray1,ant2gainarray1
                freqs=table(msname+"/SPECTRAL_WINDOW").getcol("CHAN_FREQ")
                gains=gfile.getValues()[0] # axes: pol, dir, ant, freq, times
                gfreqs=gfile.getValues()[1]["freq"]
                times=fgile.getValues()[1]["time"]
                ant1gainarray=np.zeros((nt*nbl,nchan))
                ant2gainarray=np.zeros((nt*nbl,nchan))
                for j in range(nAnt):
                    mask1=timemask*(A0arr==j)
                    mask2=timemask*(A1arr==j)
                    for k in range(nchan):
                        if freqs[k] in gfreqs:
                            freqmask=(gfreqs==k)
                            ant1gainarray1[mask1,k]=np.mean(gains[:,0,j,freqmask],axis=0)**2
                            ant2gainarray1[mask2,k]=np.mean(gains[:,0,j,freqmask],axis=0)**2
            else:
                print("Gain file type not currently supported. Assume all gain amplitudes are 1.")
                ant1gainarray1=np.ones((nt*nbl,nchan))
                ant2gainarray1=np.ones((nt*nbl,nchan))

    return ant1gainarray1,ant2gainarray1

### auxiliary functions ###
def PrintProgress(currentIter,maxIter,msg=""):
    sys.stdout.flush()
    if msg=="":
        msg="Progress:"
    sys.stdout.write("\r%s %5.1f %% "%(msg,100*(currentIter+1.)/maxIter))
    if currentIter==(maxIter-1):
        sys.stdout.write("\n")
def invSVD(A):
    u,s,v=np.linalg.svd(A)
    s[s<1.e-6*s.max()]=1.e-6*s.max()
    ssq=np.abs((1./s))
    # rebuild matrix
    Asq=np.dot(v,np.dot(np.diag(ssq),np.conj(u)))
    v0=v.T*ssq.reshape(1,ssq.size)
    return Asq
def readArguments():
    parser=argparse.ArgumentParser("Calculate visibility imagin weights based on calibration quality")
    parser.add_argument("-v","--verbose",help="Be verbose, say everything program does. Default is False",required=False,action="store_true")
    parser.add_argument("--filename",type=str,help="Name of the measurement set for which weights want to be calculated",required=True,nargs="+")
    parser.add_argument("--ntsol",type=int,help="Solution interval, in timesteps, for your calibration",required=True)
    parser.add_argument("--nfreqsol",type=int,help="Frequency interval, in channels, for your calibration. Default is 8.",required=False,default=8)
    parser.add_argument("--colname",type=str,help="Name of the weights column name you want to save the weights to. Default is CAL_WEIGHT.",required=False,default="CAL_WEIGHT")
    parser.add_argument("--gainfile",type=str,help="Name of the gain file you want to read to rebuild the calibration quality weights."+\
                        " If no file is given, equivalent to rebuilding weights for phase-only calibration.",required=False,default="")
    parser.add_argument("--uvcutkm",type=float,nargs=2,default=[0,2000],required=False,help="uvcut used during calibration, in km.")
    parser.add_argument("--phaseonly",help="Use if calibration was phase-only; this means that gain information doesn't need to be read.",required=False,action="store_true")
    parser.add_argument("--normalise",help="Normalise gains to avoid suppressing long baselines",required=False,action="store_true")
    args=parser.parse_args()
    return vars(args)

### if program is called as main ###
if __name__=="__main__":
    start_time=time.time()
    args        = readArguments()
    mslist      = args["filename"]
    ntsol       = args["ntsol"]
    nfreqsol    = args["nfreqsol"]
    colname     = args["colname"]
    gainfile    = args["gainfile"]
    uvcut       = args["uvcutkm"]
    phaseonly   = args["phaseonly"]
    normalise   = args["normalise"]
    for msname in mslist:
        print("Finding time-covariance weights for: %s"%msname)
        covweights=CovWeights(MSName=msname,ntsol=ntsol,gainfile=gainfile,uvcut=uvcut,phaseonly=phaseonly,norm=normalise)
        coefficients=covweights.FindWeights(tcorr=0,colname=colname)
        covweights.SaveWeights(coefficients,colname=colname,AverageOverChannels=True,tcorr=0)
        print("Total runtime: %f min"%((time.time()-start_time)/60.))


# n: 노드의 개수
# m: 간선의 개수
# start: 시작 노드

import sys

input = sys.stdin.readline
INF = int(1e9)

# 방문하지 않은 노드 중에서, 가장 거리가 짧은 노드의 번호를 반환
def get_smallest_node():
    result = 0
    min_value = INF

    for i in range(1, n + 1):
        if (distance[i] < min_value) and (not visited[i]):
            result = i
            min_value = distance[i]

    return result

# 다익스트라 알고리즘
def dijkstra(start):
    distance[start] = 0
    visited[start] = True

    for v, cost in graph[start]:
        distance[v] = cost

    for i in range(n - 1):
        now = get_smallest_node()
        visited[now] = True

        for v, cost in graph[now]:
            distance[v] = min(distance[v], distance[now] + cost)

n, m = map(int, input().split())
start = int(input())
graph = [[] for i in range(n + 1)]
visited = [False] * (n + 1)
distance = [INF] * (n + 1)

for _ in range(m):
    v1, v2, cost = map(int, input().split())
    graph[v1].append((v2, cost))

dijkstra(start)

for i in range(1, n + 1):
    if distance[i] == INF:
        print("INF")
    else:
        print(distance[i])

[Python] 이것이 코딩테스트다 with 파이썬/chap09/01 간단한 다익스트라 알고리즘.py

01 간단한 다익스트라 알고리즘.py


from revere.sources.base import BaseRevereSource
from sqlalchemy import create_engine

class DatabaseSource(BaseRevereSource):
    name = 'Database'

    def __init__(self, description=None, config={}):
        super(DatabaseSource, self).__init__(description, config)

        required_params = ['connection_string']
        for param in required_params:
            if param not in config or not config[param]:
                raise Exception('Database Source Improperly Configured: missing parameter: %s' % param)
        self.config = config
        self.conn = create_engine(config['connection_string'], pool_recycle=config.get('pool_recycle', 3600))
        self.conn.connect()

    def _convert_to_dict(self, row, keys):
        return dict(zip(keys, row))

    def execute(self, sql, as_dict=False):
        result = self.conn.execute(sql)
        if result.rowcount == 0:
            return None

        if as_dict:
            keys = result.keys()
            return [self._convert_to_dict(row, keys) for row in result]

        return result


from django.http import HttpResponse
from django.shortcuts import render
from post.models import Post

def post_list(request):
    posts = Post.objects.all()
    context = {
        'posts': posts
    }
    return render(request, 'post/post_list.html', context=context)

def post_detail(request, post_pk):
    post = Post.objects.get(id=post_pk)
    context = {
        'post': post,
    }
    return render(request, 'post/post_detail.html', context=context)


import os.path as osp
import time
from argparse import ArgumentParser, Namespace

from mmengine import Config, DictAction, mkdir_or_exist

from uce.edit import EDITORS, load_editor, parse_concepts, save_editor
from uce.utils import setup_logger

def parse_args() -> Namespace:
    parser = ArgumentParser('Edit model weights.')
    parser.add_argument('config', help='Path to the config file.')
    parser.add_argument(
        'edit_concepts',
        help="Concepts to edit. You can enter multiple concepts, separated by ','.  "
        "If an argument has format of 'xxx.txt', then concepts are read from "
        'a txt file, where each line is a concept. ')
    parser.add_argument(
        '--guided-concepts',
        '-g',
        help='Target concepts of the editing. You can enter multiple concepts, '
        "separated by ','. If an argument has format of 'xxx.txt', then concepts "
        'are read from a txt file, where each line is a concept. Note that the '
        'number of guided concepts must be equal to the number of edit concepts, '
        'or be 1, which means that all the editing concepts will be guided '
        'towards this single target concept.')
    parser.add_argument(
        '--preserve-concepts',
        '-p',
        help='Concepts to preserve. You can enter multiple concepts, '
        "separated by ','. If an argument has format of 'xxx.txt', "
        'then concepts are read from a txt file, where '
        'each line is a concept. ')
    parser.add_argument(
        '--np',
        type=int,
        help='Number of preserve concepts. If it is given, then randomly sample '
        'this number of concepts from the preserve concepts.')
    parser.add_argument(
        '--with-extra',
        choices=['artist', 'object'],
        help='Type of the extra prompts. If it is artist, then some extra prompts '
        "such as 'painting by {concept}' will be added to the set of "
        "edit concepts. If it is 'object', then extra prompts are like "
        "'image of {concept}'. Otherwise, no extra concepts will be used.")
    parser.add_argument(
        '--work-dir',
        '-w',
        default='./workdirs/default/',
        help='Directory to save the output files.')
    parser.add_argument(
        '--editor-ckpt',
        '-c',
        help='Checkpoint of an editor. If it is given, then it means continue to '
        'edit the given model weight, otherwise edit the pre-trained model '
        'weight. ')
    parser.add_argument(
        '--meta-info',
        '-m',
        help='Path to the meta info json file of the editor. Only needed if you '
        'wanna load a previously used editor and continue editing the model.') # Continue the training, should not be used in the first time
    parser.add_argument('--gpu-id', type=int, default=0, help='GPU ID.')
    parser.add_argument(
        '--cfg-options',
        nargs='+',
        action=DictAction,
        help='key value pairs to override the config file entries. The pairs need to '
        'be in format of xxx=yyy')

    args = parser.parse_args()
    return args
# Check this link if you want to know more about argparse
# https://docs.python.org/3/library/argparse.html

def main():
    args = parse_args()
    work_dir = args.work_dir
    mkdir_or_exist(work_dir)
    device = f'cuda:{args.gpu_id}'

    timestamp = time.strftime('%Y%m%d_%H%M%S', time.localtime(time.time()))
    log_file = osp.join(work_dir, f'{timestamp}.log')
    logger = setup_logger('uce', filepath=log_file)

    cfg = Config.fromfile(args.config)
    cfg.editor.update({'device': device})
    if args.cfg_options is not None:
        cfg.merge_from_dict(args.cfg_options)
    logger.info(f"Using config:\n{'=' * 60}\n{cfg.pretty_text}\n{'=' * 60}\n")

    editor = EDITORS.build(cfg.editor)
    if args.editor_ckpt is not None:
        editor = load_editor(editor, args.editor_ckpt, args.meta_info).to(device)

    edit_concepts, guided_concepts, preserve_concepts = parse_concepts(
        args.edit_concepts,
        guided_concepts=args.guided_concepts,
        preserve_concepts=args.preserve_concepts,
        num_preserve_concepts=args.np,
        with_extra=args.with_extra,
    )
    editor.edit(edit_concepts, guided_concepts, preserve_concepts)
    save_editor(editor, osp.join(work_dir, f'editor_{editor.id_code}.pt'))

if __name__ == '__main__':
    main()

"""
from mmengine.config import Config

cfg = Config.fromfile('learn_read_config.py')
print(cfg)

can be used for python(dict), json and yaml file
"""


def getPermutation(n,k):
    from functools import reduce
    n_list=[str(i) for i in range(1,n+1)]
    n_list="".join(n_list)#如果是n_list=" ".join(n_list)。这里" "里面是有空格的，则生成的字符串之间都是有空格的
    #permutations=reduce([lambda s,c:p[:i]+c+p[i:] for i in range(len(p)) for p in s],n_list,[])
    #生成全排列注意点
    permutations = reduce(lambda s, c: [p[:i] + c + p[i:] for p in s for i in range(len(p)+1)], n_list, [""])
    """1、n_list要先转换成字符串
       2、lambda函数的函数体用[]扩住，而不是整个lambda用[]扩住
       3、for i in range(len(p)+1),这里是最重要的。因为必须要len(p)+1，如果不+1，则最后只会生成[],因为刚开始跳不出""
       4、然后n_list, [""]，也重要的[""]里面必须有一个初始化的“”，要不然进不去for p in s
       """
    permutations.sort(key=lambda x:x)
    return permutations[k-1]
"""这样虽然正确但是时间是远远超出的"""

def test_2(n,k):
    """这里的思想是，先生成一个n_list，有序，然后用k整除n-1的阶乘，这样就知道了
    k应该掉落在那个字符开头的permutations，然后%n-1的阶乘，表示的在这个permutations中
    的位置，如果j==0，这就意味着到了这个permutations集合的最后一个了，在当前n_list[n-1]之后的
    都已经排好序了，是逆序的
    """
    from functools import reduce
    n_list = [str(i) for i in range(1, n + 1)]
    result=""
    while(n_list):
        num=reduce(lambda a,b:a*b,[x for x in range(1,len(n_list))],1)
        i=k//num
        j=k%num
        if j == 0:
            result = result + n_list[i-1]
            n_list.pop(i-1)
            result=result+"".join(n_list[::-1])
            return result
        result=result+n_list[i]
        n_list.pop(i)
        k=j
import math
def permutation_k(n,k):
    #这种是19年4月10号做的，很明显这是一个递归思想啊
    ans=[str(i) for i in range(1,n+1)]
    def f(m,temp):#这个函数的含义是在temp中的从小到大排列的所有组合中，找到第k个
        if m==0:
            return "".join(temp)
        else:
            number=math.factorial(len(temp)-1)
            return temp[m//number]+f(m%number,temp[0:m//number]+temp[m//number+1:])
    return f(k,ans)
print(permutation_k(8,7))
n=2
k=2
#getPermutation(n,k)
print(test_2(n,k))


import itertools
from datetime import datetime

from flask import (
    jsonify,
    request,
    current_app,
    Blueprint
)
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm.exc import NoResultFound

from app.dao import notifications_dao
from app.dao.dao_utils import dao_rollback
from app.dao.api_key_dao import (
    save_model_api_key,
    get_model_api_keys,
    get_unsigned_secret,
    expire_api_key)
from app.dao.inbound_numbers_dao import dao_allocate_number_for_service
from app.dao.organisation_dao import dao_get_organisation_by_service_id
from app.dao.service_data_retention_dao import (
    fetch_service_data_retention,
    fetch_service_data_retention_by_id,
    fetch_service_data_retention_by_notification_type,
    insert_service_data_retention,
    update_service_data_retention,
)
from app.dao.service_sms_sender_dao import (
    dao_add_sms_sender_for_service,
    dao_update_service_sms_sender,
    archive_sms_sender,
    dao_get_service_sms_senders_by_id,
    dao_get_sms_senders_by_service_id,
    update_existing_sms_sender_with_inbound_number
)
from app.dao.templates_dao import (
    dao_get_all_templates_for_service
)
from app.dao.services_dao import (
    dao_add_user_to_service,
    dao_archive_service,
    dao_create_service,
    dao_fetch_active_users_for_service,
    dao_fetch_all_services,
    dao_fetch_all_services_by_user,
    dao_fetch_trial_services_data,
    dao_fetch_live_services_data,
    dao_fetch_monthly_historical_stats_for_service,
    dao_fetch_monthly_historical_usage_by_template_for_service,
    dao_fetch_service_by_id,
    dao_fetch_stats_for_service,
    dao_fetch_todays_stats_for_service,
    dao_fetch_todays_stats_for_all_services,
    dao_resume_service,
    dao_remove_user_from_service,
    dao_suspend_service,
    dao_update_service,
    fetch_stats_by_date_range_for_all_services
)
from app.dao.service_whitelist_dao import (
    dao_fetch_service_whitelist,
    dao_add_and_commit_whitelisted_contacts,
    dao_remove_service_whitelist
)
from app.dao.service_email_reply_to_dao import (
    add_reply_to_email_address_for_service,
    dao_get_reply_to_by_id,
    dao_get_reply_to_by_service_id,
    update_reply_to_email_address
)
from app.dao.service_letter_contact_dao import (
    dao_get_letter_contacts_by_service_id,
    dao_get_letter_contact_by_id,
    add_letter_contact_for_service,
    update_letter_contact
)
from app.dao.provider_statistics_dao import get_fragment_count
from app.dao.users_dao import get_user_by_id
from app.dao.callback_failures_dao import (
    dao_get_callback_failing_stats_for_all_services,
    dao_get_callback_failures_for_all_services,
    dao_get_failing_callback_summary,
)
from app.errors import (
    InvalidRequest,
    register_errors
)
from app.models import (
    Service,
    EmailBranding,
    SMS_TYPE,
    EMAIL_TYPE,
    LETTER_TYPE
)
from app.schema_validation import validate
from app.service import statistics
from app.service.service_data_retention_schema import (
    add_service_data_retention_request,
    update_service_data_retention_request
)
from app.service.service_senders_schema import (
    add_service_email_reply_to_request,
    add_service_letter_contact_block_request,
    add_service_sms_sender_request
)
from app.service.utils import get_whitelist_objects
from app.service.sender import send_notification_to_service_users, send_notification_to_notify_support
from app.service.send_notification import send_one_off_notification
from app.schemas import (
    service_schema,
    api_key_schema,
    permission_schema,
    notification_with_template_schema,
    notifications_filter_schema,
    detailed_service_schema
)
from app.utils import pagination_links

service_blueprint = Blueprint('service', __name__)

register_errors(service_blueprint)

@service_blueprint.errorhandler(IntegrityError)
def handle_integrity_error(exc):
    """
    Handle integrity errors caused by the unique constraint on ix_organisation_name
    """
    if 'services_name_key' or 'services_email_from_key' in str(exc):
        return jsonify(
            result='error',
            message={'name': ["Duplicate service name '{}'".format(
                exc.params.get('name', exc.params.get('email_from', ''))
            )]}
        ), 400
    current_app.logger.exception(exc)
    return jsonify(result='error', message="Internal server error"), 500

@service_blueprint.route('', methods=['GET'])
def get_services():
    only_active = request.args.get('only_active') == 'True'
    detailed = request.args.get('detailed') == 'True'
    user_id = request.args.get('user_id', None)
    include_from_test_key = request.args.get('include_from_test_key', 'True') != 'False'

    # If start and end date are not set, we are expecting today's stats.
    today = str(datetime.utcnow().date())

    start_date = datetime.strptime(request.args.get('start_date', today), '%Y-%m-%d').date()
    end_date = datetime.strptime(request.args.get('end_date', today), '%Y-%m-%d').date()

    if user_id:
        services = dao_fetch_all_services_by_user(user_id, only_active)
    elif detailed:
        result = jsonify(data=get_detailed_services(start_date=start_date, end_date=end_date,
                                                    only_active=only_active,
                                                    include_from_test_key=include_from_test_key
                                                    ))
        return result
    else:
        services = dao_fetch_all_services(only_active)
    data = service_schema.dump(services, many=True).data
    return jsonify(data=data)

@service_blueprint.route('/trial-services-data', methods=['GET'])
def get_trial_services_data():
    data = dao_fetch_trial_services_data()
    for i, row in enumerate(data):
        data[i]["domains"] = get_service_domains(row["service_id"])

    return jsonify(data=data)

@service_blueprint.route('/live-services-data', methods=['GET'])
def get_live_services_data():
    data = dao_fetch_live_services_data()
    for i, row in enumerate(data):
        data[i]["domains"] = get_service_domains(row["service_id"])
    return jsonify(data=data)

@service_blueprint.route('/<uuid:service_id>', methods=['GET'])
def get_service_by_id(service_id):
    if request.args.get('detailed') == 'True':
        data = get_detailed_service(service_id, today_only=request.args.get('today_only') == 'True')
    else:
        fetched = dao_fetch_service_by_id(service_id)

        data = service_schema.dump(fetched).data
    return jsonify(data=data)

@service_blueprint.route('', methods=['POST'])
def create_service():
    data = request.get_json()

    if not data.get('user_id'):
        errors = {'user_id': ['Missing data for required field.']}
        raise InvalidRequest(errors, status_code=400)

    # validate json with marshmallow
    service_schema.load(data)

    user = get_user_by_id(data.pop('user_id'))

    # unpack valid json into service object
    valid_service = Service.from_json(data)

    dao_create_service(valid_service, user)
    return jsonify(data=service_schema.dump(valid_service).data), 201

@service_blueprint.route('/<uuid:service_id>', methods=['POST'])
def update_service(service_id):
    req_json = request.get_json()
    fetched_service = dao_fetch_service_by_id(service_id)
    # Capture the status change here as Marshmallow changes this later
    service_going_live = fetched_service.restricted and not req_json.get('restricted', True)
    current_data = dict(service_schema.dump(fetched_service).data.items())
    current_data.update(request.get_json())

    service = service_schema.load(current_data).data
    org_type = req_json.get('organisation_type', None)
    if org_type:
        service.crown = org_type == 'central'

    if 'email_branding' in req_json:
        email_branding_id = req_json['email_branding']
        service.email_branding = None if not email_branding_id else EmailBranding.query.get(email_branding_id)

    dao_update_service(service)

    if service_going_live:
        send_notification_to_service_users(
            service_id=service_id,
            template_id=current_app.config['SERVICE_NOW_LIVE_TEMPLATE_ID'],
            personalisation={
                'service_name': current_data['name'],
                'message_limit': '{:,}'.format(current_data['message_limit'])
            },
            include_user_fields=['name']
        )

    return jsonify(data=service_schema.dump(fetched_service).data), 200

@service_blueprint.route('/<uuid:service_id>/go-live', methods=['PUT'])
def request_to_go_live(service_id):
    fetched_service = dao_fetch_service_by_id(service_id=service_id)

    if fetched_service.restricted:
        send_notification_to_notify_support(current_app.config['REQUEST_TO_GO_LIVE_TEMPLATE_ID'], request.get_json())
    return jsonify(), 200

@service_blueprint.route('/<uuid:service_id>/api-key', methods=['POST'])
def create_api_key(service_id=None):
    fetched_service = dao_fetch_service_by_id(service_id=service_id)
    valid_api_key = api_key_schema.load(request.get_json()).data
    valid_api_key.service = fetched_service
    save_model_api_key(valid_api_key)
    unsigned_api_key = get_unsigned_secret(valid_api_key.id)
    return jsonify(data=unsigned_api_key), 201

@service_blueprint.route('/<uuid:service_id>/api-key/revoke/<uuid:api_key_id>', methods=['POST'])
def revoke_api_key(service_id, api_key_id):
    expire_api_key(service_id=service_id, api_key_id=api_key_id)
    return jsonify(), 202

@service_blueprint.route('/<uuid:service_id>/api-keys', methods=['GET'])
@service_blueprint.route('/<uuid:service_id>/api-keys/<uuid:key_id>', methods=['GET'])
def get_api_keys(service_id, key_id=None):
    dao_fetch_service_by_id(service_id=service_id)

    try:
        if key_id:
            api_keys = [get_model_api_keys(service_id=service_id, id=key_id)]
        else:
            api_keys = get_model_api_keys(service_id=service_id)
    except NoResultFound:
        error = "API key not found for id: {}".format(service_id)
        raise InvalidRequest(error, status_code=404)

    return jsonify(apiKeys=api_key_schema.dump(api_keys, many=True).data), 200

@service_blueprint.route('/<uuid:service_id>/users', methods=['GET'])
def get_users_for_service(service_id):
    fetched = dao_fetch_service_by_id(service_id)
    return jsonify(data=[x.serialize() for x in fetched.users])

@service_blueprint.route('/<uuid:service_id>/users/<user_id>', methods=['POST'])
def add_user_to_service(service_id, user_id):
    service = dao_fetch_service_by_id(service_id)
    user = get_user_by_id(user_id=user_id)

    if user in service.users:
        error = 'User id: {} already part of service id: {}'.format(user_id, service_id)
        raise InvalidRequest(error, status_code=400)

    permissions = permission_schema.load(request.get_json(), many=True).data
    dao_add_user_to_service(service, user, permissions)
    data = service_schema.dump(service).data
    return jsonify(data=data), 201

@service_blueprint.route('/<uuid:service_id>/users/<user_id>', methods=['DELETE'])
def remove_user_from_service(service_id, user_id):
    service = dao_fetch_service_by_id(service_id)
    user = get_user_by_id(user_id=user_id)
    if user not in service.users:
        error = 'User not found'
        raise InvalidRequest(error, status_code=404)

    elif len(service.users) == 1:
        error = 'You cannot remove the only user for a service'
        raise InvalidRequest(error, status_code=400)

    dao_remove_user_from_service(service, user)
    return jsonify({}), 204

@service_blueprint.route('/<uuid:service_id>/fragment/aggregate_statistics')
def get_service_provider_aggregate_statistics(service_id):
    year = request.args.get('year')
    if year is not None:
        try:
            year = int(year)
        except ValueError:
            raise InvalidRequest('Year must be a number', status_code=400)
    return jsonify(data=get_fragment_count(service_id, year=year))

# This is placeholder get method until more thought
# goes into how we want to fetch and view various items in history
# tables. This is so product owner can pass stories as done
@service_blueprint.route('/<uuid:service_id>/history', methods=['GET'])
def get_service_history(service_id):
    from app.models import (Service, ApiKey, TemplateHistory)
    from app.schemas import (
        service_history_schema,
        api_key_history_schema,
        template_history_schema
    )

    service_history = Service.get_history_model().query.filter_by(id=service_id).all()
    service_data = service_history_schema.dump(service_history, many=True).data
    api_key_history = ApiKey.get_history_model().query.filter_by(service_id=service_id).all()
    api_keys_data = api_key_history_schema.dump(api_key_history, many=True).data

    template_history = TemplateHistory.query.filter_by(service_id=service_id).all()
    template_data, errors = template_history_schema.dump(template_history, many=True)

    data = {
        'service_history': service_data,
        'api_key_history': api_keys_data,
        'template_history': template_data,
        'events': []}

    return jsonify(data=data)

@service_blueprint.route('/<uuid:service_id>/notifications', methods=['GET'])
def get_all_notifications_for_service(service_id):
    data = notifications_filter_schema.load(request.args).data
    if data.get('to'):
        notification_type = data.get('template_type')[0] if data.get('template_type') else None
        return search_for_notification_by_to_field(service_id=service_id,
                                                   search_term=data['to'],
                                                   statuses=data.get('status'),
                                                   notification_type=notification_type)
    page = data['page'] if 'page' in data else 1
    page_size = data['page_size'] if 'page_size' in data else current_app.config.get('PAGE_SIZE')
    limit_days = data.get('limit_days')
    include_jobs = data.get('include_jobs', True)
    include_from_test_key = data.get('include_from_test_key', False)

    pagination = notifications_dao.get_notifications_for_service(
        service_id,
        filter_dict=data,
        page=page,
        page_size=page_size,
        limit_days=limit_days,
        include_jobs=include_jobs,
        include_from_test_key=include_from_test_key
    )
    kwargs = request.args.to_dict()
    kwargs['service_id'] = service_id
    return jsonify(
        notifications=notification_with_template_schema.dump(pagination.items, many=True).data,
        page_size=page_size,
        total=pagination.total,
        links=pagination_links(
            pagination,
            '.get_all_notifications_for_service',
            **kwargs
        )
    ), 200

@service_blueprint.route('/<uuid:service_id>/notifications/<uuid:notification_id>', methods=['GET'])
def get_notification_for_service(service_id, notification_id):

    notification = notifications_dao.get_notification_with_personalisation(
        service_id,
        notification_id,
        key_type=None,
    )
    return jsonify(
        notification_with_template_schema.dump(notification).data,
    ), 200

def search_for_notification_by_to_field(service_id, search_term, statuses, notification_type):
    results = notifications_dao.dao_get_notifications_by_to_field(
        service_id=service_id,
        search_term=search_term,
        statuses=statuses,
        notification_type=notification_type
    )
    return jsonify(
        notifications=notification_with_template_schema.dump(results, many=True).data
    ), 200

@service_blueprint.route('/<uuid:service_id>/notifications/monthly', methods=['GET'])
def get_monthly_notification_stats(service_id):
    service = dao_fetch_service_by_id(service_id)
    try:
        return jsonify(data=dao_fetch_monthly_historical_stats_for_service(
            service.id,
            int(request.args.get('year', 'NaN'))
        ))
    except ValueError:
        raise InvalidRequest('Year must be a number', status_code=400)

def get_detailed_service(service_id, today_only=False):
    service = dao_fetch_service_by_id(service_id)
    stats_fn = dao_fetch_todays_stats_for_service if today_only else dao_fetch_stats_for_service
    stats = stats_fn(service_id)

    service.statistics = statistics.format_statistics(stats)

    return detailed_service_schema.dump(service).data

def get_detailed_services(start_date, end_date, only_active=False, include_from_test_key=True):
    if start_date == datetime.utcnow().date():
        stats = dao_fetch_todays_stats_for_all_services(include_from_test_key=include_from_test_key,
                                                        only_active=only_active)
    else:

        stats = fetch_stats_by_date_range_for_all_services(start_date=start_date,
                                                           end_date=end_date,
                                                           include_from_test_key=include_from_test_key,
                                                           only_active=only_active)
    results = []
    for service_id, rows in itertools.groupby(stats, lambda x: x.service_id):
        rows = list(rows)
        if rows[0].count is None:
            s = statistics.create_zeroed_stats_dicts()
        else:
            s = statistics.format_statistics(rows)
        sid = str(rows[0].service_id)

        s[EMAIL_TYPE]['templates'] = len(dao_get_all_templates_for_service(sid, EMAIL_TYPE))
        s[SMS_TYPE]['templates'] = len(dao_get_all_templates_for_service(sid, SMS_TYPE))
        s[LETTER_TYPE]['templates'] = len(dao_get_all_templates_for_service(sid, LETTER_TYPE))

        results.append({
            'id': sid,
            'name': rows[0].name,
            'notification_type': rows[0].notification_type,
            'research_mode': rows[0].research_mode,
            'restricted': rows[0].restricted,
            'active': rows[0].active,
            'created_at': rows[0].created_at,
            'statistics': s,
            'domains': get_service_domains(sid),
            'organisation_type': rows[0].organisation_type
        })
    return results

@service_blueprint.route('/<uuid:service_id>/whitelist', methods=['GET'])
def get_whitelist(service_id):
    from app.models import (EMAIL_TYPE, MOBILE_TYPE)
    service = dao_fetch_service_by_id(service_id)

    if not service:
        raise InvalidRequest("Service does not exist", status_code=404)

    whitelist = dao_fetch_service_whitelist(service.id)
    return jsonify(
        email_addresses=[item.recipient for item in whitelist
                         if item.recipient_type == EMAIL_TYPE],
        phone_numbers=[item.recipient for item in whitelist
                       if item.recipient_type == MOBILE_TYPE]
    )

@service_blueprint.route('/<uuid:service_id>/whitelist', methods=['PUT'])
def update_whitelist(service_id):
    # doesn't commit so if there are any errors, we preserve old values in db
    dao_remove_service_whitelist(service_id)
    try:
        whitelist_objs = get_whitelist_objects(service_id, request.get_json())
    except ValueError as e:
        current_app.logger.exception(e)
        dao_rollback()
        msg = '{} is not a valid email address or phone number'.format(str(e))
        raise InvalidRequest(msg, 400)
    else:
        dao_add_and_commit_whitelisted_contacts(whitelist_objs)
        return '', 204

@service_blueprint.route('/<uuid:service_id>/archive', methods=['POST'])
def archive_service(service_id):
    """
    When a service is archived the service is made inactive, templates are archived and api keys are revoked.
    There is no coming back from this operation.
    :param service_id:
    :return:
    """
    service = dao_fetch_service_by_id(service_id)

    if service.active:
        dao_archive_service(service.id)

    return '', 204

@service_blueprint.route('/<uuid:service_id>/suspend', methods=['POST'])
def suspend_service(service_id):
    """
    Suspending a service will mark the service as inactive and revoke API keys.
    :param service_id:
    :return:
    """
    service = dao_fetch_service_by_id(service_id)

    if service.active:
        dao_suspend_service(service.id)

    return '', 204

@service_blueprint.route('/<uuid:service_id>/resume', methods=['POST'])
def resume_service(service_id):
    """
    Resuming a service that has been suspended will mark the service as active.
    The service will need to re-create API keys
    :param service_id:
    :return:
    """
    service = dao_fetch_service_by_id(service_id)

    if not service.active:
        dao_resume_service(service.id)

    return '', 204

@service_blueprint.route('/<uuid:service_id>/notifications/templates_usage/monthly', methods=['GET'])
def get_monthly_template_usage(service_id):
    try:
        data = dao_fetch_monthly_historical_usage_by_template_for_service(
            service_id,
            int(request.args.get('year', 'NaN'))
        )

        stats = list()
        for i in data:
            stats.append(
                {
                    'template_id': str(i.template_id),
                    'name': i.name,
                    'type': i.template_type,
                    'month': i.month,
                    'year': i.year,
                    'count': i.count,
                    'is_precompiled_letter': i.is_precompiled_letter
                }
            )

        return jsonify(stats=stats), 200
    except ValueError:
        raise InvalidRequest('Year must be a number', status_code=400)

@service_blueprint.route('/<uuid:service_id>/send-notification', methods=['POST'])
def create_one_off_notification(service_id):
    resp = send_one_off_notification(service_id, request.get_json())
    return jsonify(resp), 201

@service_blueprint.route('/<uuid:service_id>/email-reply-to', methods=["GET"])
def get_email_reply_to_addresses(service_id):
    result = dao_get_reply_to_by_service_id(service_id)
    return jsonify([i.serialize() for i in result]), 200

@service_blueprint.route('/<uuid:service_id>/email-reply-to/<uuid:reply_to_id>', methods=["GET"])
def get_email_reply_to_address(service_id, reply_to_id):
    result = dao_get_reply_to_by_id(service_id=service_id, reply_to_id=reply_to_id)
    return jsonify(result.serialize()), 200

@service_blueprint.route('/<uuid:service_id>/email-reply-to', methods=['POST'])
def add_service_reply_to_email_address(service_id):
    # validate the service exists, throws ResultNotFound exception.
    dao_fetch_service_by_id(service_id)
    form = validate(request.get_json(), add_service_email_reply_to_request)
    new_reply_to = add_reply_to_email_address_for_service(service_id=service_id,
                                                          email_address=form['email_address'],
                                                          is_default=form.get('is_default', True))
    return jsonify(data=new_reply_to.serialize()), 201

@service_blueprint.route('/<uuid:service_id>/email-reply-to/<uuid:reply_to_email_id>', methods=['POST'])
def update_service_reply_to_email_address(service_id, reply_to_email_id):
    # validate the service exists, throws ResultNotFound exception.
    dao_fetch_service_by_id(service_id)
    form = validate(request.get_json(), add_service_email_reply_to_request)
    new_reply_to = update_reply_to_email_address(service_id=service_id,
                                                 reply_to_id=reply_to_email_id,
                                                 email_address=form['email_address'],
                                                 is_default=form.get('is_default', True))
    return jsonify(data=new_reply_to.serialize()), 200

@service_blueprint.route('/<uuid:service_id>/letter-contact', methods=["GET"])
def get_letter_contacts(service_id):
    result = dao_get_letter_contacts_by_service_id(service_id)
    return jsonify([i.serialize() for i in result]), 200

@service_blueprint.route('/<uuid:service_id>/letter-contact/<uuid:letter_contact_id>', methods=["GET"])
def get_letter_contact_by_id(service_id, letter_contact_id):
    result = dao_get_letter_contact_by_id(service_id=service_id, letter_contact_id=letter_contact_id)
    return jsonify(result.serialize()), 200

@service_blueprint.route('/<uuid:service_id>/letter-contact', methods=['POST'])
def add_service_letter_contact(service_id):
    # validate the service exists, throws ResultNotFound exception.
    dao_fetch_service_by_id(service_id)
    form = validate(request.get_json(), add_service_letter_contact_block_request)
    new_letter_contact = add_letter_contact_for_service(service_id=service_id,
                                                        contact_block=form['contact_block'],
                                                        is_default=form.get('is_default', True))
    return jsonify(data=new_letter_contact.serialize()), 201

@service_blueprint.route('/<uuid:service_id>/letter-contact/<uuid:letter_contact_id>', methods=['POST'])
def update_service_letter_contact(service_id, letter_contact_id):
    # validate the service exists, throws ResultNotFound exception.
    dao_fetch_service_by_id(service_id)
    form = validate(request.get_json(), add_service_letter_contact_block_request)
    new_reply_to = update_letter_contact(service_id=service_id,
                                         letter_contact_id=letter_contact_id,
                                         contact_block=form['contact_block'],
                                         is_default=form.get('is_default', True))
    return jsonify(data=new_reply_to.serialize()), 200

@service_blueprint.route('/<uuid:service_id>/sms-sender', methods=['POST'])
def add_service_sms_sender(service_id):
    dao_fetch_service_by_id(service_id)
    form = validate(request.get_json(), add_service_sms_sender_request)
    inbound_number_id = form.get('inbound_number_id', None)
    sms_sender = form.get('sms_sender')

    if inbound_number_id:
        updated_number = dao_allocate_number_for_service(service_id=service_id, inbound_number_id=inbound_number_id)
        # the sms_sender in the form is not set, use the inbound number
        sms_sender = updated_number.number
        existing_sms_sender = dao_get_sms_senders_by_service_id(service_id)
        # we don't want to create a new sms sender for the service if we are allocating an inbound number.
        if len(existing_sms_sender) == 1:
            update_existing_sms_sender = existing_sms_sender[0]
            new_sms_sender = update_existing_sms_sender_with_inbound_number(
                service_sms_sender=update_existing_sms_sender,
                sms_sender=sms_sender,
                inbound_number_id=inbound_number_id)

            return jsonify(new_sms_sender.serialize()), 201

    new_sms_sender = dao_add_sms_sender_for_service(service_id=service_id,
                                                    sms_sender=sms_sender,
                                                    is_default=form['is_default'],
                                                    inbound_number_id=inbound_number_id
                                                    )
    return jsonify(new_sms_sender.serialize()), 201

@service_blueprint.route('/<uuid:service_id>/sms-sender/<uuid:sms_sender_id>', methods=['POST'])
def update_service_sms_sender(service_id, sms_sender_id):
    form = validate(request.get_json(), add_service_sms_sender_request)

    sms_sender_to_update = dao_get_service_sms_senders_by_id(service_id=service_id,
                                                             service_sms_sender_id=sms_sender_id)
    if sms_sender_to_update.inbound_number_id and form['sms_sender'] != sms_sender_to_update.sms_sender:
        raise InvalidRequest("You can not change the inbound number for service {}".format(service_id),
                             status_code=400)

    new_sms_sender = dao_update_service_sms_sender(service_id=service_id,
                                                   service_sms_sender_id=sms_sender_id,
                                                   is_default=form['is_default'],
                                                   sms_sender=form['sms_sender']
                                                   )
    return jsonify(new_sms_sender.serialize()), 200

@service_blueprint.route('/<uuid:service_id>/sms-sender/<uuid:sms_sender_id>/archive', methods=['POST'])
def delete_service_sms_sender(service_id, sms_sender_id):
    sms_sender = archive_sms_sender(service_id, sms_sender_id)

    return jsonify(data=sms_sender.serialize()), 200

@service_blueprint.route('/<uuid:service_id>/sms-sender/<uuid:sms_sender_id>', methods=['GET'])
def get_service_sms_sender_by_id(service_id, sms_sender_id):
    sms_sender = dao_get_service_sms_senders_by_id(service_id=service_id,
                                                   service_sms_sender_id=sms_sender_id)
    return jsonify(sms_sender.serialize()), 200

@service_blueprint.route('/<uuid:service_id>/sms-sender', methods=['GET'])
def get_service_sms_senders_for_service(service_id):
    sms_senders = dao_get_sms_senders_by_service_id(service_id=service_id)
    return jsonify([sms_sender.serialize() for sms_sender in sms_senders]), 200

@service_blueprint.route('/<uuid:service_id>/organisation', methods=['GET'])
def get_organisation_for_service(service_id):
    organisation = dao_get_organisation_by_service_id(service_id=service_id)
    return jsonify(organisation.serialize() if organisation else {}), 200

@service_blueprint.route('/failing-callback-stats', methods=['GET'])
def get_callback_failing_stats_for_all_services():
    stats = dao_get_callback_failing_stats_for_all_services()
    data = {
        'total_failure_count': stats.total_failure_count,
        'failed_notification_count': stats.failed_notification_count,
    }
    return jsonify(data), 200

@service_blueprint.route('/failing-callbacks', methods=['GET'])
def get_callbacks_failing_for_all_services():
    page = int(request.args.get('page', 1))
    paginated_callbacks = dao_get_callback_failures_for_all_services(page=page)
    return jsonify(
        callbacks=[callback.serialize() for callback in paginated_callbacks.items],
        links=pagination_links(
            paginated_callbacks,
            '.get_callbacks_failing_for_all_services',
            **request.args.to_dict()
        )
    ), 200

@service_blueprint.route('/failing-callback-summary', methods=['GET'])
def get_failing_callback_summary():
    data = dao_get_failing_callback_summary()
    summary = list()

    for i in data:
        summary.append(
            {
                'failure_count': i.failure_count,
                'service_id': i.service_id,
                'service_name': i.service_name,
            }
        )

    return jsonify(summary), 200

@service_blueprint.route('/unique', methods=["GET"])
def is_service_name_unique():
    service_id, name, email_from = check_request_args(request)

    name_exists = Service.query.filter_by(name=name).first()

    email_from_exists = Service.query.filter(
        Service.email_from == email_from,
        Service.id != service_id
    ).first()

    result = not (name_exists or email_from_exists)
    return jsonify(result=result), 200

@service_blueprint.route('/<uuid:service_id>/data-retention', methods=['GET'])
def get_data_retention_for_service(service_id):
    data_retention_list = fetch_service_data_retention(service_id)
    return jsonify([data_retention.serialize() for data_retention in data_retention_list]), 200

@service_blueprint.route('/<uuid:service_id>/data-retention/notification-type/<notification_type>', methods=['GET'])
def get_data_retention_for_service_notification_type(service_id, notification_type):
    data_retention = fetch_service_data_retention_by_notification_type(service_id, notification_type)
    return jsonify(data_retention.serialize() if data_retention else {}), 200

@service_blueprint.route('/<uuid:service_id>/data-retention/<uuid:data_retention_id>', methods=['GET'])
def get_data_retention_for_service_by_id(service_id, data_retention_id):
    data_retention = fetch_service_data_retention_by_id(service_id, data_retention_id)
    return jsonify(data_retention.serialize() if data_retention else {}), 200

@service_blueprint.route('/<uuid:service_id>/data-retention', methods=['POST'])
def create_service_data_retention(service_id):
    form = validate(request.get_json(), add_service_data_retention_request)
    try:
        new_data_retention = insert_service_data_retention(
            service_id=service_id,
            notification_type=form.get("notification_type"),
            days_of_retention=form.get("days_of_retention")
        )
    except IntegrityError:
        raise InvalidRequest(
            message="Service already has data retention for {} notification type".format(form.get("notification_type")),
            status_code=400
        )

    return jsonify(result=new_data_retention.serialize()), 201

@service_blueprint.route('/<uuid:service_id>/data-retention/<uuid:data_retention_id>', methods=['POST'])
def modify_service_data_retention(service_id, data_retention_id):
    form = validate(request.get_json(), update_service_data_retention_request)

    update_count = update_service_data_retention(
        service_data_retention_id=data_retention_id,
        service_id=service_id,
        days_of_retention=form.get("days_of_retention")
    )
    if update_count == 0:
        raise InvalidRequest(
            message="The service data retention for id: {} was not found for service: {}".format(data_retention_id,
                                                                                service_id),
            status_code=404)

    return '', 204

def check_request_args(request):
    service_id = request.args.get('service_id')
    name = request.args.get('name', None)
    email_from = request.args.get('email_from', None)
    errors = []
    if not service_id:
        errors.append({'service_id': ["Can't be empty"]})
    if not name:
        errors.append({'name': ["Can't be empty"]})
    if not email_from:
        errors.append({'email_from': ["Can't be empty"]})
    if errors:
        raise InvalidRequest(errors, status_code=400)
    return service_id, name, email_from

def get_service_domains(service_id):
    domains = set()
    for user in dao_fetch_active_users_for_service(service_id):
        parts = user.email_address.split('@')
        if len(parts) != 2:
            continue
        domains.add(parts[1].lower())
    return sorted(list(domains))


# coding: utf-8
"""
Parser/type converter for data in Interactive Brokers' Flex XML format.

https://www.interactivebrokers.com/en/software/reportguide/reportguide.htm#reportguide/activity_flex_query_reference.htm
"""
# stdlib imports
import xml.etree.ElementTree as ET

# local imports
from ibflex import schemata
from ibflex.schemata import elementSchemata

class FlexParserError(Exception):
    """ Base class for errors in this module """
    pass

def parse(source):
    tree = ET.ElementTree()
    tree.parse(source)
    root = tree.getroot()
    return parse_response(root)

def parse_response(elem):
    response = schemata.FlexQueryResponse.convert(elem)
    flexStmts = elem.findall('FlexStatements')

    # Sanity check
    if len(flexStmts) != 1:
        msg = ("FlexQueryResponse must contain exactly 1 FlexStatements, "
               "not {}")
        raise FlexParserError(msg.format(len(flexStmts)))

    response['FlexStatements'] = parse_stmts(flexStmts[0])
    return response

def parse_stmts(elem):
    attrs = schemata.FlexStatements.convert(elem)
    count = attrs['count']
    flexStmt = elem.findall('FlexStatement')

    # Sanity check
    if len(flexStmt) != count:
        msg = ("FlexStatements declares count={} "
               "but contains {} FlexStatement elements")
        raise FlexParserError(msg.format(count, len(flexStmt)))

    return [parse_stmt(stmt) for stmt in flexStmt]

def parse_stmt(elem):
    stmt = schemata.FlexStatement.convert(elem)
    children = dict(parse_stmt_child(child) for child in elem)
    stmt.update(children)
    return stmt

def parse_stmt_child(elem):
    # If the tag isn't one of the special cases defined in stmt_child_parsers,
    # by default treat it as a list container
    parser = stmt_child_parsers.get(elem.tag, parse_list)
    return parser(elem)

def parse_acctinfo(elem):
    return 'AccountInformation', schemata.AccountInformation.convert(elem)

def parse_rates(elem):
    return 'ConversionRates', dict(parse_rate(rate) for rate in elem)

def parse_rate(elem):
    """
    Use input schema to convert element.

    Return duple of ((currency, date), rate) suitable for use
    as a dict item.
    """
    rate = schemata.ConversionRate.convert(elem)
    return (rate['fromCurrency'], rate['reportDate']), rate['rate']

def parse_fxpos(elem):
    if len(elem) == 0:
        items = []
    else:
        # Sanity check
        if len(elem) != 1:
            msg = "<{}> has more than 1 child: {}".format(
                elem.tag,
                ', '.join(["<{}>".format(child.tag) for child in elem])
            )
            raise FlexParserError(msg)
        fxLots = elem[0]
        if fxLots.tag != 'FxLots':
            msg = "<{}> can only contain <FxLots>, not <{}>"
            raise FlexParserError(msg.format(elem.tag, fxLots))

        attrName, items = parse_list(fxLots)

    return 'FxPositions', items

def parse_list(elem):
    """
    Use container tag to look up conversion schema for contained items.

    Return duple of (container tag, list of converted items).
    """
    tag = elem.tag
    try:
        schema = elementSchemata[tag]
    except KeyError:
        msg = "Don't know the schema to parse items in {}"
        raise FlexParserError(msg.format(tag))
    items = [schema.convert(item) for item in elem]
    return tag, items

stmt_child_parsers = {'AccountInformation': parse_acctinfo,
                      'ConversionRates': parse_rates,
                      'FxPositions': parse_fxpos, }

##############################################################################
# CLI SCRIPT
###############################################################################
def main():
    from argparse import ArgumentParser

    argparser = ArgumentParser(
        description='Parse Interactive Brokers Flex XML data')
    argparser.add_argument('file', nargs='+', help='XML data file(s)')
    args = argparser.parse_args()

    for file in args.file:
        print(file)
        response = parse(file)
        # print(response)
        for stmt in response['FlexStatements']:
            for trade in stmt['Trades']:
                print(trade)

if __name__ == '__main__':
    main()


# coding=utf-8
# Copyright (c) 2019 NVIDIA CORPORATION. All rights reserved.
# Copyright 2018 The Google AI Language Team Authors and The HugginFace Inc. team.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Run TinyBERT on SST-2."""

from __future__ import absolute_import, division, print_function
import argparse
import os
import sys
import csv
import io
from transformer.tokenization import BertTokenizer
import torch
from torch.utils.data import (DataLoader, SequentialSampler, TensorDataset)

class InputExample(object):
    """A single training/test example for simple sequence classification."""

    def __init__(self, guid, text_a, text_b=None, label=None):
        """Constructs a InputExample.

        Args:
            guid: Unique id for the example.
            text_a: string. The untokenized text of the first sequence. For single
            sequence tasks, only this sequence must be specified.
            text_b: (Optional) string. The untokenized text of the second sequence.
            Only must be specified for sequence pair tasks.
            label: (Optional) string. The label of the example. This should be
            specified for train and dev examples, but not for test examples.
        """
        self.guid = guid
        self.text_a = text_a
        self.text_b = text_b
        self.label = label

class InputFeatures(object):
    """A single set of features of data."""

    def __init__(self, input_ids, input_mask, segment_ids, label_id, seq_length=None):
        self.input_ids = input_ids
        self.input_mask = input_mask
        self.segment_ids = segment_ids
        self.seq_length = seq_length
        self.label_id = label_id

class DataProcessor(object):
    """Base class for data converters for sequence classification data sets."""

    def get_train_examples(self, data_dir):
        """Gets a collection of `InputExample`s for the train set."""
        raise NotImplementedError()

    def get_dev_examples(self, data_dir):
        """Gets a collection of `InputExample`s for the dev set."""
        raise NotImplementedError()

    def get_labels(self):
        """Gets the list of labels for this data set."""
        raise NotImplementedError()

    @classmethod
    def _read_tsv(cls, input_file, quotechar=None):
        """Reads a tab separated value file."""
        with io.open(input_file, "r", encoding="utf-8") as f:
            reader = csv.reader(f, delimiter="\t", quotechar=quotechar)
            lines = []
            for line in reader:
                if sys.version_info[0] == 2:
                    line = list(unicode(cell, 'utf-8') for cell in line)
                lines.append(line)
            return lines

class Sst2Processor(DataProcessor):
    """Processor for the SST-2 data set (GLUE version)."""

    def get_train_examples(self, data_dir):
        """See base class."""
        return self._create_examples(
            self._read_tsv(os.path.join(data_dir, "train.tsv")), "train")

    def get_dev_examples(self, data_dir):
        """See base class."""
        return self._create_examples(
            self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev")

    def get_aug_examples(self, data_dir):
        """get the augmented data"""
        return self._create_examples(
            self._read_tsv(os.path.join(data_dir, "train_aug.tsv")), "aug")

    def get_labels(self):
        """See base class."""
        return ["0", "1"]

    def _create_examples(self, lines, set_type):
        """Creates examples for the training and dev sets."""
        examples = []
        for (i, line) in enumerate(lines):
            if i == 0:
                continue
            guid = "%s-%s" % (set_type, i)
            text_a = line[0]
            label = line[1]
            examples.append(
                InputExample(guid=guid, text_a=text_a, text_b=None, label=label))
        return examples

def _truncate_seq_pair(tokens_a, tokens_b, max_length):
    """Truncates a sequence pair in place to the maximum length."""
    while True:
        total_length = len(tokens_a) + len(tokens_b)
        if total_length <= max_length:
            break
        if len(tokens_a) > len(tokens_b):
            tokens_a.pop()
        else:
            tokens_b.pop()

def convert_examples_to_features(examples, label_list, max_seq_length,
                                 tokenizer, output_mode):
    """Loads a data file into a list of `InputBatch`s."""

    label_map = {label: i for i, label in enumerate(label_list)}

    features = []
    for (ex_index, example) in enumerate(examples):

        tokens_a = tokenizer.tokenize(example.text_a)

        tokens_b = None
        if example.text_b:
            tokens_b = tokenizer.tokenize(example.text_b)
            _truncate_seq_pair(tokens_a, tokens_b, max_seq_length - 3)
        else:
            if len(tokens_a) > max_seq_length - 2:
                tokens_a = tokens_a[:(max_seq_length - 2)]

        tokens = ["[CLS]"] + tokens_a + ["[SEP]"]
        segment_ids = [0] * len(tokens)

        if tokens_b:
            tokens += tokens_b + ["[SEP]"]
            segment_ids += [1] * (len(tokens_b) + 1)

        input_ids = tokenizer.convert_tokens_to_ids(tokens)
        input_mask = [1] * len(input_ids)
        seq_length = len(input_ids)

        padding = [0] * (max_seq_length - len(input_ids))
        input_ids += padding
        input_mask += padding
        segment_ids += padding

        assert len(input_ids) == max_seq_length
        assert len(input_mask) == max_seq_length
        assert len(segment_ids) == max_seq_length

        if output_mode == "classification":
            label_id = label_map[example.label]
        elif output_mode == "regression":
            label_id = float(example.label)
        else:
            raise KeyError(output_mode)

        features.append(
            InputFeatures(input_ids=input_ids,
                          input_mask=input_mask,
                          segment_ids=segment_ids,
                          label_id=label_id,
                          seq_length=seq_length))
    return features

def get_tensor_data(output_mode, features):
    """get the data"""
    if output_mode == "classification":
        all_label_ids = torch.tensor([f.label_id for f in features], dtype=torch.long)
    elif output_mode == "regression":
        all_label_ids = torch.tensor([f.label_id for f in features], dtype=torch.float)

    all_seq_lengths = torch.tensor([f.seq_length for f in features], dtype=torch.long)
    all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
    all_input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long)
    all_segment_ids = torch.tensor([f.segment_ids for f in features], dtype=torch.long)
    tensor_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids,
                                all_label_ids, all_seq_lengths)
    return tensor_data, all_label_ids

def data_main():
    """preprocess data"""
    parser = argparse.ArgumentParser()
    parser.add_argument("--model",
                        default=None,
                        type=str,
                        required=True,
                        help="The student model dir.")
    parser.add_argument("--data_dir",
                        default=None,
                        type=str,
                        required=True,
                        help="The input data dir. Should contain the .tsv files (or other data files) for the task.")
    parser.add_argument("--max_seq_length",
                        default=64,
                        type=int,
                        help="The maximum total input sequence length after WordPiece tokenization. \n"
                             "Sequences longer than this will be truncated, and sequences shorter \n"
                             "than this will be padded.")
    parser.add_argument("--eval_batch_size",
                        default=1,
                        type=int,
                        help="Total batch size for eval.")
    parser.add_argument("--do_lower_case",
                        action='store_true',
                        help="Set this flag if you are using an uncased model.")
    parser.add_argument("--inference_tool", type = str,
                        help = "inference tool:benchmark or msame or ais_infer")
    parser.add_argument("--bin_path",
                        default="./bert_bin",
                        type=str,
                        help="The path to bin file.")
    parser.add_argument("--output_mode",
                        default='classification',
                        type=str,
                        help="The mode of output.")
    args = parser.parse_args()
    processor = Sst2Processor()
    tokenizer = BertTokenizer.from_pretrained(args.model, do_lower_case=args.do_lower_case)  # for TinyBERT

    eval_examples = processor.get_dev_examples(args.data_dir)
    label_list = ["0", "1"]
    eval_features = convert_examples_to_features(eval_examples, label_list, args.max_seq_length,
                                                 tokenizer, output_mode = "classification")

    eval_data, eval_labels = get_tensor_data(args.output_mode, eval_features)
    print("eval_labels")
    # Run prediction for full data
    eval_sampler = SequentialSampler(eval_data)
    eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, drop_last = True,
                                 batch_size = args.eval_batch_size, shuffle = False)

    if not os.path.exists(args.bin_path):
        os.makedirs(args.bin_path)
    i = -1

    if args.inference_tool == "msame" or "ais_infer":
        path1 = args.bin_path + "/input_ids"
        path2 = args.bin_path + "/segment_ids"
        path3 = args.bin_path + "/input_mask"
        if not os.path.exists(path1):
            os.mkdir(path1)
        if not os.path.exists(path2):
            os.mkdir(path2)
        if not os.path.exists(path3):
            os.mkdir(path3)

    for input_ids, input_mask, segment_ids, label_ids, seq_lengths in eval_dataloader:
        i = i + 1
        print("[info] file", "===", i)
        input_ids_np = input_ids.numpy()
        input_mask_np = input_mask.numpy()
        segment_ids_np = segment_ids.numpy()
        if args.inference_tool == "msame":
            input_ids_np.tofile(os.path.join(path1, "input" + str(i) + '.bin'))
            segment_ids_np.tofile(os.path.join(path2, "input" + str(i) + '.bin'))
            input_mask_np.tofile(os.path.join(path3, "input" + str(i) + '.bin'))
        elif args.inference_tool == "benchmark":
            input_ids_np.tofile(os.path.join(args.bin_path, "input_ids_" + str(i) + '.bin'))
            segment_ids_np.tofile(os.path.join(args.bin_path, "segment_ids_" + str(i) + '.bin'))
            input_mask_np.tofile(os.path.join(args.bin_path, "input_mask_" + str(i) + '.bin'))
        elif args.inference_tool == "ais_infer":
            input_ids_np.tofile(os.path.join(path1, "input" + str(i) + '.bin'))
            segment_ids_np.tofile(os.path.join(path2, "input" + str(i) + '.bin'))
            input_mask_np.tofile(os.path.join(path3, "input" + str(i) + '.bin'))

if __name__ == "__main__":
    data_main()


"""
Piedra Papel tijeras
¡Vamos a jugar! ¡Tienes que devolver qué jugador ganó! En caso de empate devolución Draw!.

Ejemplos:

rps('scissors','paper') // Player 1 won!
rps('scissors','rock') // Player 2 won!
rps('paper','paper') // Draw!
"""

def rps(p1, p2):
    game = {
        p1 : "Player 1",
        p2 : "Player 2"
        }

    if 'rock' in game and "scissors" in game:
        return f'{game["rock"]} won!'

    elif 'scissors' in game and "paper" in game:
        return f'{game["scissors"]} won!'

    elif 'paper' in game and "rock" in game:
        return f'{game["paper"]} won!'

    elif p1 == p2:
        return f'Draw!'

    else:
        return "Recuerde ingresar simbolo valido"

 #Otra forma
#  def rps(p1, p2):
#     beats = {'rock': 'scissors', 'scissors': 'paper', 'paper': 'rock'}
#     if beats[p1] == p2:
#         return "Player 1 won!"
#     if beats[p2] == p1:
#         return "Player 2 won!"
#     return "Draw!"

# S, P, R = 'scissors', 'paper', 'rock'
# P1, P2, D = "Player 1 won!", "Player 2 won!", 'Draw!'
# MAPPING = {
#     (S, P): P1, (S, S): D, (S, R): P2,
#     (P, P): D, (P, S): P2, (P, R): P1,
#     (R, P): P2, (R, S): P1, (R, R): D,
# }

# rps = lambda *p: MAPPING[p]

print(rps( 'paper', 'scissors'))


def jugar(intento=1):
    respuesta = input("De que color es la fruta?: \n")
    if respuesta != "naranja":
        if intento <3:
            print("\n Fallaste, intentalo de nuevo")
            intento +=1
            jugar(intento) #Llamada recursiva
        else:
            print("\n Perdiste, intentalo de nuevo")
    else:
        print("\n Ganaste, eres lo maximo!")

jugar()


MEM = {}

def applyMask(value, mask):
    binVal = list('{0:036b}'.format(value))
    for ii in range(len(mask)):
        if mask[ii] != 'X':
            binVal[ii] = mask[ii]

    returnVal = ""
    return (int(returnVal.join(binVal),base=2))

def applyMemoryMask(address, mask):
    global MEM
    binAddr = list('{0:036b}'.format(address))
    floating = []
    for ii in range(len(mask)):
        if mask[ii] == '1':
            binAddr[ii] = '1'
        elif mask[ii] == 'X':
            binAddr[ii] = '0'
            floating.append(ii)

    binAddrs = []
    for ii in range(2**len(floating)):
        binAddrs.append(binAddr.copy())

    step = 2
    for ii in range(len(floating)):
        for jj in range(0,len(binAddrs),step):
            for kk in range(jj,jj+int(step/2)):
                binAddrs[kk][floating[ii]] = '1'

        step = step*2

    returnVal = ""
    returnArray = []

    for binAddr in binAddrs:
        returnArray.append(int(returnVal.join(binAddr),base=2))

    return returnArray

if __name__ == "__main__":
    file = open("Input/DockingData.txt","r")
    lines = file.readlines()

    mask = None

    for line in lines:
        if line[1] == "a":
            mask = list(line.split(' ')[-1][0:-1])

        else:
            addr = line.split(']')[0].split('[')[1]
            value = int(line.split(' = ')[1])
            MEM[addr] = applyMask(value, mask)

    print(sum(MEM.values()))

    MEM = {}
    mask = None

    for line in lines:
        if line[1] == "a":
            mask = list(line.split(' ')[-1][0:-1])
        else:
            addr = int(line.split(']')[0].split('[')[1])
            value = int(line.split(' = ')[1])
            addr = applyMemoryMask(addr, mask)
            for ad in addr:
                MEM[ad] = value

    print(sum(MEM.values()))


import my_user
from unittest.mock import Mock

user = Mock(spec_set=my_user)
print(
    user.get_today()
)

print(
    user.get_users()
)

# AttributeError: Mock object has no attribute 'attr_a'
# user.attr_a = 123