import scanpy as sc
import anndata as ad
import pandas as pd
import numpy as np
import os,pickle,random
import matplotlib.pyplot as plt
import pickle

####  multi-processing
from tqdm import tqdm
#from tqdm.notebook import tqdm
from concurrent.futures import ProcessPoolExecutor, as_completed
import numpy as np
def poolmap(function,array, n_jobs=30,use_kwargs=None,use_args=None):
    if use_kwargs is None: use_kwargs = True if isinstance(array[0],dict) else False
    if use_args is None: use_args = True if isinstance(array[0],(list,np.ndarray)) else False
    with ProcessPoolExecutor(max_workers=n_jobs) as pool:
        if use_kwargs: futures = [pool.submit(function, **a) for a in array]
        elif use_args:futures = [pool.submit(function, *a) for a in array]
        else:futures = [pool.submit(function, a) for a in array]
        _ = list( tqdm(as_completed(futures), total=len(futures)))
    out = [ x.result() for x in futures]
    return  out
####  multi-processing

def grouped_obs_mean(adata, group_key, layer=None,
                     log_space_averaging= True):
    # define an operator for expression matrix retieval
    if layer is not None:
        getX = lambda x: x.layers[layer]
    else:
        getX = lambda x: x.X

    # split metadata dataframe into sets of rows by group_key
    grouped = adata.obs.groupby(group_key)

    # build new empty expression matrix
    out = pd.DataFrame(
        np.zeros((adata.shape[1], len(grouped)), dtype=np.float64), # nGene-by-nCelltype zero matrix
        columns = list(grouped.groups.keys()), # colnames := list of cell types
        index   = adata.var_names #rowname:= gene symbols
    )

    # compute average expression in the celltype-wise manner (by celltype column)
    for group, idx in grouped.indices.items(): # group:=celltype idx:=barcodes
        X = getX(adata[idx]) # cell-by-gene matrix
        if log_space_averaging: # averaging in the log-space, return the log of the averaged values
            out[group] = np.ravel(np.log(np.exp(X).mean(axis=0, dtype=np.float64))) # axis=0:= avg across cells
        else:
            out[group] = np.ravel(X.mean(axis=0, dtype=np.float64)) # axis=0:= avg across cells
    return out

def primary_trimming(adata, LRDB, groupby):
    # 1. check if gene symbols are occurred in the dataset
    sel=np.logical_and(LRDB["ligand_symbol"].isin(adata.var_names),
                       LRDB["receptor_symbol"].isin(adata.var_names))
    LRDB=LRDB[sel]

    genesel = np.unique(np.union1d(LRDB["ligand_symbol"],
                                   LRDB["receptor_symbol"]))
    adata   = adata[:,genesel].copy()

    # 2. exclude zero expression pairs
    avgexpr     = grouped_obs_mean(adata, group_key=groupby)
    lig_symbols = LRDB["ligand_symbol"]
    rec_symbols = LRDB["receptor_symbol"]
    lig_expr    = avgexpr.loc[lig_symbols]
    rec_expr    = avgexpr.loc[rec_symbols]
    sel = np.logical_not(np.logical_or(
              (rec_expr.T==0).all().to_numpy(),
              (lig_expr.T==0).all().to_numpy() ))
    LRDB = LRDB[sel]

    genesel = np.unique(np.union1d(LRDB["ligand_symbol"],
                                   LRDB["receptor_symbol"]))
    adata   = adata[:,genesel].copy()

    return(adata, LRDB)

def get_crosstalk(adata, groupby, LRDB):
    # prepare ligand/receptor expressions
    avgexpr     = grouped_obs_mean(adata, group_key=groupby)
    lig_symbols = LRDB["ligand_symbol"]
    rec_symbols = LRDB["receptor_symbol"]
    lig_expr    = avgexpr.loc[lig_symbols]
    rec_expr    = avgexpr.loc[rec_symbols]
    pair_symbols= LRDB["pair"]

    # compute cell-type level crosstalk
    crosstalk_by_celltype=dict()
    for ct1 in (avgexpr.columns):
        for ct2 in (avgexpr.columns):
            lig_vec = lig_expr[ct1].to_numpy()
            rec_vec = rec_expr[ct2].to_numpy()
            df = pd.DataFrame(data=lig_vec*rec_vec,
                              index=pair_symbols,
                              columns=["intensity"],
                              dtype=np.float64 )
            df["ligand_symbol"]   = lig_symbols.tolist()
            df["receptor_symbol"] = rec_symbols.tolist()
            df["ligand_avgexpr"]  = lig_vec
            df["receptor_avgexpr"]= rec_vec
            df["p_val"]     = np.nan
            df["p_val_adj"] = np.nan
            crosstalk_by_celltype[ (ct1, ct2) ]=df
    return(crosstalk_by_celltype)

def grouped_rand_mean(adata, group_key, layer=None,log_space_averaging=True):
    # define an operator for expression matrix retieval
    if layer is not None:
        getX = lambda x: x.layers[layer]
    else:
        getX = lambda x: x.X

    # shuffle the identities
    import random
    adata.obs["random"]=random.sample(adata.obs[group_key].tolist(), len(adata.obs))

    # split metadata dataframe into sets of rows by group_key
    grouped = adata.obs.groupby("random")

    # build new empty expression matrix
    out = pd.DataFrame(
        np.zeros((adata.shape[1], len(grouped)), dtype=np.float64), # nGene-by-nCelltype zero matrix
        columns=list(grouped.groups.keys()), # colnames := list of cell types
        index=adata.var_names #rowname:= gene symbols
    )

    # compute average expression in the celltype-wise manner (by celltype column)
    for group, idx in grouped.indices.items(): # group:=celltype idx:=barcodes
        X = getX(adata[idx]) # cell-by-gene matrix
        if log_space_averaging: # averaging in the log-space, return the log of the averaged values
            out[group] = np.ravel(np.log(np.exp(X).mean(axis=0, dtype=np.float64))) # axis=0:= avg across cells
        else:
            out[group] = np.ravel(X.mean(axis=0, dtype=np.float64)) # axis=0:= avg across cells
    return out

def get_significance(adata, crosstalk, LRDB, n_perm, groupby,
                     use_jupyter=True, pseudo_expr=1e-5):
    lig_symbols = LRDB["ligand_symbol"]
    rec_symbols = LRDB["receptor_symbol"]
    nLarger = dict()
    #nLower  = dict()
    avg_randintensity =dict()
    celltype_list = list(set([x[0] for x in crosstalk.keys()]))
    #celltype_list = list(randexpr.columns)

    # generate random permutations
    from tqdm import tqdm
    from tqdm import trange
    for r in trange(n_perm):
        # generate random ligand/receptor expressions
        randexpr      = grouped_rand_mean(adata, group_key=groupby)
        randexpr_lig  = randexpr.loc[lig_symbols]
        randexpr_rec  = randexpr.loc[rec_symbols]

        # enumerate cell type pairs
        for i in celltype_list: # ligand cell type
            for j in celltype_list: # receptor cell type
                # calculate pair intensities (real/random)
                intensity     = crosstalk[(i,j)]['intensity']
                randintensity = np.array(randexpr_lig[i]) * np.array(randexpr_rec[j])

                # calculate the average of random intensities
                if (i,j) not in avg_randintensity:
                    avg_randintensity[(i,j)] = randintensity
                else:
                    avg_randintensity[(i,j)] = avg_randintensity[(i,j)]+ randintensity

                # count extreme events
                if (i,j) not in nLarger:
                    #import pdb;pdb.set_trace();
                    nLarger[(i,j)] = (intensity>=randintensity).astype(int)
                else:
                    nLarger[(i,j)] = nLarger[(i,j)] + (intensity>=randintensity).astype(int)
                #if (i,j) not in nLower:
                #    nLower[(i,j)]  = (intensity<randintensity).astype(int)
                #else:
                #    nLower[(i,j)]  = nLower[(i,j)]  + (intensity<randintensity).astype(int)

    # calculate p-values and fold changes
    from statsmodels.stats.multitest import multipletests
    pseudo_expr = 0 if pseudo_expr is None else pseudo_expr
    for i in (celltype_list):
        for j in (celltype_list):
            avg_randintensity[(i,j)] = avg_randintensity[(i,j)]/n_perm
            crosstalk[(i,j)]["avg_null"] = avg_randintensity[(i,j)]
            right_tail=2.0*(1+nLarger[(i,j)])/(1+n_perm)
            #left_tail=2.0*(1+nLower[(i,j)])/(1+n_perm)
            left_tail=2.0*(1-(nLarger[(i,j)]/(1+n_perm)))
            #import pdb;pdb.set_trace();
            p_val_vec= np.minimum(
                right_tail,left_tail
            )
            crosstalk[(i,j)]["p_val"] =p_val_vec
            crosstalk[(i,j)]["p_val_adj"] =multipletests(pvals=p_val_vec,
                                                     alpha=0.05,
                                                     method="fdr_bh",
                                                     is_sorted=False,
                                                     returnsorted=False)[1]
            crosstalk[(i,j)]["cell_ligand"]= i
            crosstalk[(i,j)]["cell_receptor"]= j
            crosstalk[(i,j)]["log_avgFC"] =np.log2(
                 (pseudo_expr +crosstalk[(i,j)]["intensity"].to_numpy())/
                 (pseudo_expr +avg_randintensity[(i,j)])  )

            # filter analysis
            #sel = crosstalk[(i,j)].log_avgFC>0
            #crosstalk[(i,j)]=crosstalk[(i,j)][sel]

    return crosstalk

def parallel_func(perm_idx,):
    global GLOBAL_crosstalk,GLOBAL_crosstalk,GLOBAL_LRDB,GLOBAL_groupby

    lig_symbols = GLOBAL_LRDB["ligand_symbol"]
    rec_symbols = GLOBAL_LRDB["receptor_symbol"]
    randexpr      = grouped_rand_mean(GLOBAL_adata, group_key=GLOBAL_groupby)
    randexpr_lig  = randexpr.loc[lig_symbols]
    randexpr_rec  = randexpr.loc[rec_symbols]
    num_LRs = GLOBAL_LRDB.shape[0]
    celltype_list = list(set([x[0] for x in GLOBAL_crosstalk.keys()]))

    num_ct = len(celltype_list)
    avg_randintensity = np.zeros((num_ct,num_ct,num_LRs))
    nLarger = np.zeros((num_ct,num_ct,num_LRs))
    # enumerate cell type pairs
    for idx_i,i in  enumerate(celltype_list): # ligand cell type
        for idx_j,j in enumerate(celltype_list): # receptor cell type
            # calculate pair intensities (real/random)
            intensity     = GLOBAL_crosstalk[(i,j)]['intensity'].values
            randintensity = np.array(randexpr_lig[i]) * np.array(randexpr_rec[j])

            # calculate the average of random intensities
            avg_randintensity[idx_i,idx_j] = randintensity
            nLarger[idx_i,idx_j] = (intensity>=randintensity).astype(int)
    return avg_randintensity,nLarger

global GLOBAL_crosstalk,GLOBAL_LRDB,GLOBAL_adata,GLOBAL_groupby

def get_significance_parallel(adata, crosstalk, LRDB, n_perm, groupby,
                     use_jupyter=True, pseudo_expr=1e-5,n_jobs=30):
    lig_symbols = LRDB["ligand_symbol"]
    rec_symbols = LRDB["receptor_symbol"]

    celltype_list = list(set([x[0] for x in crosstalk.keys()]))

    global GLOBAL_crosstalk,GLOBAL_LRDB,GLOBAL_adata,GLOBAL_groupby
    GLOBAL_crosstalk = crosstalk
    GLOBAL_LRDB = LRDB
    GLOBAL_adata = adata
    GLOBAL_groupby = groupby

    pool_res = poolmap(parallel_func,range(n_perm),n_jobs = n_jobs)
    pool_res = np.array(pool_res) # [n_perm,2,num_ct,num_ct,num_LRs]
    avg_randintensity = pool_res[:,0].mean(axis=0)
    nLarger = pool_res[:,1].sum(axis=0)

    #right_tail_mat = 2.0*(1+nLarger)/(1+n_perm)
    #left_tail_mat = 2.0*(1-nLarger)/(1+n_perm)    # this is a bug, causing negative p-values, corrected as following two lines
    right_tail_mat = 2.0*     (1+nLarger)/(1+n_perm)
    left_tail_mat  = 2.0* (1- (  nLarger)/(1+n_perm) )
    p_val_vec_mat = np.minimum(right_tail_mat,left_tail_mat)

    from statsmodels.stats.multitest import multipletests
    pseudo_expr = 0 if pseudo_expr is None else pseudo_expr
    for idx_i,i in enumerate(celltype_list):
        for idx_j,j in enumerate(celltype_list):
            p_val_vec =p_val_vec_mat[idx_i,idx_j]

            crosstalk[(i,j)]["avg_null"] = avg_randintensity[idx_i,idx_j]
            crosstalk[(i,j)]["p_val"] =p_val_vec
            crosstalk[(i,j)]["p_val_adj"] =multipletests(pvals=p_val_vec,
                                                     alpha=0.05,
                                                     method="fdr_bh",
                                                     is_sorted=False,
                                                     returnsorted=False)[1]
            crosstalk[(i,j)]["cell_ligand"]= i
            crosstalk[(i,j)]["cell_receptor"]= j
            crosstalk[(i,j)]["log_avgFC"] =np.log2(
                 (pseudo_expr +crosstalk[(i,j)]["intensity"].to_numpy())/
                 (pseudo_expr +avg_randintensity[idx_i,idx_j])  )

    return crosstalk

#-----------------------------------------------------------------------------

def get_expression_fraction(anndata, gene, groupby, group):
    sel=anndata.obs[groupby]==group
    subset = anndata[sel,:]
    tot=subset.n_obs
    expressed=subset[subset[:, gene].X > 0, :].n_obs
    return float(expressed/tot)

def get_expressing_fraction(adata, genes, groups, groupby):
    gene_ids = list(set(genes))# adata.var.index.values
    clusters = adata.obs[groupby].unique().tolist()
    obs = adata[:,gene_ids].X.toarray()
    obs = pd.DataFrame(obs,columns=gene_ids,index=adata.obs[groupby])
    #average_obs = obs.groupby(level=0).mean()
    obs_bool = obs.astype(bool)
    fraction_obs = obs_bool.groupby(level=0).sum()/obs_bool.groupby(level=0).count()
    #return fraction_obs.T
    #i = pd.Series(group)
    return fraction_obs.T.lookup(genes, groups)

def crosstalk_analysis(anndata, LRDB_table, groupby, n_perm=1000):
    adata, LRDB = primary_trimming(anndata, LRDB_table, groupby=groupby)
    crosstalk = get_crosstalk(adata, groupby=groupby, LRDB=LRDB)
    import warnings;warnings.filterwarnings("ignore");
    crosstalk = get_significance(adata, crosstalk, LRDB, n_perm=n_perm, groupby=groupby)

    df_crosstalk = pd.DataFrame()
    for key in crosstalk.keys():
        cf=crosstalk[key]
        df_crosstalk=df_crosstalk.append(cf)
    df_crosstalk.index+="_"+df_crosstalk["cell_ligand"]+"_"
    df_crosstalk.index+=    df_crosstalk["cell_receptor"]

    # add gene expression fraction
    ##def add_lig_fraction(row):
    ##    return get_expression_fraction(anndata, row["ligand_symbol"], groupby=groupby, group=row["cell_ligand"])
    ##def add_rec_fraction(row):
    ##    return get_expression_fraction(anndata, row["receptor_symbol"],groupby=groupby,group=row["cell_receptor"])
    ##df_crosstalk['ligand_pct']  =df_crosstalk.apply (lambda row: add_lig_fraction(row), axis=1)
    ##df_crosstalk['receptor_pct']=df_crosstalk.apply (lambda row: add_rec_fraction(row), axis=1)
    df_crosstalk['ligand_pct']  = get_expressing_fraction(adata=anndata,
                                                          genes=df_crosstalk["ligand_symbol"],
                                                          groups=df_crosstalk["cell_ligand"],
                                                          groupby=groupby)
    df_crosstalk['receptor_pct']= get_expressing_fraction(adata=anndata,
                                                          genes=df_crosstalk["receptor_symbol"],
                                                          groups=df_crosstalk["cell_receptor"],
                                                          groupby=groupby)
    return(df_crosstalk)

#---------------------------------------------------------------------------------

def diff_pair(crosstalk1, crosstalk2,
              ident1, ident2):
    # get common, group1-only, group2-only pairs
    common_idx =  crosstalk1.index.intersection(crosstalk2.index)
    g1_idx = crosstalk1.index.difference(crosstalk2.index)
    g2_idx = crosstalk2.index.difference(crosstalk1.index)

    # chosen columns
    selected_col=["intensity","ligand_pct","receptor_pct"]

    # assembly group comparison DataFrames for common pairs
    df_group1= crosstalk1.loc[common_idx][selected_col]
    df_group1.rename(columns =  dict(zip(df_group1.columns, df_group1.columns+"_"+ident1)), inplace=True  )
    df_group2= crosstalk2.loc[common_idx][selected_col]
    df_group2.rename(columns =  dict(zip(df_group2.columns, df_group2.columns+"_"+ident2)), inplace=True  )
    df_common= df_group1.join(df_group2)

    # assembly group comparison DataFrame for group1-only pairs
    import copy
    df_group1 =crosstalk1.loc[g1_idx][selected_col]
    df_group2 =copy.copy(df_group1)
    for col in df_group2.columns:
        df_group2[col].values[:] = 0
    df_group1.rename(columns =  dict(zip(df_group1.columns, df_group1.columns+"_"+ident1)), inplace=True  )
    df_group2.rename(columns =  dict(zip(df_group2.columns, df_group2.columns+"_"+ident2)), inplace=True  )
    df_group1only= df_group1.join(df_group2)

    # assembly group comparison DataFrame for group2-only pairs
    df_group2 =crosstalk2.loc[g2_idx][selected_col]
    df_group1 =copy.copy(df_group2)
    for col in df_group1.columns:
        df_group1[col].values[:] = 0
    df_group1.rename(columns =  dict(zip(df_group1.columns, df_group1.columns+"_"+ident1)), inplace=True  )
    df_group2.rename(columns =  dict(zip(df_group2.columns, df_group2.columns+"_"+ident2)), inplace=True  )
    df_group2only= df_group1.join(df_group2)

    df= df_common.append(df_group1only).append(df_group2only)

    intensity1=df["intensity_"+ident1]
    intensity2=df["intensity_"+ident2]
    df["intensity_log2FC"]=np.log2(intensity2/intensity1)
    df= df.reindex(sorted(df.columns), axis=1)
    return df

#--------------------------------------------------------------------------------
def depair_plot(DEpair, ident1, ident2, figsize=[7,6],
                xcut=0.5, ycut=0.5, cutwidth=0.5, ptsize=10, vmin=-1, vmax=1,
                highlight=True, highlight_fc_upper=0.5, highlight_fc_lower=-0.5
               ):
    import numpy as np
    import seaborn as sns
    import matplotlib.pyplot as plt
    cmap = sns.color_palette("coolwarm", as_cmap=True,)
    f, ax = plt.subplots(figsize=figsize)

    # axis names
    xaxis,yaxis="intensity_"+ident1, "intensity_"+ident2
    ax.set(xlabel="Intensities "+ident1, ylabel="Intensities "+ident2)

    # get coordinates
    xpos=DEpair[xaxis]
    ypos=DEpair[yaxis]
    points = ax.scatter(x=xpos,
                        y=ypos,
                        c=DEpair["intensity_log2FC"], s=ptsize, cmap=cmap,
                        vmin=vmin,vmax=vmax
                        )
    # highlight points
    if highlight:
        label_sel = np.logical_or(DEpair["intensity_log2FC"]>highlight_fc_upper,
                                  DEpair["intensity_log2FC"]<highlight_fc_lower)
        label_sel = np.logical_and(label_sel, DEpair[xaxis]>xcut)
        label_sel = np.logical_and(label_sel, DEpair[yaxis]>ycut)

        label_x = DEpair[label_sel][xaxis].tolist()
        label_y = DEpair[label_sel][yaxis].tolist()
        label_text = DEpair[label_sel].index.tolist()

        texts = []
        for x, y, s in zip(label_x, label_y, label_text):
            texts.append(ax.text(x, y, s))
        from adjustText import adjust_text
        adjust_text(texts, force_points=0.2, force_text=0.2,
                    expand_points=(1, 1), expand_text=(1, 1),
                    arrowprops=dict(arrowstyle="-", color='black', lw=0.5))

    # bound of axis
    bound= DEpair[xaxis].max()
    bound= max(bound, DEpair[yaxis].max())
    bound = int(bound+1)

    # set ticks
    ax.xaxis.set_ticks(np.linspace(start=0,stop=bound, num=int(bound+1)))
    ax.yaxis.set_ticks(np.linspace(start=0,stop=bound, num=int(bound+1)))
    from matplotlib.ticker import AutoMinorLocator
    ax.xaxis.set_minor_locator(AutoMinorLocator())
    ax.yaxis.set_minor_locator(AutoMinorLocator())

    # draw auxilary lines
    plt.plot([0, bound], [0, bound], 'r--',linewidth=cutwidth)
    plt.plot([xcut, ycut], [0, bound], 'r--',linewidth=cutwidth)
    plt.plot([0,bound], [xcut, ycut], 'r--',linewidth=cutwidth)
    f.colorbar(points)


"""Ghozt
Lab3D

Instructions
    Write a program that prompts a user to input an integer and calculates the factorial of that number using a while loop.
"""

# Request user input as int
number = int(raw_input("Enter a number: "))

# Make placeholder for solution
tempNumber = number
count = (number - 1)

# Loop until no more numbers to mulitply by
while count > 0:
    print('%d X %d = %d' % (tempNumber, count, tempNumber * count))
    tempNumber *= count
    # Decrease amount of remaining
    count -= 1
# No numbers left to multipe by. Loop complete!
else:
	# #profit
    print('\nWhile loop complete! \n%d! = %d' % (number, tempNumber))


import unittest

from django_remote_queryset.queries import QueryDecoder, Filter, CompositeQuery

class QueryTest(unittest.TestCase):

    def test_filter(self):
        json_query = \
            {
                '_query_class': 'compositequery',
                '_sub_queries': [{
                    '_query_class': 'filter',
                    '_condition': 'name__icontains',
                    '_value': 'nicola'
                }]
            }

        query = QueryDecoder.decodeJSONQuery(json_query)

        assert query

        assert isinstance(query, CompositeQuery)

        assert isinstance(query.get_sub_queries()[0], Filter)


import dolphindb.orca as orca
import matplotlib.pyplot as plt

orca.connect('localhost', 8848, 'admin', '123456')

def main():
    quotes = orca.read_table('dfs://TAQ', 'quotes')

    len(quotes)

    # calculate the average bid-ask spread of LEH on 2007.08.31
    tmp = quotes[
        quotes.date == '2007.08.31',
        quotes.symbol == 'LEH',
        quotes.time.between('09:30:00', '15:59:59'),
        quotes.ofr > quotes.bid,
        quotes.bid > 0,
        quotes.ofr / quotes.bid < 1.2
    ]
    tmp.set_index('time', inplace=True)
    avg_spread = ((tmp.ofr-tmp.bid) / (tmp.ofr+tmp.bid) * 2).groupby(tmp.index.hourminute, lazy=True).mean()
    avg_spread.plot()
    plt.show()

    # calculate the average bid-ask spread of all stocks on 2007.09.25
    tmp = quotes[
        quotes.date == '2007.09.25',
        quotes.time.between('09:30:00', '15:59:59'),
        quotes.ofr > quotes.bid,
        quotes.bid > 0,
        quotes.ofr / quotes.bid < 1.2
    ]
    tmp.set_index('time', inplace=True)
    avg_spread = ((tmp.ofr-tmp.bid) / (tmp.ofr+tmp.bid) * 2).groupby(tmp.index.hourminute, lazy=True).mean()
    avg_spread.plot()
    plt.show()

    tmp = quotes[
        quotes.date.between('2007.09.25', '2007.09.27'),
        quotes.time.between('09:30:00', '15:59:59'),
        quotes.ofr > quotes.bid,
        quotes.bid > 0,
        quotes.ofr / quotes.bid < 1.2
    ]
    tmp.set_index('time', inplace=True)
    avg_spread = ((tmp.ofr-tmp.bid) / (tmp.ofr+tmp.bid) * 2).groupby(tmp.index.hourminute, lazy=True).mean()
    avg_spread.plot()
    plt.show()

    tmp = quotes[
        quotes.date.between('2007.09.25', '2007.09.27'),
        quotes.time.between('09:30:00', '15:59:59'),
        quotes.ofr > quotes.bid,
        quotes.bid > 0,
        quotes.ofr / quotes.bid < 1.2
    ]
    tmp.set_index('time', inplace=True)
    quote_size = (tmp.bidsiz + tmp.ofrsiz).groupby(tmp.index.hourminute, lazy=True).mean()
    quote_size.plot()
    plt.show()

    # Trades
    trades = orca.read_table('dfs://TAQ', 'trades')
    tmp = trades[
        trades.date.between('2007.09.25', '2007.09.27'),
        trades.time.between('09:30:00', '15:59:59')
    ]
    volumn = tmp.groupby(tmp.time.hourminute, lazy=True)['size'].sum()
    volumn.plot()
    plt.show()

    # EQY
    trade = orca.read_table('dfs://EQY', 'trade')
    nbbo = orca.read_table('dfs://EQY', 'nbbo')

    tmp = nbbo[
        nbbo.time.dt.time.between('09:30:00', '15:59:59'),
        nbbo.offer_price > nbbo.bid_price,
        nbbo.bid_price > 0,
        nbbo.offer_price / nbbo.bid_price < 1.2,
    ]
    tmp.set_index('time', inplace=True)
    avg_spread = ((tmp.offer_price-tmp.bid_price) / (tmp.offer_price+tmp.bid_price) * 2).groupby(tmp.index.hourminute, lazy=True).mean()
    avg_spread.plot()
    plt.show()

    quote_size = (tmp.offer_size+tmp.bid_size).groupby(tmp.index.hourminute, lazy=True).mean()
    quote_size.plot()
    plt.show()

    tmp = trade[trade.time.dt.time.between('09:30:00', '15:59:59')]
    tmp.set_index('time', inplace=True)
    volume = tmp.groupby(tmp.index.hourminute, lazy=True)['trade_volume'].sum()
    volume.plot()
    plt.show()

    # Market open
    tmp = quotes[
        quotes.date.between('2007.09.25', '2007.09.27'),
        quotes.bid > 0,
        quotes.bid < quotes.ofr,
        quotes.ofr < quotes.bid * 1.2,
        quotes.time.between('09:30:00', '15:59:59'),
        ~quotes.symbol.isin(['ZVZZT', 'ZXZZT', 'ZWZZT'])
    ]
    tmp.set_index('time', inplace=True)
    price = orca.DataFrame()
    price['price'] = ((tmp.bid+tmp.ofr) / 2).groupby(['symbol', 'date', tmp.index.hourminute], lazy=True).mean()
    price['ret'] = price.price.pct_change().groupby(level='symbol', lazy=True).transform()
    disp = price[price.ret < 1].groupby(level='time', lazy=True)['ret'].std()
    disp.plot(x='time')
    plt.show()

if __name__ == '__main__':
    main()


import pytest
import os
from pathlib import Path
import numpy as np
import openvino.runtime as ov

from nncf.common.quantization.structs import QuantizationPreset
from nncf.experimental.openvino_native.statistics.aggregator import OVStatisticsAggregator
from nncf.quantization.algorithms.min_max.algorithm import MinMaxQuantization
from nncf.quantization.algorithms.min_max.algorithm import MinMaxQuantizationParameters

from tests.openvino.conftest import OPENVINO_NATIVE_TEST_ROOT
from tests.openvino.omz_helpers import convert_model
from tests.openvino.omz_helpers import download_model
from tests.openvino.native.common import get_dataset_for_test
from tests.openvino.native.common import load_json
from tests.openvino.native.common import dump_to_json
from tests.openvino.native.models import SYNTHETIC_MODELS
from tests.openvino.native.models import LinearModel
from tests.openvino.native.models import ConvModel
from tests.openvino.native.models import MatMul2DModel
from tests.openvino.native.models import FPModel

REFERENCE_SCALES_DIR = OPENVINO_NATIVE_TEST_ROOT / 'data' / 'reference_scales'

def get_fq_nodes_stats_algo(model):
    nodes = {}
    for op in model.get_ops():
        if op.get_type_name() == 'FakeQuantize':
            input_low = op.input_value(1).get_node().get_data()
            input_high = op.input_value(2).get_node().get_data()
            output_low = op.input_value(3).get_node().get_data()
            output_high = op.input_value(4).get_node().get_data()

            nodes[op.get_friendly_name()] = {
                    'input_low': input_low,
                    'input_high': input_high,
                    'output_low': output_low,
                    'output_high': output_high
            }
    return nodes

def compare_stats(expected, actual):
    assert len(expected) == len(actual)
    for ref_name in expected:
        ref_stats = expected[ref_name]
        ref_input_low, ref_input_high = ref_stats['input_low'], ref_stats['input_high']
        ref_output_low, ref_output_high = ref_stats['output_low'], ref_stats['output_high']

        stats = actual[ref_name]
        input_low, input_high = stats['input_low'], stats['input_high']
        output_low, output_high = stats['output_low'], stats['output_high']

        assert np.allclose(ref_input_low, input_low, atol=1e-6)
        assert np.allclose(ref_input_high, input_high, atol=1e-6)
        assert np.allclose(ref_output_low, output_low, atol=1e-6)
        assert np.allclose(ref_output_high, output_high, atol=1e-6)

# pylint: disable=protected-access
def quantize_model(ov_model, preset):
    dataset = get_dataset_for_test(ov_model)

    min_max_algo = MinMaxQuantization(MinMaxQuantizationParameters(number_samples=1, preset=preset))
    statistics_aggregator = OVStatisticsAggregator(dataset)
    statistic_points = min_max_algo.get_statistic_points(ov_model)
    statistics_aggregator.register_stastistic_points(statistic_points)
    statistics_aggregator.collect_statistics(ov_model)
    quantized_model = min_max_algo._apply(ov_model, statistics_aggregator.statistic_points)
    return quantized_model

@pytest.mark.parametrize('preset', [QuantizationPreset.PERFORMANCE, QuantizationPreset.MIXED],
                         ids=[QuantizationPreset.PERFORMANCE.value, QuantizationPreset.MIXED.value])
@pytest.mark.parametrize('model_creator_func', SYNTHETIC_MODELS.values())
def test_syntetic_models_fq_scales(model_creator_func, preset):
    model = model_creator_func()
    quantized_model = quantize_model(model.ov_model, preset)
    nodes = get_fq_nodes_stats_algo(quantized_model)

    ref_stats_name = model.ref_graph_name.split(".")[0] + f'_{preset.value}.json'
    ref_stats_path = REFERENCE_SCALES_DIR / ref_stats_name
    if not os.path.exists(ref_stats_path):
        dump_to_json(ref_stats_path, nodes)
    ref_nodes = load_json(ref_stats_path)

    compare_stats(ref_nodes, nodes)

OMZ_MODELS = [
    'mobilenet-v2-pytorch',
    'resnet-18-pytorch',
    'yolo-v4-tiny-tf',
]

@pytest.mark.skip(reason='Ticket 100948')
@pytest.mark.parametrize('preset', [QuantizationPreset.PERFORMANCE, QuantizationPreset.MIXED],
                         ids=[QuantizationPreset.PERFORMANCE.value, QuantizationPreset.MIXED.value])
@pytest.mark.parametrize('model_name', OMZ_MODELS)
def test_omz_models_fq_scales(model_name, preset, tmp_path):
    _ = download_model(model_name, tmp_path)
    model_path = convert_model(model_name, tmp_path)
    model = ov.Core().read_model(model_path)
    quantized_model = quantize_model(model, preset)
    nodes = get_fq_nodes_stats_algo(quantized_model)

    ref_stats_name = str(Path(model_path).name).rsplit('.', maxsplit=1)[0] + f'_{preset.value}.json'
    ref_stats_path = REFERENCE_SCALES_DIR / ref_stats_name
    if not os.path.exists(ref_stats_path):
        dump_to_json(ref_stats_path, nodes)
    ref_nodes = load_json(ref_stats_path)

    compare_stats(ref_nodes, nodes)

REF_NODES_SHAPES = {
    'LinearModel': {'Input/fq_output_0': (), 'MatMul/fq_weights_1': (1, 1, 1, 1)},
    'ConvModel': {'Conv/fq_weights_1': (3, 1, 1, 1), 'Sub/fq_output_0': ()},
    'MatMul2DModel': {'Input/fq_output_0': (), 'MatMul/fq_weights_1': (5, 1)},
}

@pytest.mark.parametrize('model_creator_func, ref_shapes',
                         zip([LinearModel, ConvModel, MatMul2DModel], REF_NODES_SHAPES.values()))
def test_syntetic_models_fq_shapes(model_creator_func, ref_shapes):
    model = model_creator_func()
    quantized_model = quantize_model(model.ov_model, QuantizationPreset.PERFORMANCE)
    nodes = get_fq_nodes_stats_algo(quantized_model)
    for node_name, node in nodes.items():
        assert node['input_low'].shape == ref_shapes[node_name]
        assert node['input_high'].shape == ref_shapes[node_name]
        assert node['output_low'].shape == ref_shapes[node_name]
        assert node['output_high'].shape == ref_shapes[node_name]

@pytest.mark.parametrize('precision', ['FP16', 'FP32'])
def test_syntetic_models_fq_precision(precision):
    model = FPModel(precision)
    quantized_model = quantize_model(model.ov_model, QuantizationPreset.PERFORMANCE)
    dtype = ov.Type(np.float32) if precision == 'FP32' else ov.Type(np.float16)
    for op in quantized_model.get_ops():
        if op.get_type_name() == 'Constant':
            if op.get_element_type().is_real():
                assert op.get_element_type() == dtype


from collections import Counter
from typing import Iterable, List
import numpy as np
import pytest

from decisionai import dataset_excerpt_helpers, constants
from decisionai.simulation import Simulation
from decisionai.variable import Variable
from decisionai.dataset import Dataset
from decisionai.policy import PolicyDefinition

def pols(**kwargs) -> List[PolicyDefinition]:
    """
    kwargs should be structured such that keys are the names of attributes
    and values are lists of strings, each representing a formula.
    Returns a list of PolicyDefinition dicts. Policies will be named "p1, "p2",
    etc.
    """
    pols = []
    npols = None
    for attr, formulas in kwargs.items():
        if npols is None:
            npols = len(formulas)
            for i in range(1, 1+npols):
                pols.append( {'policy_name': f'p{i}'} )
        assert len(formulas) == npols
        for i, formula in enumerate(formulas):
            pols[i][attr] = formula
    return pols

class TestSim(Simulation):
    """A wrapper around Simulation with some additional methods useful for testing.
    """

    def dataset_df(self, label=None):
        # NB: because we're using the excerpt code path, there will be no
        # simId column, and we'll only have data for one sim.
        if label is None:
            assert len(self.datasets) == 1, "Explicit label must be provided if n datasets != 1"
            label = list(self.datasets.keys())[0]
        df_dict = dataset_excerpt_helpers.dataset_var_values_excerpt_dfs(self, [label])
        return df_dict[label]

    def assert_values_match_across_time(self, varname, expected, **kwargs):
        assert 't' not in kwargs
        if '.' in varname:
            # For dataset variables, we'll be using the excerpt df, which has only
            # a limited set of timesteps
            steps = range(
                    constants.EXCERPT_START,
                    min(constants.EXCERPT_END, self.num_steps)+1
            )
        else:
            steps = range(self.num_steps)
        for t in steps:
            kwargs['t'] = t
            self.assert_values_match(varname, expected, **kwargs)

    def get_values(self, varname,
            policy=None, sim_id=None,
            t=None,
            row=None,
    ):
        if '.' in varname:
            assert sim_id is None, ("sim_id filtering not supported for dataset"
                    " variables (we always use sim 0 due to using excerpt data)")
            label, varname = varname.split('.')
            df = self.dataset_df(label)
        else:
            df = self.results
        if policy is not None:
            df = df[df.policyId == policy]
        if sim_id is not None:
            df = df[df.simId == sim_id]
        if t is not None:
            df = df[df.t == t]
        if row is not None:
            df = df[df.rowId == row]
        if t is None:
            sortkey = 't'
        elif row is None:
            sortkey='rowId'
        else:
            sortkey = ['t', 'rowId']
        return df.sort_values(by=sortkey).loc[:,varname].values

    def assert_values_match(self, varname, expected, **kwargs):
        """Asserts that the flattened values for varname (when sorted by t then
        rowId) match the given expected value(s). Notes:
        - expected may be a list, numpy array, or a scalar. We do a broadcasting comparison.
        - if varname is the name of a dataset variable, then we will compare against
          our *excerpt* data, which ranges from timesteps 1-5 only, and only
          corresponds to a single simulation. (This is not ideal, but is arguably the
          least bad option. Alternative would be passing in some flag to the simulation
          for testing telling it to store data for all timesteps in its excerpt, or to
          use some drop-in replacement for DatasetVariableValueHolder that remembers
          everything. But then we're testing something substantially different from
          the 'real' code. But at least, should maybe consider having a differently
          named method like assert_dataset_excerpt_values_match, for clarity.)
        """
        # NB: expected may be a scalar, in which case we broadcast
        vals = self.get_values(varname, **kwargs)
        np.testing.assert_allclose(vals, expected)

    def assert_null_values(self, varname):
        assert self.results[varname].isnull().all()

    def errors(self):
        return (
            [err for var in self.all_basevariables for err in var.errors]
            + self.extra_errors
        )

    def assert_no_errors(self):
        __tracebackhide__ = True
        errs = self.errors()
        if len(errs) > 0:
            pytest.fail(f"Expected no errors. Got {len(errs)}: {errs}")

    def assert_errors_match(self, patterns: List[List[str]]):
        """patterns should be a list of length 2 or 3 lists. The first two elements
        are prefixes of an expected Error (i.e. variable name, and error type). The third
        element, if present, is an expected substring of the third element of the Error
        list, i.e. the message.
        """
        # TODO: should give nicer messages on assertion failures
        __tracebackhide__ = True
        errs = self.errors()
        if len(patterns) != len(errs):
            pytest.fail(f"Expected {len(patterns)} errors, but got {len(errs)}."
                    f" Errors were: {errs}")
        # NB: Assuming (for now) that [name, type] prefix is unique
        pre_map = {tuple(err[:2]) : err for err in errs}
        pat_map = {tuple(pat[:2]) : pat for pat in patterns}
        if pre_map.keys() != pat_map.keys():
            pytest.fail(f"Expected error signatures to match {pat_map.keys()!r}. Were: {pre_map.keys()!r}")
        for pat in patterns:
            if len(pat) < 3:
                continue
            name, errtype, substr = pat
            matched_msg = pre_map[(name, errtype)][2]
            if not substr in matched_msg:
                pytest.fail(f"Error message was {matched_msg!r}. Failed to match "
                        f"substring {substr!r}")

    def assert_errors_include(self, patterns):
        errs = self.errors()
        assert all(len(pat) == 2 for pat in patterns), "Other pattern lengths not implemented"
        patterns = map(tuple, patterns)
        prefixes = [tuple(err[:2]) for err in errs]
        for pat in patterns:
            assert pat in prefixes

def run_sim(
    variables: Iterable[Variable] = None,
    policies: Iterable[PolicyDefinition] = None,
    datasets: Iterable[Dataset] = None,
    models = None,
    num_steps: int = 3,
    num_sims: int = 1,
    allow_errs=False,
    # convenience params for the common case where there's only one dataset/model
    dataset : Dataset = None,
    model = None,
):
    variables = variables or []
    policies = policies or []
    datasets = datasets or []
    if dataset:
        assert not datasets, "Params datasets and dataset are mutually exclusive"
        datasets = [dataset]
    models = models or []
    if model:
        assert not models
        models = [model]
    sim = TestSim(variables, policies, datasets, models, num_steps, num_sims)
    sim.run()
    if not allow_errs:
        sim.assert_no_errors()
    return sim


# -*- coding: utf-8 -*-

# allow the notebook to access the parent directory so we can import the other modules
# https://stackoverflow.com/a/35273613
import os
import sys
nb_dir = os.path.split(os.getcwd())[0]
if nb_dir not in sys.path:
    sys.path.append(nb_dir)

dataset_folder_path = os.path.join("files", "dataset")

#%%
from data.DataSet import DataSet
dataset = DataSet()
dataset.load(dataset_folder_path, test_set_percentage=0.2, validation_set_percentage=0.3333)

print("Training Data Len:", len(dataset.train_data))
print("Validation Data Len:", len(dataset.valid_data))
print("Test Data Len:", len(dataset.test_data))

#%%
import numpy as np
from utils.preprocessing import *
from functools import partial
from utils.research_preprocessing import add_occlusions
from models.regularized_3x512_gru import Regularized3x512GRU
import os.path
import pickle

dataset.apply(apply_mean_centering)
dataset.apply(apply_unit_distance_normalization)

ANGLES_TO_ROTATE = [5, 10, 15, 45, -5, -10, -15, -45]

NUM_EPOCHS = 20
PARAM_BATCH_SIZE = 300

#NUM_SAMPLES_TO_TRY =  [300, 200, 150, 100, 75, 50, 25, 10]
NUM_SAMPLES_TO_TRY =  [350, 250, 75]

scores_valid = []
scores_test = []

for num_samples in NUM_SAMPLES_TO_TRY:
    curr = dataset.copy()
    curr.apply(partial(spline_interpolate_and_resample, num_samples=num_samples))
    curr.expand(reverse_digit_sequence)
    curr.expand_many(partial(rotate_digit, degrees=ANGLES_TO_ROTATE))

    X_train = np.array(curr.train_data)
    X_valid = np.array(curr.valid_data)
    X_test = np.array(curr.test_data)
    # Convert labels to numpy array and OneHot encode them
    encoder, Y_train, Y_valid, Y_test = curr.onehot_encode_labels()
    # evaluate

    mymodel = Regularized3x512GRU(X_train.shape[1:])
    mymodel.batch_size = PARAM_BATCH_SIZE
    mymodel.num_epochs = NUM_EPOCHS
    mymodel.disable_callbacks()
    mymodel.initialize()
    mymodel.train(X_train, Y_train, X_valid, Y_valid)

    score_valid = mymodel.model.evaluate(X_valid, Y_valid)[1] * 100.0
    score_test = mymodel.model.evaluate(X_test, Y_test)[1] * 100.0
    print("num samples: %d,   valid acc: %.3f%%,   test acc: %.3f%%" % (num_samples, score_valid, score_test))
    scores_valid.append(score_valid)
    scores_test.append(score_test)

    with open(os.path.join("files", "pickles", "Q_spline_num_samples_2.pkl"), 'wb') as fd:
        pickle.dump([scores_valid, scores_test], fd)

#%%
import matplotlib.ticker as mtick
import matplotlib.pyplot as plt

fig = plt.figure(1, (7,4))
ax = fig.add_subplot(1,1,1)
ax.plot(NUM_SAMPLES_TO_TRY, scores_valid, c='blue', label="Validation Set")
ax.scatter(NUM_SAMPLES_TO_TRY, scores_valid, c='blue')

ax.plot(NUM_SAMPLES_TO_TRY, scores_test, c='red', label="Test Set")
ax.scatter(NUM_SAMPLES_TO_TRY, scores_test, c='red')

fmt = '%.0f%%' # Format you want the ticks, e.g. '40%'
xticks = mtick.FormatStrFormatter(fmt)
ax.yaxis.set_major_formatter(xticks)
plt.xlabel("Number of Frames to Sample from Spline")
plt.ylabel("Prediction Accuracy")
plt.legend(loc='lower right')
#plt.legend()


"""
    @Difficulty : M
    @Status     : AC
    @Time       : 2020/10/18 10:35
    @Author     : Chen Runwen
"""
import math

class Solution:
    def findLexSmallestString(self, s: str, a: int, b: int) -> str:
        """ 找到每一种轮转，计算这种轮转下的最小字典序，然后比较 """
        n = len(s)
        res = s
        can = b % 2 != 0  # 如果b为偶数则永远只能调整奇数位
        nums = [ord(c) - 48 for c in s]

        # 所有轮转
        for i in range(n // math.gcd(n, b)):
            nums = nums[-b:] + nums[:-b]

            odd = nums[1] % math.gcd(10, a) - nums[1]
            for j in range(1, n, 2):
                nums[j] = (nums[j] + odd) % 10

            if can:
                even = nums[0] % math.gcd(10, a) - nums[0]
                for j in range(0, n, 2):
                    nums[j] = (nums[j] + even) % 10

            t = ''.join([str(c) for c in nums])
            res = min(res, t)

        return res

if __name__ == '__main__':
    s = "5525"
    a = 9
    b = 2
    print(Solution().findLexSmallestString(s, a, b))

contests/20201018/5544. 执行操作后字典序最小的字符串.py

5544. 执行操作后字典序最小的字符串.py


import h5py
import numpy as np
from friday_harbor.structure import Ontology
import utilities
import numpy.testing as nptest

class LinearModel(object):
    def __init__(self, W, col_labels, row_labels, data_dir='.', P=[]):
        self.data_dir=data_dir
        self.W = W
        self.col_labels = list(col_labels)
        self.row_labels = list(row_labels)
        self.P = P
        self.ontology=Ontology(data_dir=data_dir)

    def export_to_dictionary(self):
        return {'W':self.W,
                'col_labels':self.col_labels,
                'row_labels':self.row_labels,
                'P':self.P,
                'data_dir':self.data_dir}

    def save_to_hdf5(self, file_name):
        f = h5py.File(file_name, 'w')
        utilities.write_dictionary_to_group(f, self.export_to_dictionary())
        f.close()

    @staticmethod
    def load_from_hdf5(file_name):
        f = h5py.File(file_name, 'r')
        D = utilities.read_dictionary_from_group(f)
        f.close()
        if 'P' in D.keys():
            return LinearModel(D['W'], D['col_labels'], D['row_labels'],
                               data_dir=D['data_dir'], P=D['P'])
        else:
            return LinearModel(D['W'], D['col_labels'], D['row_labels'],
                               data_dir=D['data_dir'])

    def get_w_val(self, row_val, col_val):
        if isinstance(row_val, str):
            row_val = self.ontology.acronym_id_dict[row_val]
        if isinstance(col_val, str):
            col_val = self.ontology.acronym_id_dict[col_val]
        col_ind = self.col_labels.index(col_val)
        row_ind = self.row_labels.index(row_val)

        return self.W[row_ind, col_ind]

    def get_p_val(self, row_val, col_val):
        if isinstance(row_val, str):
            row_val = self.ontology.acronym_id_dict[row_val]
        if isinstance(col_val, str):
            col_val = self.ontology.acronym_id_dict[col_val]
        col_ind = self.col_labels.index(col_val)
        row_ind = self.row_labels.index(row_val)
        return self.P[row_ind, col_ind]

    def run_regression(self, A, B, col_labels, row_labels,
                       default_p_value=np.Inf):
        import statsmodels.api as sm

        if self.P != []:
            raise Exception
        else:
            double_check_w_dict = {}
            p_value_dict = {}
            for jj, col_and_curr_col_label in enumerate(zip(B.T, col_labels)):
                col, curr_col_label = col_and_curr_col_label
                b = col.T

                # For curr col, determine which rows should be kept for
                # regression
                nonzero_ind_list = []
                nonzero_label_list = []
                for ii, curr_row_label in enumerate(row_labels):
                    if self.get_w_val(curr_row_label, curr_col_label) != 0.0:
                        nonzero_ind_list.append(ii)
                        nonzero_label_list.append(curr_row_label)
                res = sm.OLS(b,A[:,nonzero_ind_list]).fit()

                # Create double-check dictionary:
                for kk, curr_w in enumerate(res.params):
                    double_check_w_dict[nonzero_label_list[kk], curr_col_label] = curr_w

                # Create p-value dictionary:
                for kk, curr_p in enumerate(res.pvalues):
                    p_value_dict[nonzero_label_list[kk], curr_col_label] = curr_p

            # Assign p-values to matrix:
            P = np.zeros((np.shape(A)[1], np.shape(B)[1]))
            for ii, curr_row_label in enumerate(self.row_labels):
                for jj, curr_col_label in enumerate(self.col_labels):
                    if (curr_row_label, curr_col_label) in p_value_dict.keys():
                        P[ii, jj] = p_value_dict[curr_row_label, curr_col_label]
                    else:
                        P[ii, jj] = default_p_value
                    # Double-check coefficient:
                    if self.get_w_val(curr_row_label, curr_col_label) != 0.0:
                        w_opt = self.get_w_val(curr_row_label, curr_col_label)
                        w_reg = double_check_w_dict[curr_row_label,
                                                    curr_col_label]
                        nptest.assert_almost_equal(w_opt, w_reg, 7, err_msg='Optimization does not match regression')
            self.P = P
#
#     # Store results in a dictionary:
#     STD_dict = {}
#     P_dict = {}
#     for ii, from_id in enumerate(from_id_list_regression):
#         STD_dict[from_id] = res.bse[ii]
#         P_dict[from_id] = res.pvalues[ii]
#
#     # For debugging, to ensure that regression discovers same value as optimization:
#     for ii, from_id in enumerate(from_id_list_regression):
#         abs_error = np.abs((res.params[ii]- optimization_dict[from_id][0]))
#         if abs_error > .00001:
#             print 'WARNING: %s, %s' % (res.params[ii], optimization_dict[from_id][0])
#
#     # Store away final values in matrix:
#     fill_val = np.Inf
#     target_ind = ind_acronym_dict[curr_target_node.acronym]
#     for source_id in from_id_list:
#         source_ind = ind_acronym_dict[id_acronym_dict[source_id]]
#
#         if source_id in from_id_list_regression:
#             curr_W_value = optimization_dict[source_id][0]
#             curr_P_value = P_dict[source_id]
#             curr_STD_value = STD_dict[source_id]
#         else:
#             curr_W_value = fill_val
#             curr_P_value = fill_val
#             curr_STD_value = fill_val
#
#         W[source_ind, target_ind] = curr_W_value
#         P[source_ind, target_ind] = curr_P_value
#         STD[source_ind, target_ind] = curr_STD_value

class OldLinearModel(object):

    def __init__(self, W, col_labels, row_labels, P=[]):

        self.W = W
        self.col_labels = list(col_labels)
        self.row_labels = list(row_labels)
        self.P = P

    @staticmethod
    def load_from_hdf5(file_name):
        import h5py
        import utilities
        f = h5py.File(file_name, 'r')
        D = utilities.read_dictionary_from_group(f)
        f.close()
        if 'P' in D.keys():
            return OldLinearModel(D['W'], D['col_labels'], D['row_labels'],
                                  P=D['P'])
        else:
            return OldLinearModel(D['W'], D['col_labels'], D['row_labels'])


from typing import List, Optional

import yaml
from fastapi import FastAPI, HTTPException, Request, Body
from pydantic import BaseModel, ValidationError
from starlette import status
from starlette.responses import JSONResponse

app = FastAPI()

class Item(BaseModel):
    name: str
    tags: List[str]

@app.post(
    "/items/",
    openapi_extra={
        "requestBody": {
            "content": {"application/x-yaml": {"schema": Item.schema()}},
            "required": True,
        },
    },
)
async def create_item(request: Request):
    raw_body = await request.body()
    print(raw_body)
    try:
        data = yaml.safe_load(raw_body)
    except yaml.YAMLError:
        raise HTTPException(status_code=422, detail="Invalid YAML")
    try:
        item = Item.parse_obj(data)
        print(item)
    except ValidationError as e:
        raise HTTPException(status_code=422, detail=e.errors())
    return item

items = {"foo": {"name": "Fighters", "size": 6}, "bar": {"name": "Tenders", "size": 3}}

@app.put("/items/{item_id}")
async def upsert_item(
    item_id: str, name: Optional[str] = Body(None), size: Optional[int] = Body(None)
):
    if item_id in items:
        item = items[item_id]
        item["name"] = name
        item["size"] = size
        return item
    else:
        item = {"name": name, "size": size}
        items[item_id] = item
        return JSONResponse(status_code=status.HTTP_201_CREATED, content=item)


from tools.translate import _
from osv import osv, fields

class account_journal_entries_report(osv.osv_memory):
    _name = "account.journal.entries.report"
    _description = "Print journal by entries"
    _columns = {
        'journal_ids': fields.many2many('account.journal', 'account_journal_entries_journal_rel', 'acc_journal_entries_id','journal_id','Journal', required=True),
        'period_ids': fields.many2many('account.period','account_journal_entries_account_period_rel','acc_journal_entries_id','account_period_id','Period'),
        'sort_selection': fields.selection([
                   ('date','By date'),
                   ("to_number(name,'999999999')",'By entry number'),
                   ('ref','By reference number')],'Entries Sorted By', required=True),
        'landscape': fields.boolean('Landscape mode')
    }
    def default_get(self, cr, uid, fields, context=None):
        if context is None: context = {}
        res = {}
        res['sort_selection'] = 'date'
        res['journal_ids'] = self.pool.get('account.journal').search(cr,uid, [])
        res['period_ids'] = self.pool.get('account.period').search(cr, uid, [('fiscalyear_id', '=', self.pool.get('account.fiscalyear').find(cr, uid))])
        return res

    def _check_data(self, cr, uid, ids, context=None):
        if ids:
            current_obj = self.browse(cr, uid, ids[0], context=context)
            if current_obj.period_ids and current_obj.journal_ids:
                period_id =  current_obj.period_ids[0].id
                journal_id = current_obj.journal_ids[0].id
                if type(period_id)==type([]):
                    ids_final = []
                    for journal in journal_id:
                        for period in period_id:
                            ids_journal_period = self.pool.get('account.journal.period').search(cr,uid, [('journal_id','=',journal),('period_id','=',period)])

                            if ids_journal_period:
                                ids_final.append(ids_journal_period)

                    if not ids_final:
                        raise osv.except_osv(_('No Data Available'), _('No records found for your selection!'))
        return True

    def _check(self, cr, uid, ids, context=None):
         if ids:
            current_obj = self.browse(cr, uid, ids[0], context=context)
            if current_obj.landscape==True:
                return 'report_landscape'
            else:
                return 'report'

    def print_report(self, cr, uid, ids, context=None):
        """prints report"""
        if context is None:
            context = {}

        data = self.read(cr, uid, ids)[0]
        datas = {
             'ids': context.get('active_ids',[]),
             'model': 'ir.ui.menu',
             'form': data
        }
        if self._check_data(cr, uid, ids, context=context):
            if self._check(cr, uid, ids, context=context) == 'report_landscape':
                return {
                    'type': 'ir.actions.report.xml',
                    'report_name': 'account.journal.entries.report.wzd1',
                    'datas': datas
                    }
            else:
                return {
                    'type': 'ir.actions.report.xml',
                    'report_name': 'account.journal.entries.report.wzd',
                    'datas': datas
                    }

account_journal_entries_report()
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:


# Project Description:
# --------------------
# This Program will be able to display information about
# Flights leaving from Providence to Orlando the user will
# be able to get information such as what specific flight
# they want, finding the cheapest flight, and being able
# find the average price of tickets for all flights and more

# Function that allows the user to type in a file they would like to use & correct the user if the file is Invalid
def openFile():
    goodFile = False
    while goodFile == False:
        fName = input("Please enter a file name: ")
        try:
            gradFile = open(fName,"r")
            goodFile = True
        except IOError:
            print("Invalid file name try again... ")
    return gradFile

# get the data from the csv file and sorts all data in the correct list
def getData():
    airlineSchedule = openFile()
    airlineList = []
    flightNumberList = []
    departureTimeList = []
    arrivalTimeList = []
    priceList = []
    for line in airlineSchedule:
        line = line.strip()
        airline, flightNumber, departure, arrival,price = line.split(",")
        airlineList.append(airline)
        flightNumberList.append(flightNumber)
        departureTimeList.append(departure)
        arrivalTimeList.append(arrival)
        priceList.append(price)
    airlineSchedule.close()
    return airlineList, flightNumberList, departureTimeList,arrivalTimeList, priceList

    #This function displays the menu of choices for the user
    #It reads in the user's choice and returns it as an integer
def getChoice():
    print("")
    print("Please choose one of the following options:")
    print("1 -- Find flight information by airline and flight number")
    print("2 -- Find flights shorter than a specified duration")
    print("3 -- Find the cheapest flight by a given airline")
    print("4 -- Find flight departing after a specified time")
    print("5 -- Find the average price of all flights")
    print("6 -- Write a file with flights sorted by departure time")
    print("7 -- Quit")
    OK = False
    while OK == False:
        try:
            choice = int(input("Choice ==> "))
            if choice >= 0 and choice <= 7:
                OK = True
            else:
                print("Entry must be between 1 and 7")
        except ValueError:
            print("Entry must be a number")
    print("")
    return choice

#Informs the user the flight number picked is not a valid option if the flight number does not exist
def exception(flightNumberList):
   OK = False
   while OK == False:
       try:
           flightNumber = input("Enter flight number: ")
           if flightNumber in flightNumberList:
                OK = True
           else:
                print("Invalid input -- try again")
       except ValueError:
           print("Invalid entry, try again... ")
   return flightNumber

#Informs the user the airline picked is not a valid option if the airline does not exist
def exceptionAirline(airlineList):
   OK = False
   while OK == False:
       try:
           airlineName = input("Enter airline name: ")
           if airlineName in airlineList:
                OK = True
           else:
                print("Invalid input -- try again")
       except ValueError:
           print("Invalid entry, try again... ")
   return airlineName

#Finds the specifc flight a user is searching for using the airline and flight numbers
def findSpecificFlight(airlineList,flightNumberList):
    airlineName = exceptionAirline(airlineList)
    flightNumber = exception(flightNumberList)
    #searches for the flight number and airline the user is searching for and then returns the index of that flight
    for i in range(len(flightNumberList)):
        if flightNumberList[i] == flightNumber and airlineList[i] == airlineName:
            index = i
            return index
    return -1

#This functions tells the user the entry must be a number in the case a letter is used by accident
def exceptionSpecifedFlight():
   OK = False
   while OK == False:
       try:
           maxDur = int(input("Enter maximum duration (in minutes): "))
           if maxDur != "" :
                OK = True
       except ValueError:
           print("Entry must be a number")
   return maxDur

#finds shorter than specifed flights
def shorterThanSpecified(arrivalTimeList,departureTimeList):
    flightIndexList = []
    durationList = []
    arrivalInMin = []
    departInMin = []
    maxDur = exceptionSpecifedFlight()
    #Converts the time in the arrival time list from military time to minutes and adds it to a new list that has all of the arival time in minutes
    for i in arrivalTimeList:
        hour, minute = i.split(":")
        hourToMin = int(hour) * 60
        timeInMinutes = int(hourToMin) + int(minute)
        arrivalInMin.append(timeInMinutes)

    #Converts the time in the departure time list from military time to minutes and adds it to a new list that has all of the departure time in minutes
    for i in departureTimeList:
        hour, minute = i.split(":")
        hourToMin = int(hour) * 60
        timeInMinutes = int(hourToMin) + int(minute)
        departInMin.append(timeInMinutes)

    #gets the total duration of each flight and adds it a list
    for i in range(len(arrivalTimeList)):
       duration = int(arrivalInMin[i]) - int(departInMin[i])
       durationList.append(duration)

    #finds the index of each flight that is less than or equal to the user input then adds it to the flight index list
    for i in range(len(durationList)):
        if durationList[i] <= maxDur:
            flightIndexList.append(i)

    return flightIndexList

#This functions finds the cheapest flight in a specific airline
def theCheapestFlight(airlineList,priceList):
    newPriceList = []
    airlinePriceindex = []
    airlineName = exceptionAirline(airlineList)

    #gets the index of all the flights of a specific airline the user is requesting for and adds it to a list
    for i in range(len(airlineList)):
        if airlineList[i] == airlineName:
            airlinePriceindex.append(i)

    #takes all the prices in price list and converts it to a integer, then adds it to a list with all the prices being a tnteger
    for i in priceList:
        dollarSign, price = i.split("$")
        intPrice = int(price)
        newPriceList.append(intPrice)

    lowestPrice = newPriceList[28]
    #searches through the list of indexes of the specfic flights, Then uses those index to search through the new price list to obtain the lowest price
    for i in airlinePriceindex:
        if newPriceList[i] < lowestPrice:
           lowestPrice = newPriceList[i]
           index = i
    return index

#Finds the flights that depart after a specific time
def flightDepartAfterSpecTime(arrivalTimeList,departureTimeList):
    departInMin = []
    departFlightIndexList = []
    specTime = input("Enter earliest departure time: ")

    #Checks to see if the input the user is valid
    OK = False
    while OK == False:
        if len(specTime) == 5:
         try:
            hour, minute = specTime.split(":")
            if int(hour) >= 0 and int(hour) <= 24 and int(minute) >= 0 and int(minute) <= 60:
                 OK = True
         except ValueError:
            specTime = input("Invalid time - Try again ")

        if OK == False:
            specTime = input("Invalid time - Try again ")

    #converts the time the user inputted to mintues
    hourToMin = int(hour) * 60
    specTimeInMinutes = int(hourToMin) + int(minute)

   #Converts the time in the departure time list from military time to minutes and adds it to a new list that has all of the departure time in minutes
    for i in departureTimeList:
        hour, minute = i.split(":")
        hourToMin = int(hour) * 60
        timeInMinutes = int(hourToMin) + int(minute)
        departInMin.append(timeInMinutes)

    #compares all the departure time in minutes to the users time, if the departure time is greater than the user time the index of that flight is added to the list
    for i in range(len(departureTimeList)):
        if departInMin[i] > specTimeInMinutes:
             departFlightIndexList.append(i)

    return departFlightIndexList

#Gets the average price for all flights
def flightAveragePrice(priceList):
    newPriceList = []
    sum = 0
    #converts the prices in priceList from strings to Integers and adds it to a new list with prices being a integer
    for i in priceList:
        dollarSign, price = i.split("$")
        intPrice = int(price)
        newPriceList.append(intPrice)

    #summates all the prices in the new price list
    for i in range(len(newPriceList)):
        sum = sum + newPriceList[i]

   #converts the sum of all prices and gets the average, then converts the average price back in to a string
    averagePrice = float(sum) / len(priceList)
    averagePrice = "{:,.2f}".format(averagePrice)
    averagePrice = str(averagePrice)
    averagePrice = '$' + averagePrice

    return averagePrice

def selectionSort(departureTimeList):
    indexList = []
    departWholeNum = []
    for i in range(len(departureTimeList)):
        departTime = departureTimeList[i].replace(":","")
        wholeNum = int(departTime)
        departWholeNum.append(wholeNum)
        indexList.append(i)

    for i in range(0, len(departWholeNum)):
        min = i
        for j in range(i + 1, len(departWholeNum)):
            # comparison
            if  departWholeNum[j] < departWholeNum[min]:
                min = j
        # swap
        departWholeNum[i],  departWholeNum[min] =  departWholeNum[min],  departWholeNum[i]
        indexList[i],  indexList[min] =  indexList[min],  indexList[i]
    return  indexList

def outputFile(airlineList, flightNumberList, departureTimeList,arrivalTimeList, priceList,indexList):
 outFile = open("time-sorted-flights.csv", 'w')
 for i in indexList:
        outFile.write(airlineList[i] + "," + flightNumberList[i] + "," + departureTimeList[i] + "," + arrivalTimeList[i]+","+priceList[i] + '\n')
        i = i + 1
 outFile.close()
 print("Sorted data has been written to file: time-sorted-flights.csv")

 return

def main():

    airlineList, flightNumberList, departureTimeList,arrivalTimeList, priceList =  getData()
    choice = getChoice()
    while choice != 7:

        if choice == 1:
            index = findSpecificFlight(airlineList,flightNumberList)
            if index == -1:
                print("")
                print("No flight meets your criteria")
                choice = getChoice()
            else:
                print("")
                print("The flight that meets your criteria is:")
                print("")
                print("AIRLINE  FLT#     DEPART  ARRIVE PRICE   ")
                print(airlineList[index].ljust(8), flightNumberList[index].ljust(6), departureTimeList[index].rjust(7), arrivalTimeList[index].rjust(7), priceList[index].rjust(3))
                choice = getChoice()

        if choice == 2:
            flightIndexList = shorterThanSpecified(arrivalTimeList,departureTimeList)
            if flightIndexList == []:
                print("")
                print("No flights meet your criteria")
                choice = getChoice()
            else:
                print("")
                print("The flights that meet your criteria are: ")
                print("")
                print("AIRLINE  FLT#     DEPART  ARRIVE PRICE   ")
                for index in flightIndexList:
                 print(airlineList[index].ljust(8), flightNumberList[index].ljust(6), departureTimeList[index].rjust(7), arrivalTimeList[index].rjust(7), priceList[index].rjust(5))
                choice = getChoice()

        if choice == 3:
           index = theCheapestFlight(airlineList,priceList)
           print("")
           print("The flight that meets your criteria is:")
           print("")
           print("AIRLINE  FLT#     DEPART  ARRIVE PRICE   ")
           print(airlineList[index].ljust(8), flightNumberList[index].ljust(6), departureTimeList[index].rjust(7), arrivalTimeList[index].rjust(7), priceList[index].rjust(5))
           choice = getChoice()

        if choice == 4:
            departFlightIndexList = flightDepartAfterSpecTime(arrivalTimeList,departureTimeList)
            if departFlightIndexList == []:
                print("")
                print("No flights meet your criteria")
                choice = getChoice()
            else:
                    print("")
                    print("The flights that meet your criteria are:")
                    print("")
                    print("AIRLINE  FLT#     DEPART  ARRIVE PRICE   ")
                    for index in departFlightIndexList:
                       print(airlineList[index].ljust(8), flightNumberList[index].ljust(6), departureTimeList[index].rjust(7), arrivalTimeList[index].rjust(7), priceList[index].rjust(5))
                    choice = getChoice()

        if choice == 5:
            averagePrice = flightAveragePrice(priceList)
            print("The average price is", averagePrice )
            choice = getChoice()

        if choice == 6:
             indexList = selectionSort(departureTimeList)
             outputFile(airlineList, flightNumberList, departureTimeList,arrivalTimeList, priceList,indexList)
             choice = getChoice()

    if choice == 7:
        print("")
        print("Thank you for flying with us")


# PYHTON 3.7 COLOCAR VERSÃO?

import serial
import matplotlib.pyplot as plt
import matplotlib.animation as animation
from matplotlib import style
import numpy as np
import time
import glob
import argparse
import sys

# Arguments
ap = argparse.ArgumentParser()
ap.add_argument("-s", "--savelog", type=str, required=True,
	help="Want to save the data? True or False")
ap.add_argument("-r", "--refreshrate", type=int, required=True,
	help="Every x milliseconds the data will be acquired")
args = vars(ap.parse_args())

# Works on linux change it to use in Windows | BAUDRATE = 9600
port1 = "/dev/ttyACM0"
port2 = "/dev/ttyUSB0"

if (glob.glob(port1)==[port1]):
	comport = serial.Serial(port1, 9600)
else:
	comport = serial.Serial(port2, 9600)

print ('Using port: {!r}'.format(comport.name))

# Set this from 1.5 to 2.0s
time.sleep(1.8)

# Setup for plotting in Matplotlib
style.use('fivethirtyeight')

# Define a vector to graphic data
xs = []
ys = []

# Variable to count the samples that are being received
count = 0

# Reference voltage
Vref = 5;

# Create figure, add a plot 1x1 and use plot 1
fig = plt.figure()
ax1 = fig.add_subplot(1,1,1)
fig.subplots_adjust(0.16, 0.12, 0.95, 0.9, 0.2, 0.2)
dataFromSerial = 0

# Store what the user gave to us
saveLog = args["savelog"]
refreshRate = args["refreshrate"]

def readFromSerial():
    """ Callback that is called everytime we want to plot a new point, the rate
    that this callback is called is defined by the user. This basically hears the
    serial port. """
	# try send 't' serially, read data from serial, convert to int and fits from 0 to 5V
    try:
        PARAM_CARACTER='t'
        comport.write(PARAM_CARACTER.encode())
        dataFromSerial = int.from_bytes(comport.read(), "big")
        return dataFromSerial*Vref/255
    except(KeyboardInterrupt):
        comport.close()

def plotData(i):
    """ Plot the retrieved data from Arduino via serial communication using matplotlib. """
    global count, ys
    count += 1
    data = readFromSerial()
    # The data must me plotted as float not as a string!
    xs.append(float(count))
    ys.append(float(data))

    # Clean everything before we plot
    ax1.clear()
    ax1.plot(xs,ys)

	# Adding labels and title
    ax1.set_xlabel('milliseconds (ms)')
    ax1.set_ylabel('Volts (V)')
    ax1.set_title('Sensor Acquistion')

def saveData():
    """ Store the retrieved data into a txt file to the local where the script is running. """
    outputFile = "logFromATMega328.txt"
    global xs,ys
    # save data to 2d column, set the decimal precision to 3 floating-point values and ',' as delimiter
    np.savetxt(outputFile, np.column_stack((xs, ys)), fmt='%0.3f', delimiter=',')

def main():
    """ Calls the plotData() constantly """
	# Create the fig, calls plotData every refreshRate milliseconds
    animt = animation.FuncAnimation(fig, plotData, interval=refreshRate)
    plt.show()
    if saveLog=='True':
        saveData()

if __name__ == "__main__":
    try:
        main()
    except(KeyboardInterrupt):
        comport.close()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Aug  1 10:32:35 2020
-----    Time-stamp: <2020-08-02T14:16:35.633361-04:00 hedfp>      -----

@author: (of mods) Carl Schmiedekamp
@Original Source: https://gist.github.com/thomasaarholt/b89b3f29ad82cd32176f2605834fa8ac

WARNING: Does not work in Jupyter Lab!

Inssstructions:
    1. pip install classroom_gizmos
    in notebook:
        from classroom_gizmos.jupyter_file_browser import box, get_file_path
        box    ## creates the file selection ipywidget box.
        ---
        path = get_file_path()  ## every time this cell is executed, the currently
                                ## selected file's path is returned.

Instructions, using source:
    1. paste code in Jupyter notebook cell
    2. execute the cell and select file
    3. in another cell call get_file_path() which returns selected file.

"""

import ipywidgets as widgets
## Ref: https://gist.github.com/thomasaarholt/b89b3f29ad82cd32176f2605834fa8ac

import os
from pathlib import Path

newpath = None ## path to selected file or folder
newfile = None ## full path returned by get_file_path()

cwd = Path(os.getcwd())

FOLDERLABEL = '-------FOLDERS-------'
FILESLABEL = '-------FILES-------'

def get_folder_contents(folder):
    'Gets contents of folder, sorting by folder then files, hiding hidden things'
    folder = Path(folder)
    folders = [item.name for item in folder.iterdir() if item.is_dir() and not item.name.startswith('.')]
    files = [item.name for item in folder.iterdir() if item.is_file() and not item.name.startswith('.')]
    # print( 'get_folder_contents--')
    return [FOLDERLABEL] + sorted(folders) + [FILESLABEL] + sorted(files)

def go_to_address(address):
    address = Path(address)
    if address.is_dir():
        address_field.value = str(address)
        base.value = os.path.basename( str(address))
        select.unobserve(selecting, names='value')
        select.options = get_folder_contents(folder=address)
        select.observe(selecting, names='value')
        select.value = None
    # print( 'go_to_address--')

def newaddress(value):
    go_to_address(address_field.value)
    ##print( 'newaddress--')

def selecting(value):
    global newpath
    if value['new'] and value['new'] not in [FOLDERLABEL, FILESLABEL]:
        path = Path(address_field.value)
        newpath = path / value['new']
        if newpath.is_dir():
            go_to_address(newpath)

        elif newpath.is_file():
            #some other condition
            pass
        # print( 'selecting-- path:{} newpath: {}'.format( path, newpath))
        # print( '  types path: {}, newpath: {}, value[ new]: {}'.format( type( path), type( newpath), type( value['new'])))

def parent(value):
    new = Path(address_field.value).parent
    go_to_address(new)
    # print( 'parent--')

def load(value):
    global newfile
    ##filepath = path / value['new']
    # load using your favourite python software!
    newfile = newpath
    # print( 'load-- newpath: {}, newfile: {}'.format( newpath, newfile))

def get_file_path():
    '''User access to last file selected.'''
    # print( 'getFilepath-- newfile: {}, newpath: {}'.format( newfile, newpath))
    return newfile

address_field = widgets.Text(value=str(cwd))
up_button = widgets.Button(description='Up')
select = widgets.Select(options=get_folder_contents(cwd), rows=15, value=None)
load_button = widgets.Button(description='Select')
base = widgets.Text( str( os.path.basename( cwd)))
box = widgets.HBox([widgets.VBox([address_field, base, select]), widgets.VBox([up_button, load_button])])

select.observe(selecting, names='value')
up_button.on_click(parent)
#load_button.observe(load)
load_button.on_click(load)
address_field.on_submit(newaddress)

##in jupyter execute box
##box


# 7562 나이트의 이동
import sys
from collections import deque
moves = [[2, 1], [1, 2], [-1, 2], [-2, 1], [-2, -1], [-1, -2], [1, -2], [2, -1]]
test_case = int(input())

# 원래는 이 3개 값을 맨 아래 for문에서 받았는데
# 그러니까 값을 다 입력하기도 전에 앞 테스트케이스들의 결과가 출력돼서 따로 받았습니다
i = deque()
start = deque()
end = deque()
for _ in range(test_case):
    i.append(int(input()))
    start.append(list(map(int, sys.stdin.readline().split())))
    end.append(list(map(int, sys.stdin.readline().split())))

# 인접리스트(딕셔너리), 최소경로 저장(딕셔너리), 방문검사(배열)로 해서
# 경우의 수가 많으니까 큐에 값을 넣을 때만 인접리스트를 갱신하고
# not in list 혹은 not in keys()로 검사했더니 오래걸리더라고요... 매번 배열 길이 O(N)만큼 걸림
# 매번 인접리스트를 만드는 것도 시간이 오래 걸리더라고요
# 그래서 구글링해서 참고했습니다...
# visited를 아예 IxI 크기만큼 미리 만들어놓고 검사하는게 not in보다 빨랐어요
# 인접리스트도 따로 안만들고 그때그때 큐에 바로 넣었습니다
# 그때그때 인접리스트 만들 생각은 했으면서 왜 큐에 바로 넣을 생각은 안했지...?
# 최소 step 구하는것도 따로 딕셔너리 만들지 않고 큐에 바로 넣었어요
# (x, y, step) 이런 식으로 큐에 넣어 구현했습니다

for cnt in range(test_case):
    x = start[cnt][0]
    y = start[cnt][1]
    dstX = end[cnt][0]
    dstY = end[cnt][1]

    queue = deque()
    queue.append((x, y, 0))
    visited = [[0 for _ in range(i[cnt])] for _ in range(i[cnt])]

    while len(queue) > 0:
        top = queue.popleft()

        # 목적지에 도착하면 step 프린트 후 break
        if top[0] == dstX and top[1] == dstY:
            print(top[2])
            break

        # 방문한 노드가 아닐 경우
        if visited[top[0]][top[1]] == 0:
            visited[top[0]][top[1]] = 1

            # 나이트 이동 경로를 통해 인접노드 큐에 넣음
            for move in moves:
                nextX = top[0] + move[0]
                nextY = top[1] + move[1]
                if nextX >= 0 and nextX < i[cnt] and nextY >= 0 and nextY < i[cnt]:
                    queue.append((nextX, nextY, top[2]+1))


from collections import deque

FAV = ['I', 'C', 'P', 'C', 'A', 'S', 'I', 'A', 'S', 'G']
DIRECTIONS = [
    [-2, -1], [-1, -2],  # Up-Left
    [2, -1], [1, -2],  # Up-Right
    [-2, 1], [-1, 2],  # Down-Left
    [2, 1], [1, 2]  # Down-Right
]

def bfs(N: int, field: list, start: list) -> bool:
    queue = deque(start)

    while len(queue) > 0:
        cell = queue.popleft()

        for direction in DIRECTIONS:
            x = cell[0] + direction[0]
            y = cell[1] + direction[1]
            next_idx = cell[2] + 1

            if (x < 0) or (x >= N) or (y < 0) or (y >= N):
                continue

            if field[y][x] == FAV[next_idx]:
                if next_idx == 9:  # len(FAV)-1
                    return True

                queue.append([x, y, next_idx])

    return False

def main():
    N = int(input())
    S = list(input())

    field = [[None for _ in range(N)] for _ in range(N)]
    start_cells = []

    idx = 0
    for i in range(N):
        for j in range(N):
            c = S[idx]
            field[i][j] = c
            idx += 1

            if c == FAV[0]:
                start_cells.append([j, i, 0])  # [x, y, fav_idx]

    res = 'YES' if bfs(N, field, start_cells) else 'NO'
    print(res)

if __name__ == '__main__':
    main()


#!/usr/bin/python3

"""7-add_item
This script adds all arguments to a python list and
save them into a specific given file
"""

from sys import argv

save_to_json_file = __import__("5-save_to_json_file").save_to_json_file
load_from_json_file = __import__("6-load_from_json_file").load_from_json_file

filename = "add_item.json"

try:
    arg_list = load_from_json_file(filename)

except FileNotFoundError:
    arg_list = []

for arguments in argv[1:]:
    arg_list.append(arguments)

save_to_json_file(arg_list, filename)


# stocks/urls.py
from django.urls import path, include
from .views import (
    HomePageView,
    SearchResultsListView,
    SectorDetailView,
    CompanyDetailView,
    IndustryDetailView,
)
from django.views.generic import TemplateView
from . import views
from django.views.decorators.cache import cache_page

urlpatterns = [
    path("", HomePageView.as_view(), name="home"),
    path(
        "sector/<slug:slug>/",
        SectorDetailView.as_view(template_name="SectorDetail.html"),
        name="sectordetail",
    ),
    path(
        "industry/<slug:slug>/",
        IndustryDetailView.as_view(template_name="IndustryDetail.html"),
        name="industrydetail",
    ),  # default sorting leverage
    path(
        "company/<slug:pk>/",
        CompanyDetailView.as_view(template_name="CompanyDetail.html"),
        name="companydetail",
    ),
    path("search/", SearchResultsListView.as_view(), name="search_results"),
    path("about/", TemplateView.as_view(template_name="about.html"), name="about"),
]


"""Measures a speed of an HSA robot

This assumes that we have something already running and no serial communication is required.

All we do is write to a csv, and print out the speed of the robot

"""

import cv2
import pyrealsense2 as rs
import numpy as np
import yaml
import csv
import time
import readchar
import threading
from pupil_apriltags import Detector
import argparse
import modern_robotics as mr
from datetime import datetime

parser = argparse.ArgumentParser()
parser.add_argument('filename', help='output file name')
args = parser.parse_args()

filename = args.filename
now = datetime.now().strftime("%y%m%d_%H%M%S")
filename = filename + "_" + now + ".csv"
print(filename)
at_detector = Detector(families='tag36h11',
                       nthreads=1,
                       quad_decimate=1.0,
                       quad_sigma=0.0,
                       refine_edges=1,
                       decode_sharpening=0.25,
                       debug=0)

pipeline = rs.pipeline()
config = rs.config()

# Get device product line for setting a supporting resolution
pipeline_wrapper = rs.pipeline_wrapper(pipeline)
pipeline_profile = config.resolve(pipeline_wrapper)
device = pipeline_profile.get_device()
device_product_line = str(device.get_info(rs.camera_info.product_line))

found_rgb = False
for s in device.sensors:
    if s.get_info(rs.camera_info.name) == 'RGB Camera':
        found_rgb = True
        break
if not found_rgb:
    print("The demo requires Depth camera with Color sensor")
    exit(0)

config.enable_stream(rs.stream.depth, 640, 480, rs.format.z16, 30)

if device_product_line == 'L500':
    config.enable_stream(rs.stream.color, 960, 540, rs.format.bgr8, 30)
else:
    config.enable_stream(rs.stream.color, 640, 480, rs.format.bgr8, 30)

#########################################

# Start streaming
cfg = pipeline.start(config)
profile = cfg.get_stream(rs.stream.color)
intr = profile.as_video_stream_profile().get_intrinsics()
print(intr)
pause_flag = 0
tag_xyz = np.zeros((3,1))

def start_csv(filename):
    n = filename
    with open(n, 'a') as f:
        writer = csv.writer(f)
        t = datetime.now().strftime("%y%m%d_%H%M%S")
        # writer.writerow(['motor1', 'motor2', 'motor3', 'motor4', 'leg_trans_x', 'leg_trans_y', 'leg_trans_z', 'R_0_0', 'R_0_1', 'R_0_2', 'R_1_0', 'R_1_1', 'R_1_2','R_2_0', 'R_2_1', 'R_2_2', t])

def write_csv(filename, R):
    n = filename
    # Verify the ordering of the transform and everything is right
    with open(filename, 'a') as f:
        writer = csv.writer(f)
        t = datetime.now().strftime("%y%m%d_%H%M%S")

        # Could be different ordering of motors
        writer.writerow([0, 0, 0, 0, R[0][3], R[1][3], R[2][3], R[0][0], R[0][1], R[0][2], R[1][0], R[1][1], R[1][2], R[2][0], R[2][1], R[2][2], t])
        f.flush()

# start_csv(filename)

while True:

    frames = pipeline.wait_for_frames()
    depth_frame = frames.get_depth_frame()
    color_frame = frames.get_color_frame()

    if not depth_frame or not color_frame:
        continue

    # Convert images to numpy arrays
    depth_image = np.asanyarray(depth_frame.get_data())
    color_image = np.asanyarray(color_frame.get_data())
    cv2.namedWindow('RealSense', cv2.WINDOW_AUTOSIZE)
    cv2.imshow('RealSense', color_image)
    cv2.waitKey(delay=1)
    gray = cv2.cvtColor(color_image, cv2.COLOR_BGR2GRAY)

    tags = at_detector.detect(gray, estimate_tag_pose=True, camera_params=[intr.fx, intr.fy, intr.ppx, intr.ppy], tag_size=0.055)

    if len(tags) != 0:
        tag_xyz = tags[0].pose_t
        tag_R = tags[0].pose_R
        tag_xyz = np.array(tag_xyz)
        tag_R = np.array(tag_R)
        print(tag_xyz)
        print(tag_R)
        camera_to_tag = mr.RpToTrans(tag_R, tag_xyz)
        print("WRITING TO CSV")
        write_csv(filename, camera_to_tag)


import torch
import torch.nn as nn
import math

__all__ = ['OctaveConv', 'Conv_BN', 'Conv_BN_ACT']

class OctaveConv(nn.Module):
    def __init__(self,
                in_channels,
                out_channels,
                kernel_size,
                alpha_in=0.5,
                alpha_out=0.5,
                stride=1,
                padding=0,
                dilation=1,
                groups=1,
                bias=False):
        super(OctaveConv, self).__init__()
        '''
        H2H, H2L, L2H, L2L octave convolution 연산 수행하는 커널
        args
            in_channels : input channels
            out_channels: output channels
            ...
            alpha_in :    input alpha
            alpha_out:    output alpha
            ...
            groups   :    group conv할때 쓰는거인듯
            ...
        '''
        self.downsample = nn.AvgPool2d(kernel_size=(2,2), stride=2)
        self.upsample   = nn.Upsample(scale_factor=2, mode='nearest')

        # assert (stride == 1 or stride == 2), "stride should be 1 or 2"
        self.stride     = stride
        # groups == in_channels : depthwise convolution
        self.is_dw      = groups == in_channels
        self.alpha_in   = alpha_in
        self.alpha_out  = alpha_out

        self.conv_l2l = None if alpha_in == 0 or alpha_out == 0 else \
                        nn.Conv2d(int(alpha_in * in_channels), int(alpha_out * out_channels),
                                  kernel_size, 1, padding, dilation, math.ceil(alpha_in * groups), bias)
        self.conv_l2h = None if alpha_in == 0 or alpha_out == 1 or self.is_dw else \
                        nn.Conv2d(int(alpha_in * in_channels), out_channels - int(alpha_out * out_channels),
                                  kernel_size, 1, padding, dilation, groups, bias)
        self.conv_h2l = None if alpha_in == 1 or alpha_out == 0 or self.is_dw else \
                        nn.Conv2d(in_channels - int(alpha_in * in_channels), int(alpha_out * out_channels),
                                  kernel_size, 1, padding, dilation, groups, bias)
        self.conv_h2h = None if alpha_in == 1 or alpha_out == 1 else \
                        nn.Conv2d(in_channels - int(alpha_in * in_channels), out_channels - int(alpha_out * out_channels),
                                  kernel_size, 1, padding, dilation, math.ceil(groups - alpha_in * groups), bias)

    def forward(self, x):
        '''
        input : 1) (x_h, x_l) /or/ 2) (x_h, None)
        -> 1)번 case의 경우는 그대로 진행하면 됨.
        -> 2)번 case의 경우는 x_l을 None으로 따로 지정해 줘야함

        downsampling : stride가 2이면 size 줄어야 하므로 self.downsample 해줌.
        아니면 안해줌 ; first block case
        -> misalignment 때문에 stride 2 conv 보다 downsample + stride 1 conv 해줌
        '''
        x_h, x_l = x if type(x) is tuple else (x, None)

        # stride == 2 : transition phase -> 이미지 크기를 줄여줘야 함
        # downsample(avgpool) 한 다음에 conv(stride==1) 함
        # 근데 H path에서 transition phase에서도 이렇게 해줘야 하나?
        # avgpool+conv는 애초에 upsample에서의 misalign을 방지하기 위함이 아니었나? 잘 모르겠네
        x_h   = self.downsample(x_h) if self.stride == 2 else x_h
        # print(x_h.shape)
        # x_h2h
        x_h2h = self.conv_h2h(x_h)
        # x_h2l : transition phase : downsample이 두번 됨.
        # center of patch는 계속 유지됨.
        x_h2l = self.conv_h2l(self.downsample(x_h)) if self.alpha_out > 0 and not self.is_dw else None

        if x_l is not None:
            # transition phase : stride 2 conv 대신 downsample + stride 1 conv
            x_l2l = self.downsample(x_l) if self.stride == 2 else x_l
            # x_l2l
            x_l2l = self.conv_l2l(x_l2l) if self.alpha_out > 0 else None

            if self.is_dw:
                return x_h2h, x_l2l
            else :
                x_l2h = self.conv_l2h(x_l)
                x_l2h = self.upsample(x_l2h) if self.stride == 1 else x_l2h
                x_h   = x_l2h + x_h2h
                # x_h2l, x_l2l 모두 존재할 때 => x_l 존재
                x_l   = x_h2l + x_l2l if x_h2l is not None and x_l2l is not None else None
                return x_h, x_l
        else :
            # x_l is None : network 시작부분
            return x_h2h, x_h2l

class Conv_BN(nn.Module):
    def __init__(self,
                in_channels,
                out_channels,
                kernel_size,
                alpha_in,
                alpha_out,
                stride=1,
                padding=0,
                dilation=1,
                groups=1,
                bias=False,
                norm_layer=nn.BatchNorm2d):
        super(Conv_BN, self).__init__()
        '''
        conv-bn module
        conv의 output이 두개일 경우가 대부분이라 다루기 힘들기 때문에, bn까지 같이 붙여준다.
        '''
        self.conv = OctaveConv(in_channels,
                            out_channels,
                            kernel_size,
                            alpha_in,
                            alpha_out,
                            stride,
                            padding,
                            dilation,
                            groups,
                            bias)
        self.bn_h = None if alpha_out == 1 else norm_layer(int(out_channels * (1 - alpha_out)))
        self.bn_l = None if alpha_out == 0 else norm_layer(int(out_channels * alpha_out))

    def forward(self, x):
        x_h, x_l = self.conv(x)
        x_h      = self.bn_h(x_h)
        x_l      = self.bn_l(x_l) if x_l is not None else None
        return x_h, x_l

class Conv_BN_ACT(nn.Module):
    def __init__(self,
                in_channels,
                out_channels,
                kernel_size,
                alpha_in,
                alpha_out,
                stride=1,
                padding=0,
                dilation=1,
                groups=1,
                bias=False,
                norm_layer=nn.BatchNorm2d,
                activation_layer=nn.ReLU):
        super(Conv_BN_ACT, self).__init__()
        '''
        conv-bn-relu
        '''
        self.conv = OctaveConv(in_channels,
                            out_channels,
                            kernel_size,
                            alpha_in,
                            alpha_out,
                            stride,
                            padding,
                            dilation,
                            groups,
                            bias)
        self.bn_h = None if alpha_out == 1 else norm_layer(int(out_channels * (1 - alpha_out)))
        self.bn_l = None if alpha_out == 0 else norm_layer(int(out_channels * alpha_out))
        self.act  = activation_layer(inplace=True)

    def forward(self, x):
        x_h, x_l = self.conv(x)
        x_h      = self.act(self.bn_h(x_h))
        x_l      = self.act(self.bn_l(x_l)) if x_l is not None else None
        return x_h, x_l


#!/usr/bin/python3
# Filename: maxSonarTTY.py

# Reads serial data from Maxbotix ultrasonic rangefinders
# Gracefully handles most common serial data glitches
# Use as an importable module with "import MaxSonarTTY"
# Returns an integer value representing distance to target in millimeters

from time import time
from serial import Serial

class MaxSonarTTY:
    """Uses the MaxSonarTTY to find a range"""

    def __init__(self, serialDevice = "/dev/ttyS0", maxWait = 3):
        self.serialDevice = serialDevice
        self.maxWait = maxWait

        print('self.device', self.serialDevice)
        print('self.maxWait', self.maxWait)

    def measure(self):
        print("serialDevice", self.serialDevice)
        ser = Serial(self.serialDevice, 9600, 8, 'N', 1, timeout=1)
        timeStart = time()
        valueCount = 0

        print(ser);
        while time() < timeStart + self.maxWait:
            if ser.inWaiting():
                print("it's waiting")
                bytesToRead = ser.inWaiting()
                valueCount += 1
                if valueCount < 2: # 1st reading may be partial number; throw it out
                    continue
                testData = ser.read(bytesToRead)
                if not testData.startswith(b'R'):
                    # data received did not start with R
                    continue
                try:
                    sensorData = testData.decode('utf-8').lstrip('R')
                except UnicodeDecodeError:
                    # data received could not be decoded properly
                    continue
                try:
                    mm = int(sensorData)
                except ValueError:
                    # value is not a number
                    continue
                ser.close()
                return(mm)

        ser.close()


import threading
import RPi.GPIO as GPIO
import time

BUTTON = 4
LED1, LED2= 14, 18
SERVO = 23
BUZZER = 24
timer = 0.2
servo_max, servo_min = 7, 2
servo_current = servo_min
state, prev_state = 0, 0
# STATE = {
#     NO_TRAIN: 0,        # 0: no train
#     COME_WARNING: 1,    # 1: coming warning
#     PASSING: 2,         # 2: passing
#     LEAVE_WARNING: 3    # 3: leaving warning
# }

# Set up GPIO pins
GPIO.setmode(GPIO.BCM)
GPIO.setup(BUZZER, GPIO.OUT)
GPIO.setup(LED1, GPIO.OUT)
GPIO.setup(LED2, GPIO.OUT)
GPIO.setup(SERVO, GPIO.OUT)
GPIO.setup(BUTTON, GPIO.IN)
servo = GPIO.PWM(SERVO, 50) # 50 Hz PWM frequency
servo.start(2) # start at 0 degrees

def led_buzzer_flash():
    while not stop_event.is_set():
        if state in {1,2,3}:
            GPIO.output(LED1, GPIO.HIGH)
            GPIO.output(LED2, GPIO.LOW)
            GPIO.output(BUZZER, GPIO.HIGH)
            time.sleep(timer)
            GPIO.output(LED1, GPIO.LOW)
            GPIO.output(LED2, GPIO.HIGH)
            GPIO.output(BUZZER, GPIO.LOW)
            time.sleep(timer)
        else:
            GPIO.output(LED1, GPIO.LOW)
            GPIO.output(LED2, GPIO.LOW)
            GPIO.output(BUZZER, GPIO.LOW)

def motor():
    global servo_current
    while not stop_event.is_set():
        if state == 1:
            if servo_current < servo_max:
                servo_current += 0.5
            else:
                servo_current = servo_max
        elif state == 3:
            if servo_current > servo_min:
                servo_current -= 0.5
            else:
                servo_current = servo_min
        else:
            None

        servo.ChangeDutyCycle(servo_current)
        time.sleep(timer)

if __name__ == '__main__':
    stop_event = threading.Event()
    thread1 = threading.Thread(target=led_buzzer_flash)
    thread2 = threading.Thread(target=motor)

    thread1.start()
    thread2.start()

    try:
        None
        while True:
            if prev_state == 0:
                if GPIO.input(BUTTON):
                    print("pressed")
                    state = 1
                    time.sleep(timer)
                else:
                    state = 0
            elif prev_state == 1:
                if servo_current == servo_max:
                    state = 2
                else:
                    state = 1
            elif prev_state == 2:
                if GPIO.input(BUTTON):
                    print("pressed")
                    state = 3
                    time.sleep(timer)
                else:
                    state = 2
            elif prev_state == 3:
                if servo_current == servo_min:
                    state = 0
                else:
                    state = 3
            else:
                print("ERROR STATE!")

            # if GPIO.input(PIN_BUTTON):
            #     print("pressed")
            #     time.sleep(timer)
            prev_state = state
    except KeyboardInterrupt:
        print("Keyboard interrupt")
        stop_event.set()
    finally:
        # thread1.join()
        # thread2.join()
        GPIO.cleanup()
        print("clean up")
        # print("Done")


import re
from typing import Any, Dict, List, Union  # noqa: F401

from databuilder.models.neo4j_csv_serde import Neo4jCsvSerializable, \
    RELATION_START_KEY, RELATION_START_LABEL, RELATION_END_KEY, \
    RELATION_END_LABEL, RELATION_TYPE, RELATION_REVERSE_TYPE

from databuilder.models.table_metadata import TableMetadata

class TableLineage(Neo4jCsvSerializable):
    # type: (...) -> None
    """
    Table Lineage Model. It won't create nodes but create upstream/downstream rels.
    """
    LABEL = 'Lineage'
    KEY_FORMAT = '{db}://{cluster}.{schema}/{tbl}/'
    ORIGIN_DEPENDENCY_RELATION_TYPE = 'UPSTREAM'
    DEPENDENCY_ORIGIN_RELATION_TYPE = 'DOWNSTREAM'

    def __init__(self,
                 db_name,  # type: str
                 schema_name,  # type: str
                 table_name,  # type: str
                 cluster,  # type: str
                 downstream_deps=None,  # type: List
                 ):
        # type: (...) -> None
        self.db = db_name.lower()
        self.schema = schema_name.lower()
        self.table = table_name.lower()

        self.cluster = cluster.lower() if cluster else 'gold'
        # a list of downstream dependencies, each of which will follow
        # the same key
        self.downstream_deps = downstream_deps
        self._node_iter = iter(self.create_nodes())
        self._relation_iter = iter(self.create_relation())

    def create_next_node(self):
        # type: (...) -> Union[Dict[str, Any], None]
        # return the string representation of the data
        try:
            return next(self._node_iter)
        except StopIteration:
            return None

    def create_next_relation(self):
        # type: (...) -> Union[Dict[str, Any], None]
        try:
            return next(self._relation_iter)
        except StopIteration:
            return None

    def get_table_model_key(self, db, cluster, schema, table):
        # type: (...) -> str
        return '{db}://{cluster}.{schema}/{table}'.format(db=db,
                                                          cluster=cluster,
                                                          schema=schema,
                                                          table=table)

    def create_nodes(self):
        # type: () -> List[Union[Dict[str, Any], None]]
        """
        It won't create any node for this model
        :return:
        """
        return []

    def create_relation(self):
        # type: () -> List[Dict[str, Any]]
        """
        Create a list of relation between source table and all the downstream tables
        :return:
        """
        results = []
        for downstream_tab in self.downstream_deps:
            # every deps should follow '{db}://{cluster}.{schema}/{table}'
            # todo: if we change the table uri, we should change here.
            m = re.match('(\w+)://(\w+)\.(\w+)\/(\w+)', downstream_tab)
            if m:
                # if not match, skip those records
                results.append({
                    RELATION_START_KEY: self.get_table_model_key(db=self.db,
                                                                 cluster=self.cluster,
                                                                 schema=self.schema,
                                                                 table=self.table),
                    RELATION_START_LABEL: TableMetadata.TABLE_NODE_LABEL,
                    RELATION_END_KEY: self.get_table_model_key(db=m.group(1),
                                                               cluster=m.group(2),
                                                               schema=m.group(3),
                                                               table=m.group(4)),
                    RELATION_END_LABEL: TableMetadata.TABLE_NODE_LABEL,
                    RELATION_TYPE: TableLineage.ORIGIN_DEPENDENCY_RELATION_TYPE,
                    RELATION_REVERSE_TYPE: TableLineage.DEPENDENCY_ORIGIN_RELATION_TYPE
                })
        return results

    def __repr__(self):
        # type: () -> str
        return 'TableLineage({!r}, {!r}, {!r}, {!r})'.format(self.db,
                                                             self.cluster,
                                                             self.schema,
                                                             self.table)