''' class_list_pickle1.py
create a list of class instances
The class itself cannot be pickled, but the list of instances can.
To use the pickled data file in another program, you have to
recode the class in that program.
tested with Python273 and Python33  by vegaseat  08jan2013
also works with IronPython273
'''
import operator
import pickle
class Person:
    def __init__(self, name=None, sex=None, slogan=None):
        self.name = name
        self.sex = sex
        self.slogan = slogan
# a csv type data string for testing
data_str = '''\
Porke,Emma,female,Vote for Bush and Dick
Schidd,Jack,male,Never give a schidd!
Negerschwarz,Arnold,often,I know how to spell nukilar!
'''
# make a list of class Person instances using the data_str
person_list = []
for line in data_str.split('\n'):
    if line:
        line = line.split(',')
        pname = "%s %s" % (line[1], line[0])
        person_list.append(Person(pname, line[2], line[3]))
# testing ...
print("Show all persons and what they say:")
for person in person_list:
    print("{} says \"{}\"".format(person.name, person.slogan))
print('-'*50)
print("Show one particular item:")
print(person_list[0].name)
print('-'*50)
print("Sort the person_list by name and show ...")
plist_sorted = sorted(person_list, key=operator.attrgetter('name'))
for person in plist_sorted:
    print(person.name)
print('-'*50)
# pickle the list of instances
fname = "person_instance_list.pkl"
# pickle dump the list object
with open(fname, "wb") as fout:
    # default protocol is zero
    # -1 gives highest prototcol and smallest data file size
    pickle.dump(person_list, fout, protocol=-1)
print("Data has been pickled to file {}".format(fname))
# pickle load the list object back in (senses protocol)
with open(fname, "rb") as fin:
    person_list2 = pickle.load(fin)
print("Pickled data has been reloaded ..")
print('-'*50)
# test loaded list of instances
for person in person_list2:
    print("{} says \"{}\"".format(person.name, person.slogan))
''' result ...
Show all persons and what they say:
Emma Porke says "Vote for Bush and Dick"
Jack Schidd says "Never give a schidd!"
Arnold Negerschwarz says "I know how to spell nukilar!"
--------------------------------------------------
Show one particular item:
Emma Porke
--------------------------------------------------
Sort the person_list by name and show ...
Arnold Negerschwarz
Emma Porke
Jack Schidd
--------------------------------------------------
Data has been pickled to file person_instance_list.pkl
Pickled data has been reloaded ..
--------------------------------------------------
Emma Porke says "Vote for Bush and Dick"
Jack Schidd says "Never give a schidd!"
Arnold Negerschwarz says "I know how to spell nukilar!"
'''


from KratosMultiphysics import *

from sympy import *
from sympy_fe_utilities import *
import pprint

from params_dict import params
import ConvectiveFlux
import DiffusiveFlux
import SourceTerm
import StabilizationMatrix

dim = params["dim"]                         # Define Dimension in params.py
BlockSize = dim+2 					        # Dimension of the vector of Unknowns
do_simplifications = False
mode = "c"                                  # Output mode to a c++ file

if(dim == 2):
    nnodes = 3
elif(dim == 3):
    nnodes = 4

impose_partion_of_unity = False
N,DN = DefineShapeFunctions(nnodes, dim, impose_partion_of_unity)

# Unknown fields definition (Used later for the gauss point interpolation)
U = DefineMatrix('U',nnodes,BlockSize)	     # Vector of Unknowns ( Density,Velocity[dim],Total Energy )
Un = DefineMatrix('Un',nnodes,BlockSize)     # Vector of Unknowns one step back
Unn = DefineMatrix('Unn',nnodes,BlockSize)   # Vector of Unknowns two steps back
r = DefineVector('r',nnodes)                 # Sink term    #COMMENT for manufactured solution

# Test functions defintiion
w = DefineMatrix('w',nnodes,BlockSize)	    # Variables field test

# External terms definition
f_ext = DefineMatrix('f_ext',nnodes,dim)    # Forcing term #COMMENT for manufactured solution

# Definition of other symbols
bdf0 = Symbol('bdf0')                       # Backward differantiation coefficients
bdf1 = Symbol('bdf1')
bdf2 = Symbol('bdf2')
v_sc = Symbol('v_sc')                       # Shock capturing Viscosity
k_sc = Symbol('k_sc')                       # Shock capturing Conductivity

### Construction of the variational equation
Ug = DefineVector('Ug',BlockSize)			# Dofs vector
H = DefineMatrix('H',BlockSize,dim)			# Gradient of U
f = DefineVector('f',dim)			        # Body force vector
rg = Symbol('rg', positive = True)		    # Source/Sink term
V = DefineVector('V',BlockSize)			    # Test function
Q = DefineMatrix('Q',BlockSize,dim)			# Gradient of V
acc = DefineVector('acc',BlockSize)         # Derivative of Dofs/Time
#G = DefineMatrix('G',BlockSize,dim)		# Diffusive Flux matrix
Gsc = DefineMatrix('G',BlockSize,dim)       # Diffusive Flux matrix with Shock Capturing

## Matrix Computation

S = SourceTerm.computeS(f,rg,params)
#SourceTerm.printS(S,params)
A = ConvectiveFlux.computeA(Ug,params)
#ConvectiveFlux.printA(A,params)
#G = DiffusiveFlux.computeG(Ug,params,H,G)
Gsc = DiffusiveFlux.computeGsc(Ug,params,H,Gsc,v_sc,k_sc)
#DiffusiveFlux.printK(Gsc,params)
Tau = StabilizationMatrix.computeTau(params)
#StabilizationMatrix.printTau(Tau,params)

## Nonlinear operator definition
l1 = Matrix(zeros(dim+2,1))		            # Convective Matrix*Gradient of U
A_small = []
for j in range(0,dim):
    A_small = A[j]
    for ll in range(BlockSize):
        for mm in range(BlockSize):
            l1[ll] += A_small[ll,mm]*H[mm,j]

l3 = S*Ug				                    # Source term
print("\nCompute Non-linear operator\n")
L = l1-l3                                   # Nonlinear operator

## Residual definition
res = -acc - L

## Nonlinear adjoint operator definition
m1 = Matrix(zeros(dim+2,1))		            # Convective term
psi = Matrix(zeros(dim+2,dim))

for j in range(0,dim):
    A_T = A[j].transpose()
    for l in range(0,dim+2):
        for m in range(0,dim+2):
            psi[l,j] += A_T[l,m]*Q[m,j]
            for n in range(0,dim+2):
                psi[l,j] +=diff(A_T[l,m],Ug[n])*H[n,j]*V[m]

for s in range(0,dim+2):
    for j in range(0,dim):
        m1[s] += psi[s,j]

m3 = S.transpose()*V			            # Source term

L_adj = -m1-m3                              # Nonlinear adjoint operator

## Istotropic Residual Based Shock Capturing
res_m = Matrix(zeros(dim,1))                # Momentum residual
for i in range(0,dim):
    res_m[i,0] = res[i+1,0]

res_e = Matrix(zeros(1,1))                  # Energy residual
res_e[0,0] = res[dim+1]

## Variational Formulation - Final equation
n1 = V.transpose()*acc		                # Mass term - FE scale

temp = zeros(dim+2,1)
A_smalll = []
for i in range(0,dim):
    A_smalll = A[i]
    for ll in range(BlockSize):
        for mm in range(BlockSize):
            temp[ll] += A_smalll[ll,mm]*H[mm,i]

n2 = V.transpose()*temp			            # Convective term - FE scale

n3 = Matrix(zeros(1,1))                     # Diffusive term - FE scale

for j in range(0,dim):
    for k in range(BlockSize):
        n3[0,0] += Q[k,j]*(-Gsc[k,j])       # G with shock capturing - FE scale

n4 = -V.transpose()*(S*Ug)		            # Source term - FE scale

n5 = L_adj.transpose()*(Tau*res)	        # VMS_adjoint - Subscales

print("\nCompute Variational Formulation\n")
rv = n1+n2+n3+n4+n5 			            # VARIATIONAL FORMULATION - FINAL EQUATION

### Substitution of the discretized values at the gauss points
print("\nSubstitution of the discretized values at the gauss points\n")

## Data interpolation at the gauss points
U_gauss = U.transpose()*N
w_gauss = w.transpose()*N
f_gauss = f_ext.transpose()*N                     #COMMENT for manufactured solution
acc_gauss = (bdf0*U+bdf1*Un+bdf2*Unn).transpose()*N
r_gauss = (r.transpose()*N)[0]                    #COMMENT for manufactured solution
#r_gauss = Symbol('r_gauss', positive = True)     #USED fro manufactured solution

## Gradients computation
grad_U = DfjDxi(DN,U).transpose()
grad_w = DfjDxi(DN,w).transpose()

print("\nSubstitution in the variational formulation\n")
SubstituteMatrixValue(rv, Ug, U_gauss)
SubstituteMatrixValue(rv, acc, acc_gauss)
SubstituteMatrixValue(rv, H, grad_U)
SubstituteMatrixValue(rv, V, w_gauss)
SubstituteMatrixValue(rv, Q, grad_w)
SubstituteMatrixValue(rv, f, f_gauss)           #COMMENT for manufactured solution
SubstituteScalarValue(rv, rg, r_gauss)          #COMMENT for manufactured solution

print("\nSubstitution in the residual of momentum\n")
SubstituteMatrixValue(res_m, Ug, U_gauss)
SubstituteMatrixValue(res_m, acc, acc_gauss)
SubstituteMatrixValue(res_m, H, grad_U)
SubstituteMatrixValue(res_m, f, f_gauss)       #COMMENT for manufactured solution
SubstituteScalarValue(res_m, rg, r_gauss)      #COMMENT for manufactured solution

print("\nSubstitution in the residual of total energy\n")
SubstituteMatrixValue(res_e, Ug, U_gauss)
SubstituteMatrixValue(res_e, acc, acc_gauss)
SubstituteMatrixValue(res_e, H, grad_U)
SubstituteMatrixValue(res_e, f, f_gauss)       #COMMENT for manufactured solution
SubstituteScalarValue(res_e, rg, r_gauss)      #COMMENT for manufactured solution

dofs = Matrix(zeros(nnodes*(dim+2),1))
testfunc = Matrix(zeros(nnodes*(dim+2),1))
for i in range(0,nnodes):
        for j in range(0,dim+2):
            dofs[i*(dim+2)+j] = U[i,j]
            testfunc[i*(dim+2)+j] = w[i,j]

## Compute LHS and RHS
print("\nCompute RHS\n")
rhs = Compute_RHS(rv.copy(), testfunc, do_simplifications)
rhs_out = OutputVector_CollectingFactors(rhs, "rhs", mode)

print("\nCompute LHS\n")
lhs = Compute_LHS(rhs, testfunc, dofs, do_simplifications) # Compute the LHS
lhs_out = OutputMatrix_CollectingFactors(lhs, "lhs", mode)

## Residual for shock capturing
res_m_out = OutputMatrix_CollectingFactors(res_m, "res_m", mode)
res_e_out = OutputMatrix_CollectingFactors(res_e, "res_e", mode)

## Reading Template File
print("\nReading compressible_navier_stokes_cpp_template.cpp\n")
if(dim==2):
        templatefile = open("compressible_navier_stokes_cpp_template2D.cpp")
        outstring=templatefile.read()
        outstring = outstring.replace("//substitute_lhs_2D", lhs_out)
        outstring = outstring.replace("//substitute_rhs_2D", rhs_out)
        outstring = outstring.replace("//substitute_res_m_2D", res_m_out)
        outstring = outstring.replace("//substitute_res_e_2D", res_e_out)
        ## Write the modified template
        print("\nWriting compressible_navier_stokes2D.cpp\n")
        out = open("compressible_navier_stokes2D.cpp",'w')
        out.write(outstring)
        out.close()
elif(dim == 3):
        templatefile = open("compressible_navier_stokes_cpp_template3D.cpp")
        outstring=templatefile.read()
        outstring = outstring.replace("//substitute_lhs_3D", lhs_out)
        outstring = outstring.replace("//substitute_rhs_3D", rhs_out)
        outstring = outstring.replace("//substitute_res_m_3D", res_m_out)
        outstring = outstring.replace("//substitute_res_e_3D", res_e_out)

        ## Write the modified template
        print("\nWriting compressible_navier_stokes3D.cpp\n")
        out = open("compressible_navier_stokes3D.cpp",'w')
        out.write(outstring)
        out.close()
print("\nCompressible Navier Stokes Element Generated\n")


# -*- coding: utf-8 -*-
import os
import subprocess
import re
import socket

from libqtile.config import Key, Screen, Group, Drag, Click
from libqtile.command import lazy
from libqtile import layout, bar, widget, hook

from libqtile import xcbq
xcbq.keysyms["XF86AudioRaiseVolume"] = 0x1008ff13
xcbq.keysyms["XF86AudioLowerVolume"] = 0x1008ff11
xcbq.keysyms["XF86AudioMute"] = 0x1008ff12

xcbq.keysyms["XF86MonBrightnessUp"] = 0x1008ff02
xcbq.keysyms["XF86MonBrightnessDown"] = 0x1008ff03

mod = "mod4"
home = os.path.expanduser('~')

keys = [
    Key(
        [mod, "control"], "F1",
        lazy.spawn("sh -c " + home + "/.screenlayout/1screen.sh"),
    ),
    Key(
        [mod, "control"], "F2",
        lazy.spawn("sh -c " + home + "/.screenlayout/2displaysabove.sh"),
    ),
    Key(
        [mod, "control"], "F3",
        lazy.spawn("sh -c " + home + "/.screenlayout/home.sh"),
    ),

    # Switch between windows in current stack pane
    Key(
        [mod], "k",
        lazy.layout.down()
    ),
    Key(
        [mod], "j",
        lazy.layout.up()
    ),

    # Move windows up or down in current stack
    Key(
        [mod, "control"], "k",
        lazy.layout.shuffle_down()
    ),
    Key(
        [mod, "control"], "j",
        lazy.layout.shuffle_up()
    ),

    # Switch window focus to other pane(s) of stack
    Key(
        [mod], "space",
        lazy.layout.next()
    ),

    # Swap panes of split stack
    Key(
        [mod, "shift"], "space",
        lazy.layout.rotate()
    ),

    # Toggle between split and unsplit sides of stack.
    # Split = all windows displayed
    # Unsplit = 1 window displayed, like Max layout, but still with
    # multiple stack panes
    Key(
        [mod, "shift"], "Return",
        lazy.layout.toggle_split()
    ),
    Key([mod], "Return", lazy.spawn("gnome-terminal")),
    Key(["control", "mod1"], "l", lazy.spawn("/home/mfa/bin/i3lock-fancy/lock")),

    # Toggle between different layouts as defined below
    Key([mod], "space", lazy.next_layout()),
    Key([mod, "control"], "c", lazy.window.kill()),

    Key([mod, "control"], "r", lazy.restart()),
    Key([mod, "control"], "q", lazy.shutdown()),
    Key([mod], "r", lazy.spawncmd()),
    Key([mod, "control"], "space", lazy.window.toggle_floating()),

    # move cursor to other screen(s)
    Key([mod], "Up", lazy.prev_screen()),
    Key([mod], "Down", lazy.next_screen()),

    # Switch between windows in current stack pane
    Key([mod], 'Tab', lazy.layout.down()),
    Key([mod, 'shift'], 'Tab', lazy.layout.up()),

    Key(
        [mod], "F12",
        lazy.window.toggle_fullscreen()
    ),

    # music / mpd
    Key(
        [mod, "control"], "m",
        lazy.spawn("mpc toggle"),
    ),
    Key(
        [mod, "control"], "n",
        lazy.spawn("mpc next"),
    ),

    # screenshot
    Key(
        [mod], "Print",
        lazy.spawn("bash /home/mfa/bin/takescreenshot.sh")
    ),

    Key([], "XF86AudioRaiseVolume",
        lazy.spawn("amixer sset Master 5%+")),
    Key([], "XF86AudioLowerVolume",
        lazy.spawn("amixer sset Master 5%-")),
    Key([], "XF86AudioMute",
        lazy.spawn("amixer -D pulse set Master 1+ toggle")),
    Key(["shift"], "XF86AudioRaiseVolume",
        lazy.spawn("mpc volume +5")),
    Key(["shift"], "XF86AudioLowerVolume",
        lazy.spawn("mpc volume -5")),
    Key(["shift"], "XF86AudioMute",
        lazy.spawn("mpc toggle")),

    Key([], "XF86MonBrightnessUp",
        lazy.spawn("xbacklight -inc 10")),
    Key([], "XF86MonBrightnessDown",
        lazy.spawn("xbacklight -dec 10")),
]

groups = [Group(i) for i in "123456789"]

for i in groups:
    # mod1 + letter of group = switch to group
    keys.append(
        Key([mod], i.name, lazy.group[i.name].toscreen())
    )

    # mod1 + shift + letter of group = switch to & move focused window to group
    keys.append(
        Key([mod, "shift"], i.name, lazy.window.togroup(i.name))
    )

layouts = [
    layout.tile.Tile(),
    layout.Max(),
    layout.floating.Floating(),
]

widget_defaults = dict(
    font='DejaVu Sans Mono',
    fontsize=12,
    padding=3,
)

def getIp4():
    import requests
    r = requests.get("http://ipv4.nsupdate.info/myip")
    ip = r.text
    try:
        return socket.gethostbyaddr(ip)[0]
    except:
        return r.text

def getIp6():
    import requests
    try:
        r = requests.get("http://ipv6.nsupdate.info/myip")
        return r.text
    except:
        return "no ipv6"

def mpcCurrent():
    s = subprocess.Popen(["mpc", "current"], stdout=subprocess.PIPE)
    return '♬ 🎷 🎸 ' + s.stdout.read().decode('utf-8').replace('\n', '')

screens = [
    Screen(
        top=bar.Bar(
            [
                widget.GroupBox(),
                widget.Prompt(),
                widget.TaskList(),
                widget.Sep(),
                widget.GenPollText(func=mpcCurrent, update_interval=2),
                widget.Sep(),
                widget.CPUGraph(graph_color="#FF0000",
                                border_color="#FF0000"),
                widget.NetGraph(graph_color="#086FA1",
                                border_color="#086FA1",
                                interface="wlan0",
                                bandwidth_type="down"),
                widget.Systray(),
                widget.Clock(format='%Y-%m-%d %a %H:%M',
                             foreground="#ffff00",
                             fontsize=14, font="Monospace"),
            ],
            25,
        ),
        bottom=bar.Bar(
            [
                widget.WindowName(),
                widget.Notify(),
                widget.Sep(),
                widget.GenPollText(func=getIp6, update_interval=60),
                widget.Sep(),
                widget.GenPollText(func=getIp4, update_interval=60),
                widget.Sep(),
                widget.ThermalSensor(tag_sensor='Core 0', update_interval=3),
                widget.ThermalSensor(tag_sensor='Core 1', update_interval=3),
                widget.ThermalSensor(tag_sensor='temp1', update_interval=3),
                widget.Sep(),
                widget.TextBox(text=u'☼'),
                widget.Backlight(backlight_name='intel_backlight'),
                widget.Sep(),
                widget.TextBox(text=u'♬'),
                widget.Volume(update_interval=0.2),
                widget.Sep(),
                widget.Battery(),
                widget.Sep(),
                widget.Battery(battery_name="BAT1"),
                widget.Sep(),
                widget.CurrentLayout(),
            ],
            15,
        ),
    ) for i in range(3)
]

# Drag floating layouts.
mouse = [
#    Drag([mod], "Button1", lazy.window.set_position_floating(),
#         start=lazy.window.get_position()),
    Drag([mod], "Button3", lazy.window.set_size_floating(),
         start=lazy.window.get_size()),
    Click([mod], "Button2", lazy.window.bring_to_front())
]

dgroups_key_binder = None
dgroups_app_rules = []
main = None
follow_mouse_focus = True
bring_front_click = False
cursor_warp = False
floating_layout = layout.Floating()
auto_fullscreen = True

# XXX: Gasp! We're lying here. In fact, nobody really uses or cares about this
# string besides java UI toolkits; you can see several discussions on the
# mailing lists, github issues, and other WM documentation that suggest setting
# this string if your java app doesn't work correctly. We may as well just lie
# and say that we're a working one by default.
#
# We choose LG3D to maximize irony: it is a 3D non-reparenting WM written in
# java that happens to be on java's whitelist.
wmname = "LG3D"

def is_running(process):
    s = subprocess.Popen(["ps", "axuw"], stdout=subprocess.PIPE)
    for x in s.stdout:
        if re.search(process, x):
            return True
    return False

def execute_once(process):
    if not is_running(process):
        return subprocess.Popen(process, shell="/bin/bash")

@hook.subscribe.startup
def startup():
    subprocess.Popen("sleep 3".split())
#    execute_once("nm-applet")
    subprocess.Popen("feh --bg-scale /home/mfa/Pictures/1.jpg".split())
#    execute_once('xautolock -detectsleep -time 15 -locker "~/bin/i3lock-fancy/lock" & disown'.split())
#    execute_once("eval `gnome-keyring-daemon`")


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
#  dataProcess.py
#
#  Copyright 2020 Luiz Oliveira
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 2 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
#  MA 02110-1301, USA.
#
#
"""
This code processes the data imported from importcsv.py
The data is ensembled averaged and then exported to plot.py script
"""

import re
import pandas as pd
import openpyxl
from scipy.interpolate import interp1d

def dfRename(var, dtf):
    names = ['x', 'y', 'z']
    names.extend(var)
    dtf.columns = names

def clearLimits(df,x0,x1,y0,y1,z0,z1):
# =============================================================================
#     Clear extra values inside variables.
#     This script uses user values of the bound coordinates
# =============================================================================

    df.drop(df[df.x < x0].index, inplace=True)
    df.drop(df[df.x > x1].index, inplace=True)
    df.drop(df[df.y < y0].index, inplace=True)
    df.drop(df[df.y > y1].index, inplace=True)
    df.drop(df[df.z < z0].index, inplace=True)
    df.drop(df[df.z > z1].index, inplace=True)
    return df

def excelExport(var, name):
# =============================================================================
#     Creates and appends planes into an spreadsheet
# =============================================================================

    if not os.path.isfile('preTreatment/results/Excel/'+name+'.xlsx'):
        wb = openpyxl.Workbook()
        wb.save('preTreatment/results/Excel/'+name+'.xlsx')

    with pd.ExcelWriter('preTreatment/results/Excel/'+name+'.xlsx',
                        engine="openpyxl", mode='a') as writer:
        for df_name, df in var.items():
            df.to_excel(writer, sheet_name=df_name, index=False)

def csvExport(df, name):
# =============================================================================
#     Creates and appends planes into separated csv files
# =============================================================================
    df.to_csv('preTreatment/results/CSV/'+name+'.csv')

def varTreatment(planes, physicalVar, colNames, nColumns, direction,
                 varName, roundPar, first, last):
# =============================================================================
#     Treats data in an ensemble averaging proceedure in the provided direction
# =============================================================================

    # Local Variable Declaration
    varDict = dict()
    kk = first * nColumns
    startPos = first
    stopPos = last * nColumns
    ll = 0

    # Reads all the files and ensemble in a dict in that each ii is a plane
    for ii in planes:
        key = ii
        key = int(re.sub('\D', '',key))
        if key < startPos:
            continue
        if kk > stopPos:
            break
        for jj in range(nColumns):
            ll = kk + jj
            if ll%nColumns == 0: # number of columns
                varDict[ii] = physicalVar.iloc[:,ll]
            else:
                varDict[ii] = \
                    pd.concat([varDict[ii], physicalVar.iloc[:,ll]], axis=1)

        kk = kk + nColumns

        dfRename(colNames, varDict[ii])
        clearLimits(varDict[ii], 0.25, 0.50, 0.30, 0.45, 0, 0.1)
        varDict[ii] = varDict[ii].dropna(axis=0, how='all')
        varDict[ii] = varDict[ii].dropna(axis=1, how='all')

        # Get vector magnitude
        if nColumns > 4:
            df = pd.DataFrame()
            for i in colNames:
                df[i] = varDict[ii][i]**2
            df['mag'] = (df.sum(axis=1))**(1/2)
            varDict[ii]['mag'] = df.mag
            expColNames = colNames + ['mag']
        else:
            expColNames = colNames

        if direction == "x":
            varDict[ii] = varDict[ii].drop(columns=['y', 'z'])
            varDict[ii] = varDict[ii].\
            groupby(varDict[ii].x.round(roundPar),as_index=False).mean()
            varDict[ii][direction] = (varDict[ii][direction] - 0.25)/L
            varDict[ii].columns = ['('+direction+'-x0)'+'/L'] + expColNames
        elif direction == 'y':
            varDict[ii] = varDict[ii].drop(columns=['x', 'z'])
            varDict[ii] = varDict[ii].\
            groupby(varDict[ii].y.round(roundPar),as_index=False).mean()
            varDict[ii][direction] = (varDict[ii][direction] - 0.30)/H
            varDict[ii].columns = ['('+direction+'-y0)'+'/H'] + expColNames
        elif direction == 'z':
            varDict[ii] = varDict[ii].drop(columns=['x', 'y'])
            varDict[ii] = varDict[ii].\
            groupby(varDict[ii].z.round(roundPar),as_index=False).mean()
            varDict[ii][direction] = varDict[ii][direction]/H
            varDict[ii].columns = [direction+'/H'] + expColNames

        excelExport(varDict, varName+"Dir_"+direction)
        csvName = varName.split("_")[0]
        csvExport(varDict[ii], csvName+"_"+ii+"_Dir_"+direction)
    return varDict

# =============================================================================
# Planes 0 -> 4
# Vertical Planes Varying the Y axis from Y = 0.30 to Y = 0.45
# =============================================================================
## RMean
colNames = ['xx', 'yy', 'zz', 'xy', 'yz', 'xz']
RMean_00_04_Dirx = varTreatment(uniqueRaw, RMean, colNames, 9,'x',
                               'RMean_00_4_',2, 0, 4)
RMean_00_04_Dirz = varTreatment(uniqueRaw, RMean, colNames, 9,'z',
                               'RMean_00_4_',2, 0, 4)

## UMean
colNames = ['u', 'v', 'w']
UMean_00_04_Dirx = varTreatment(uniqueRaw, UMean, colNames, 6,'x',
                               'UMean_00_4_',2, 0, 4)
UMean_00_04_Dirz = varTreatment(uniqueRaw, UMean, colNames, 6,'z',
                               'UMean_00_4_',2, 0, 4)

## lambVectorMean
colNames = ['lambVectorMean_x', 'lambVectorMean_y', 'lambVectorMean_z']
lambVectorMean_00_04_Dirx = varTreatment(uniqueRaw, lambVectorMean, colNames, 6,
                                    'x','lambVectorMean_00_4_',3, 0, 4)
lambVectorMean_00_04_Dirz = varTreatment(uniqueRaw, lambVectorMean, colNames,
                                    6,'z','lambVectorMean_00_4_',2, 0, 4)

## pMean
colNames = ['pMean']
pMean_00_04_Dirx = varTreatment(uniqueRaw, pMean, colNames, 4,'x',
                               'pMean_00_4_',3, 0, 4)
pMean_00_04_Dirz = varTreatment(uniqueRaw, pMean, colNames, 4,'z',
                               'pMean_00_4_',2, 0, 4)

## vorticityMean
colNames = ['vorticityMean_x', 'vorticityMean_y', 'vorticityMean_z']
vorticityMean_00_04_Dirx = varTreatment(uniqueRaw, vorticityMean, colNames, 6,
                                  'x','vorticityMean_00_4_',3, 0, 4)
vorticityMean_00_04_Dirz = varTreatment(uniqueRaw, vorticityMean, colNames, 6,
                                  'z','vorticityMean_00_4_',2, 0, 4)

# =============================================================================
# Planes 5 -> 11
# Vertical Planes Varying the X axis from X = 0.25 to X = 0.50
# =============================================================================
## RMean
colNames = ['xx', 'yy', 'zz', 'xy', 'yz', 'xz']
RMean_05_11_Diry = varTreatment(uniqueRaw, RMean, colNames, 9,'y',
                                'RMean_05_11_',2, 5, 11)
RMean_05_11_Dirz = varTreatment(uniqueRaw, RMean, colNames, 9,'z',
                                'RMean_05_11_',2, 5, 11)

## UMean
colNames = ['u', 'v', 'w']
UMean_05_11_Diry = varTreatment(uniqueRaw, UMean, colNames, 6,'y',
                                'UMean_05_11_',2, 5, 11)
UMean_05_11_Dirz = varTreatment(uniqueRaw, UMean, colNames, 6,'z',
                                'UMean_05_11_',2, 5, 11)

## lambVectorMean
colNames = ['lambVectorMean_x', 'lambVectorMean_y', 'lambVectorMean_z']
lambVectorMean_05_11_Diry = varTreatment(uniqueRaw, lambVectorMean, colNames, 6,
                                    'y','lambVectorMean_05_11_',3, 5, 11)
lambVectorMean_05_11_Dirz = varTreatment(uniqueRaw, lambVectorMean, colNames,
                                    6,'z','lambVectorMean_05_11_',2, 5, 11)

## pMean
colNames = ['pMean']
pMean_05_11_Diry = varTreatment(uniqueRaw, pMean, colNames, 4,'y',
                                'pMean_05_11_',3, 5, 11)
pMean_05_11_Dirz = varTreatment(uniqueRaw, pMean, colNames, 4,'z',
                                'pMean_05_11_',2, 5, 11)

## vorticityMean
colNames = ['vorticityMean_x', 'vorticityMean_y', 'vorticityMean_z']
vorticityMean_05_11_Diry = varTreatment(uniqueRaw, vorticityMean, colNames, 6,
                                  'y','vorticityMean_05_11_',3, 5, 11)
vorticityMean_05_11_Dirz = varTreatment(uniqueRaw, vorticityMean, colNames, 6,
                                  'z','vorticityMean_05_11_',2, 5, 11)

# =============================================================================
# Planes 12 -> 21
# Horizontal Planes Varying the Z axis from Z = 0 to Z = 0.10
# =============================================================================
## RMean
colNames = ['xx', 'yy', 'zz', 'xy', 'yz', 'xz']
RMean_12_21_Dirx = varTreatment(uniqueRaw, RMean, colNames, 9,'x',
                                'RMean_12_21_',2, 12, 21)
RMean_12_21_Diry = varTreatment(uniqueRaw, RMean, colNames, 9,'y',
                                'RMean',2, 12, 21)

## UMean
colNames = ['u', 'v', 'w']
UMean_12_21_Dirx = varTreatment(uniqueRaw, UMean, colNames, 6,'x',
                                'UMean_12_21_',2, 12, 21)
UMean_12_21_Diry = varTreatment(uniqueRaw, UMean, colNames, 6,'y',
                                'UMean_12_21_',2, 12, 21)

## lambVectorMean
colNames = ['lambVectorMean_x', 'lambVectorMean_y', 'lambVectorMean_z']
lambVectorMean_12_21_Dirx = varTreatment(uniqueRaw, lambVectorMean, colNames, 6,
                                    'x','lambVectorMean_12_21_',3, 12, 21)
lambVectorMean_12_21_Diry = varTreatment(uniqueRaw, lambVectorMean, colNames,
                                    6,'y','lambVectorMean_12_21_',2, 12, 21)

## pMean
colNames = ['pMean']
pMean_12_21_Dirx = varTreatment(uniqueRaw, pMean, colNames, 4,'x',
                                'pMean_12_21_',3, 12, 21)
pMean_12_21_Diry = varTreatment(uniqueRaw, pMean, colNames, 4,'y',
                                'pMean_12_21_',2, 12, 21)

## vorticityMean
colNames = ['vorticityMean_x', 'vorticityMean_y', 'vorticityMean_z']
vorticityMean_12_21_Dirx = varTreatment(uniqueRaw, vorticityMean, colNames, 6,
                                  'x','vorticityMean_12_21_',3, 12, 21)
vorticityMean_12_21_Diry = varTreatment(uniqueRaw, vorticityMean, colNames, 6,
                                  'y','vorticityMean_5_11_',2, 12, 21)

# =============================================================================
# Validation Data
# =============================================================================
## Data Treatment
colNames = ['u', 'v', 'w']
fig4aOur = varTreatment(['p17'], UMean, colNames, 6, "y",'UMean_p17_',
                            3, 17, 17)
fig4aOur = fig4aOur['p17']
fig4aOur.u = fig4aOur.u/U

# Errors from experimental
error = dict()
error['X'] = literatureExp.iloc[:,0]
error['Numerical_u'] = interp1d(fig4aOur.iloc[:,0],fig4aOur.u)(literatureExp.iloc[:,0])
error['Experimental_u'] = literatureExp.iloc[:,1]

error = pd.DataFrame(data=error)
error.eval('Error = Experimental_u - Numerical_u', inplace=True)
error.eval('Abs_Error = abs(Experimental_u - Numerical_u)', inplace=True)
error.eval('Rel_Error = (Experimental_u - Numerical_u)/Experimental_u', inplace=True)
error.eval('Abs_Rel_Error = abs((Experimental_u - Numerical_u)/Experimental_u)', inplace=True)

description = error.describe()

if not os.path.isfile('preTreatment/results/Excel/validationData.xlsx'):
    wb = openpyxl.Workbook()
    wb.save('preTreatment/results/Excel/validationData.xlsx')

with pd.ExcelWriter('preTreatment/results/Excel/validationData.xlsx',
                    engine="openpyxl", mode='a') as writer:
    error.to_excel(writer, sheet_name='Errors', index=False)
    description.to_excel(writer, sheet_name='Statistical Description')

del lambVectorMean, pMean, RMean, vorticityMean, colNames


#!/usr/bin/env python3

def in_corner(x, y, half, quart):
    return (x < half and y < quart) or (x < quart and y < half)

def in_center(x, y, half, quart):
    return x >= quart and y >= quart and in_corner(x - quart, y - quart, half, quart)

def main():
    x, y = map(int, input().split())
    half = 1 << 62
    orientation = 0
    while half >= 2:
        quart = half >> 1
        if in_corner(x, y, half, quart):
            # bottom left
            pass
        elif in_center(x, y, half, quart):
            x -= quart
            y -= quart
        elif x >= half:
            # bottom right
            orientation += 3
            x -= half
            # rotate about center
            x, y = y, half - 1 - x
        else:
            # top left
            orientation += 1
            y -= half
            # rotate about center
            x, y = half - 1 - y, x

        half = quart

    orientation %= 4
    print(orientation)

T = int(input())
for _ in range(T):
    main()


import open3d as o3d
import numpy as np
import time
import copy
from sklearn.neighbors import KDTree
import matplotlib.pyplot as plt

def draw_registration_result(source, target, transformation):
    source_temp = copy.deepcopy(source)
    target_temp = copy.deepcopy(target)
    source_temp.estimate_normals(
        o3d.geometry.KDTreeSearchParamHybrid(radius=0.15, max_nn=30))
    target_temp.estimate_normals(
        o3d.geometry.KDTreeSearchParamHybrid(radius=0.15, max_nn=30))
    source_temp.paint_uniform_color([1, 0.706, 0])
    target_temp.paint_uniform_color([0, 0.651, 0.929])
    source_temp.transform(transformation)
    o3d.visualization.draw_geometries([source_temp, target_temp])

def preprocess_point_cloud(pcd, voxel_size):
    print(":: Downsample with a voxel size %.3f." % voxel_size)
    pcd_down = pcd.voxel_down_sample(voxel_size)

    radius_normal = voxel_size * 2
    # print(":: Estimate normal with search radius %.3f." % radius_normal)
    pcd_down.estimate_normals(
        o3d.geometry.KDTreeSearchParamHybrid(radius=radius_normal, max_nn=30))

    radius_feature = voxel_size * 5
    # print(":: Compute FPFH feature with search radius %.3f." % radius_feature)
    pcd_fpfh = o3d.pipelines.registration.compute_fpfh_feature(
        pcd_down,
        o3d.geometry.KDTreeSearchParamHybrid(radius=radius_feature, max_nn=100))
    return pcd_down, pcd_fpfh

def prepare_dataset(source, target, voxel_size):
    source_down, source_fpfh = preprocess_point_cloud(source, voxel_size)
    target_down, target_fpfh = preprocess_point_cloud(target, voxel_size)
    # print('source_fpfh', source_fpfh.num, target_fpfh.num)
    #     print('source_fpfh',source_fpfh,np.asarray(source_fpfh.data))
    return source, target, source_down, target_down, source_fpfh, target_fpfh

def execute_global_registration(source_down, target_down, source_fpfh,
                                target_fpfh, voxel_size):
    distance_threshold = voxel_size * 1.5
    # print(":: RANSAC registration on downsampled point clouds.")
    # print("   Since the downsampling voxel size is %.3f," % voxel_size)
    # print("   we use a liberal distance threshold %.3f." % distance_threshold)
    result = o3d.pipelines.registration.registration_ransac_based_on_feature_matching(
        source_down, target_down, source_fpfh, target_fpfh, distance_threshold,
        o3d.pipelines.registration.TransformationEstimationPointToPoint(False),
        4, [
            o3d.pipelines.registration.CorrespondenceCheckerBasedOnEdgeLength(
                0.9),
            o3d.pipelines.registration.CorrespondenceCheckerBasedOnDistance(
                distance_threshold)
        ], o3d.pipelines.registration.RANSACConvergenceCriteria(2000000, 500))
    return result

def file2matrix(filename):
    fr = open(filename)
    numberOfLines = len(fr.readlines())  # get the number of lines in the file
    trans = np.eye(4)  # prepare matrix to return
    truth = []  # prepare labels return
    fr = open(filename)
    index = 0
    for line in fr.readlines():
        line = line.strip()
        # listFromLine = line.split('\t')
        listFromLine = line.split()
        listFromLine = [float(x) for x in listFromLine]

        if (index % 5 == 0):
            index = 0
        elif (index % 5 == 1):
            trans[0, :] = np.array(listFromLine)
        elif (index % 5 == 2):
            trans[1, :] = np.array(listFromLine)
        elif (index % 5 == 3):
            trans[2, :] = np.array(listFromLine)
        elif (index % 5 == 4):
            trans[3, :] = np.array(listFromLine)
            truth.append(trans.copy())  # 这里不用copy的话，，，每个元素都是一样的
        index += 1
    return truth

if __name__ == '__main__':
    root_path = '/Bill/DataSet/RedWood/'
    dataset_names = ['loft', 'lobby', 'apartment', 'bedroom', 'boardroom']
    root_save_path = '/ransac_2M/src2ref'
    dataset_numbers = [252, 199, 319, 219, 243]
    for i in range(len(dataset_names)):
        # for i in  range(1):
        file_path = root_path + dataset_names[i]
        end = dataset_numbers[i]
        save_path = dataset_names[i] + root_save_path

        print(file_path)
        groud_truth = file2matrix(file_path + '/reg_output.log')

        voxel_size = 0.05  # means 5cm for this dataset
        err_R = []
        err_T = []
        trans_all = []
        fail_list = []
        start = 0
        # end = 251
        for j in range(start, end):
            print(
                'j', j
            )
            # index_src = j + 1
            # index_ref = j

            index_src = j
            index_ref = j + 1

            source_show = o3d.io.read_point_cloud(file_path + "/mesh_%s.ply" % (index_src))
            target_show = o3d.io.read_point_cloud(file_path + "/mesh_%s.ply" % (index_ref))
            source, target, source_down, target_down, source_fpfh, target_fpfh = prepare_dataset(source_show,
                                                                                target_show,
                                                                                voxel_size)
            result_ransac = execute_global_registration(source_down, target_down,
                                                        source_fpfh, target_fpfh,
                                                        voxel_size)
            print(result_ransac.transformation)
            total_trans = result_ransac.transformation

            R = total_trans[:3, :3].reshape(3, 3)
            t = total_trans[:3, 3].reshape(-1, 1)
            if index_src > index_ref:
                err_R.append(np.arccos((np.trace(R.T @ groud_truth[j][:3, :3]) - 1) / 2) * 180 / np.pi)
                err_T.append(np.linalg.norm(t - groud_truth[j][:3, 3].reshape(-1, 1), ord=2, axis=0))
                trans_all.append((total_trans))
            else:
                err_R.append(np.arccos((np.trace(R @ groud_truth[j][:3, :3]) - 1) / 2) * 180 / np.pi)
                err_T.append(np.linalg.norm(-R.T @ t - groud_truth[j][:3, 3].reshape(-1, 1), ord=2, axis=0))
                trans_all.append((total_trans))

            # print(total_trans[:3,:3] @ groud_truth[j][:3,:3], np.trace(total_trans[:3,:3] @ groud_truth[j][:3,:3] - np.eye(3)))
            # print(total_trans, groud_truth[j])
            print('err_R err_T', err_R[j - start], err_T[j - start], total_trans)
        if index_src > index_ref:
            #
            #     location = str(start) + '_' + str(end)

            err_all = [err_R, err_T]
            plt.figure("ERR_R ref2src")  # 图像窗口名称
            plt.plot(err_R)
            plt.savefig(save_path + '/%s_%s_err_All_ref2src.jpg' % (start, end))
            # plt.show()
            plt.close()
            plt.figure("ERR_T ref2src")  # 图像窗口名称
            plt.plot(err_T)
            plt.savefig(save_path + '/%s_%s_trans_all_ref2src.jpg' % (start, end))
            # plt.show()
            plt.close()
            np.savetxt(save_path + '/%s_%s_fail_list_ref2src.txt' % (start, end), fail_list)
            np.save(save_path + '/%s_%s_err_All_ref2src.npy' % (start, end), err_all)
            np.savetxt(save_path + '/%s_%s_err_All_ref2src.txt' % (start, end), err_all)
            np.save(save_path + '/%s_%s_trans_all_ref2src.npy' % (start, end), trans_all)
            np.savetxt(save_path + '/%s_%s_trans_all_ref2src.txt' % (start, end), np.array(trans_all).reshape(-1, 4),
                       fmt='%0.8f')

        else:
            err_all = [err_R, err_T]
            plt.figure("ERR_R src2ref")  # 图像窗口名称
            plt.plot(err_R)
            plt.savefig(save_path + '/%s_%serr_All_src2ref.jpg' % (start, end))
            # plt.show()
            plt.close()
            plt.figure("ERR_T src2ref")  # 图像窗口名称
            plt.plot(err_T)
            plt.savefig(save_path + '/%s_%strans_all_src2ref.jpg' % (start, end))
            # plt.show()
            plt.close()
            np.savetxt(save_path + '/%s_%s_fail_list_src2ref.txt' % (start, end), fail_list)
            np.savetxt(save_path + '/%s_%serr_All_src2ref.txt' % (start, end), err_all)
            np.save(save_path + '/%s_%serr_All_src2ref.npy' % (start, end), err_all)
            np.save(save_path + '/%s_%strans_all_src2ref.npy' % (start, end), trans_all)
            np.savetxt(save_path + '/%s_%strans_all_src2ref.txt' % (start, end), np.array(trans_all).reshape(-1, 4),
                       fmt='%0.8f')


"""
Methods and tools used throughout the package, which do not pertain exclusively to data or simulation.
"""
from ._mathfuncs import *
from ._alignment import *
from ._savetools import *
from ._readtools import *
from ._ald import *
from ._displaying import *

__all__ = ['Gauss',
           'ExpGauss',
           'GaussDiff',
           'align_by_max',
           'align_by_steep',
           'csv_to_dict',
           'dict_from_list',
           'dict_to_csv',
           'find_nearest',
           'list_to_array',
           'normalized',
           'precision',
           'printparamsexp',
           'roll_by_array_shift',
           'find_baseline',
           'save_fit',
           'traces_to_csv']


#!/usr/bin/env python

from __future__ import print_function
import argparse
import atexit
import os
from confluent_kafka import Consumer, KafkaError, TopicPartition, Producer, OFFSET_BEGINNING, OFFSET_END
import sys
import traceback
import time
import threading
from xmlrpclib import _datetime_type

verbose = False
simulated = False

def get_offsets_for_timestamps(client, name, list_partitions, rec_time):
    #print(list_partitions)
    tps = [TopicPartition(name, p, rec_time) for p in list_partitions]
    offset_metadata = client.offsets_for_times(tps)
    offsets = {}
    for met in offset_metadata:
        offsets[met.partition] = met.offset
    return offsets

def wait_for_threads():
    seconds = 0
    while threading.active_count() != 0:
        print ('Waiting for threads to exit.')
        time.sleep(1)
        seconds += 1
        if seconds == 10:
            sys.exit(0)

def main():
    parser = argparse.ArgumentParser(description="Add/remove containers to server.")

    parser.add_argument("-f", "--from-topic", help="Topic name", metavar="NAME", required=True)
    parser.add_argument("-F", "--from-broker", help="Kafka bootstrap broker source", metavar="hostname", required=True)
    parser.add_argument("-k", "--key-filter", help="Only copy messages with these terms in the key", metavar="TERM", action="append")
    parser.add_argument("--port", help="Kafka bootstrap broker port", metavar="port", type=int, default=9092)
    parser.add_argument("-p", "--partition", help="Topic partitions to consume from", metavar="partition", type=int, action="append")
    parser.add_argument("-o", "--offset", help="Copy all data or most recent", choices=['beginning', 'end'], default='beginning')
    parser.add_argument("-O", "--abs-offset", help="Absolute offset", type=int, default=0)
    parser.add_argument("-t", "--to-topic", help="Topic name", metavar="NAME", required=True)
    parser.add_argument("-T", "--to-broker", help="Kafka bootstrap broker destination", metavar="hostname")
    parser.add_argument("-v", "--verbose", help="Verbose output.", action="store_true")
    parser.add_argument("--execute", help="Actual.", action="store_true")

    parser.add_argument("-S", "--start-time", help="Starting timestamp.", type=int)
    parser.add_argument("-E", "--end-time", help="Ending timestamp.", type=int)

    args = parser.parse_args()

    global verbose
    global simulated

    verbose = args.verbose
    simulated = True
    if args.execute:
        simulated = False
    else:
        print("---------------------Simulation only, use --execute to copy for real---------------------")

    if verbose:
        print (args)

    if (not args.to_broker):
        args.to_broker = args.from_broker
    bootstrap_src = '{0}:{1}'.format(args.from_broker, args.port)
    bootstrap_dst = '{0}:{1}'.format(args.to_broker, args.port)

    try:
        consumer_conf = {'bootstrap.servers': bootstrap_src,
                'enable.auto.commit': 'false',
                'session.timeout.ms': 6000,
                'group.id': 'log_copier',
                'api.version.request': True,
                'default.topic.config': {'auto.offset.reset': 'earliest' if args.offset == 'beginning' else 'latest'}}

        consumer = Consumer(consumer_conf)
        partitions = args.partition if args.partition else consumer.list_topics(args.from_topic).topics.get(args.from_topic).partitions.keys()

        if args.abs_offset == 0:
            tps = [TopicPartition(args.from_topic, p, OFFSET_BEGINNING if args.offset == 'beginning' else OFFSET_END) for p in partitions]
        elif args.abs_offset > 0:
            tps = [TopicPartition(args.from_topic, p, args.abs_offset) for p in partitions]
        else:
            end_offs = get_offsets_for_timestamps(consumer, args.from_topic, args.partition, OFFSET_END)
            tps = [TopicPartition(args.from_topic, p, end_offs[p] + args.abs_offset) for p in partitions]
        #print(tps)

        consumer.assign(tps)

        producer = Producer({'bootstrap.servers': bootstrap_dst})

        if verbose:
            print ("Source topic: {0}".format(consumer.list_topics(args.from_topic).topics))
            print ("Destination topic: {0}".format(producer.list_topics(args.to_topic).topics))

        counter = 0
        copy_count = 0
        keep_running = True
        while keep_running:
            messages = consumer.consume(500, 5)
            for message in messages:
                if message is None:
                    continue
                if message.error():
                    if message.error().code() == KafkaError._PARTITION_EOF:
                        continue
                    else:
                        keep_running = False
                        print(msg.error())
                        break
                counter += 1
                print ("Read {:,} messages.".format(counter).rjust(200), end='\r', file=sys.stderr)
                if args.key_filter and message.key():
                    if not any(x in message.key() for x in args.key_filter):
                        continue

                time_type = message.timestamp()[0]
                time_stamp = message.timestamp()[1]
                if args.start_time:
                    if time_stamp < args.start_time:
                        continue

                if args.end_time:
                    if time_stamp > args.end_time:
                        continue

                copy_count += 1

                if verbose:
                    dateStr = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(message.timestamp()[1] / 1000))
                    print ("{} - {}".format(message.timestamp()[1], dateStr))
#                     print ("%s:%d:%d: key=%s value=%s" % (message.topic(), message.partition(),
#                                                   message.offset(), message.key(),
#                                                   message.value()))
                if not simulated:
                    producer.produce(topic=args.to_topic
                                  ,value=message.value()
                                  ,partition=message.partition()
                                  ,key=message.key()
                                  #,timestamp=time_stamp
                                  )

            producer.flush()

    except KeyboardInterrupt as e:
        print ("Stopped - copied {} messages".format(copy_count))
        consumer.close()
        producer.flush()

if __name__ == "__main__":
    atexit.register(wait_for_threads)
    main()


import tensorflow as tf

def get_tensor_shape(x):
    a = x.get_shape().as_list()
    b = [tf.shape(x)[i] for i in range(len(a))]
    def _select_one(aa, bb):
        if type(aa) is int:
            return aa
        else:
            return bb
    return [_select_one(aa, bb) for aa, bb in zip(a, b)]

def pass_net_nx(func, in_img, n):
    b, h, w, c, = get_tensor_shape(in_img)
    def _get_nx(x):
        s, r = x//n, x%n
        s = tf.cond(
            tf.equal(r, 0),
            lambda: s,
            lambda: s + 1,
        )
        return n*s
    nx_h = _get_nx(h)
    nx_w = _get_nx(w)
    def _get_rl_rr(x, nx):
        r = nx - x
        rl = r//2
        rr = r - rl
        return rl, rr
    in_img = tf.pad(in_img, [[0, 0], _get_rl_rr(h, nx_h), _get_rl_rr(w, nx_w), [0, 0]], mode='symmetric')
    in_img = tf.reshape(in_img, [b, nx_h, nx_w, c])
    out_img = func(in_img)
    out_img = tf.image.resize_image_with_crop_or_pad(out_img, h, w)
    return out_img

def sample_1d(
    img,   # [b, h, c]
    y_idx, # [b, n], 0 <= pos < h, dtpye=int32
):
    b, h, c = get_tensor_shape(img)
    b, n    = get_tensor_shape(y_idx)

    b_idx = tf.range(b, dtype=tf.int32) # [b]
    b_idx = tf.expand_dims(b_idx, -1)   # [b, 1]
    b_idx = tf.tile(b_idx, [1, n])      # [b, n]

    y_idx = tf.clip_by_value(y_idx, 0, h - 1) # [b, n]
    a_idx = tf.stack([b_idx, y_idx], axis=-1) # [b, n, 2]

    return tf.gather_nd(img, a_idx)

def interp_1d(
    img, # [b, h, c]
    y,   # [b, n], 0 <= pos < h, dtype=float32
):
    b, h, c = get_tensor_shape(img)
    b, n    = get_tensor_shape(y)

    y_0 = tf.floor(y) # [b, n]
    y_1 = y_0 + 1

    _sample_func = lambda y_x: sample_1d(
        img,
        tf.cast(y_x, tf.int32)
    )
    y_0_val = _sample_func(y_0) # [b, n, c]
    y_1_val = _sample_func(y_1)

    w_0 = y_1 - y # [b, n]
    w_1 = y - y_0

    w_0 = tf.expand_dims(w_0, -1) # [b, n, 1]
    w_1 = tf.expand_dims(w_1, -1)

    return w_0*y_0_val + w_1*y_1_val

def apply_rf(
    x,  # [b, s...]
    rf, # [b, k]
):
    b, *s, = get_tensor_shape(x)
    b, k,  = get_tensor_shape(rf)
    x = interp_1d(
        tf.expand_dims(rf, -1),                              # [b, k, 1]
        tf.cast((k - 1), tf.float32)*tf.reshape(x, [b, -1]), # [b, ?]
    ) # [b, ?, 1]
    return tf.reshape(x, [b] + s)

def get_l2_loss(a, b):
    return tf.reduce_mean(tf.square(a - b))

def quantize(
    img, # [b, h, w, c]
    s=255,
):
    _clip = lambda x: tf.clip_by_value(x, 0, 1)
    img = _clip(img)
    img = tf.round(s*img)/s
    img = _clip(img)
    return img

def rand_quantize(
    img, # [b, h, w, c]
    is_training,
):
    b, h, w, c, = get_tensor_shape(img)

    rand_bit  = tf.cast(tf.random_uniform([b, 1, 1, 1], minval=8, maxval=12, dtype=tf.int32), tf.float32)
    const_bit = tf.constant(8.0, tf.float32, [b, 1, 1, 1])

    bit = tf.cond(is_training, lambda: rand_bit, lambda: const_bit)
    s   = (2**bit) - 1

    return quantize(img, s)


import ConfigParser, os, string

def FormGeneralSetData(uideflist, uiname, pobjconst):
  config = uideflist.config
  dataset = uideflist.userlist.Dataset
  userobj = config.CreatePObjImplProxy('UserApp')
  sessionpath = config.SecurityContext.SessionDirectory

  sessionlist = os.listdir(sessionpath)

  for session in sessionlist:
    session_part = session.split('_')
    user_id = session_part[0]
    if user_id == 'root':
      Record = dataset.AddRecord()
      Record.session_id = session
      Record.user_id = user_id
      Record.UserName = user_id
      Record.branch_code = user_id
      Record.BranchName = user_id
    else:
      try:
        userobj.Key = user_id
        branch = userobj.LBranchLocation
        Record = dataset.AddRecord()
        Record.session_id = session
        Record.user_id = user_id
        Record.UserName = userobj.UserName
        Record.branch_code = branch.branch_code
        Record.BranchName = branch.BranchName
      except:
        os.remove(sessionpath + session)
        continue

  # stop further processing
  return 0


# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html

# useful for handling different item types with a single interface

# Scraped data --> items containers --> json,xml files
# Scraped data --> items containers --> pipelines --> SQL/Mongo database

import sqlite3

from itemadapter import ItemAdapter

class QuotetutorialPipeline:

    def __init__(self):
        self.create_connection()
        self.create_table()

    def create_connection(self):
        self.conn = sqlite3.connect('myquotes.db')
        self.curr = self.conn.cursor()

    def create_table(self):
        self.conn.execute("""DROP TABLE IF EXIST quotes_tb""")
        self.curr.execute("""create table quotes_tb(
                            title text,
                            author text,
                            tag text
                            )""")

    def process_item(self, item, spider):
        self.store_db(item)
        return item

    def store_db(self, item):
        self.conn.execute("""insert into quotes_tb values(?,?,?)""",(
            item['title'][0],
            item['author'][0],
            item['tag'][0]
        ))
        self.conn.commit()


#=============================================================================
#
#                                    parse_spec.py
#
# GENERAL DESCRIPTION
#
# EXTERNAL FUNCTIONS
#        None.
#
# INITIALIZATION AND SEQUENCING REQUIREMENTS
#        None.
#
#             Copyright (c) 2013  by Qualcomm Technologies, Inc.  All Rights Reserved.
#=============================================================================
from ezxml import Element

def c_stringify(s, double=False):
    # Prepend the string with its own length encoded
    #  as a 32-bit integer
    import struct
    s = struct.pack('<L', len(s)) + s
    col = 0
    ret = ''
    pre = ''
    for c in s:
        o = ord(c)
        if c == '\n':
            e = '\\n'               # Encode newline as \n
        elif o < 32 or o > 126:
            e = '\\%03o' % o        # Encode other non-printables as \<octal>
        elif c in r'\"':
            e = '\\' + c            # Encode backslash and double quote as \<char>
        else:
            e = c                   # Other printable characters are used directly
        if col+len(e) > 71:
            ret += '"\n'            # Terminate current line
            col = 0
        if col == 0:
            ret += pre+'"'          # Start new line
            col = len(pre)+1
        ret += e                    # Add encoded character
        col += len(e)
        if c == '\n':
            col = 80                # If we just added a new-line, force line break
        pre = '   '
    if col > 0:
        ret += '"'

    # If double, return with all backslashes doubled so that the resulting string
    #  is appropriate as a regular expression substitution.

    if double:
        ret = '\\\\'.join(ret.split('\\'))

    return (ret,len(s))

class XMLPassThrough(object):
    def __init__(self, __name, el=None):
        self.name = __name
        self.tag = __name
        self.el = el
    def walkdom(self, el):
        self.cfg.build_elements.append(el)
        return XMLPassThrough(self.name, el)

class QurtImageParse(Element):

    # Our top level wrapper around the Element class.
    # This allows us to selectively modify the behavior
    #  of the top level of the parser, and allows us
    #  to do arbitrary reads and writes of the raw XML
    #  parse data.

    def __init__(self,cfg):
        from machine_xml  import Machine_el
        from kernel_xml   import Kernel_el
        from physpool_xml import PhysicalPool_el, SrmResources_el
        from program_xml  import Program_el

        cfg.c_stringify = c_stringify

        Element.__init__(self,
                         'image',
                         XMLPassThrough('build'),
                         Machine_el,
                         Kernel_el,
                         PhysicalPool_el,
                         SrmResources_el,
                         Program_el)

        # Set "cfg" correctly on the entire tree, as an attribute
        #  on each parsing object.
        empty = dict()
        elist = [self]
        while elist:
            e = elist.pop()
            e.cfg = cfg
            elist.extend(getattr(e,'elements',empty).values())

    # Our walkdom() function gets control after the
    #  raw XML has been parsed but before we've done
    #  any specific interpretation of the XML.
    # This is the spot where we can do out-of-band
    #  parsing or modifying of the XML contents.
    # For now, what we do is pretty simple...  If
    #  no <build> element is found at the top level,
    #  we do nothing special.
    # If one or more <build> elements are found at
    #  the top level, we create a string representation
    #  for the XML file <build> elements in C syntax and store
    #  it along with its length into these attributes:
    #   self.cfg['XMLCSTRING']
    #   self.cfg['XMLLENGTH']
    #  After creating that, we strip the <build>
    #  elements from the XML tree.  It is expected
    #  that all parsing of <build> elements will
    #  be done at a later point by qurt-image-build.py.

    def walkdom(self, el):

        pcount = 0
        for program in el.getElementsByTagName('program'):
            pmap = program.ownerDocument.createElement('program_name_map')
            pmap.setAttribute('name', program.getAttribute('name'))
            pmap.setAttribute('number', '%d' % pcount)
            self.cfg.build_elements.append(pmap)
            pcount += 1

        ret = Element.walkdom(self, el)

        (cstr,len) = c_stringify(''.join(['<image>'] +
                                         [e.toxml('utf-8') for e in self.cfg.build_elements] +
                                         ['</image>']), double=True)

        self.cfg['XMLCSTRING'] = cstr
        self.cfg['XMLLENGTH'] = '%u' % len

        return ret

    # Our do_parse() function is the "public" method
    #  that the upper level calls which actually
    #  does the parsing.  All of the needed inputs
    #  are attached to self.cfg already.

    def do_parse(self):
        s = self.cfg.spec_file
        if not self.cfg.spec_is_string:
            f = open(s, 'rb')
            s = f.read()
            f.close()
        self.cfg.parsed = self.parse_xml_str(s)


import tensorflow as tf
import data_loader
import generator_deep
import shutil
import time
import math
import numpy as np
import argparse
import sys
import os
import random
import ast

tf.set_random_seed(10)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--top_k', type=int, default=5,
                        help='sample from top k predictions')
    parser.add_argument('--beta1', type=float, default=0.9,
                        help='hyperpara-Adam')
    parser.add_argument('--datapath', type=str, default="Data/movielen_20/movielen_20.csv",
                        help='data path')
    parser.add_argument('--save_dir', type=str, default="Models/ml20_baseline_16_emb64_bs256",
                        help='save dir path')
    parser.add_argument('--eval_iter', type=int, default=1000,
                        help='output evry x steps')
    parser.add_argument('--early_stop', type=int, default=10,
                        help='after x eval_iter early stop')
    parser.add_argument('--step', type=int, default=400000,
                        help='trainging step')
    parser.add_argument('--tt_percentage', type=float, default=0.2,
                        help='0.2 means 80% training 20% testing')
    parser.add_argument('--data_ratio', type=float, default=0.4,
                        help='real training data')
    parser.add_argument('--learning_rate', type=float, default=0.001,
                        help='learning rate')
    parser.add_argument('--L2', type=float, default=0,
                        help='L2 regularization')
    parser.add_argument('--dilation_count', type=int, default=16,
                        help='dilation count number')
    parser.add_argument('--method', type=str, default="from_scratch",
                        help='from_scratch, StackR, stackC, stackA')
    parser.add_argument('--load_model', type=ast.literal_eval, default=False,
                        help='whether loading pretrain model')
    parser.add_argument('--model_path', type=str, default="Models/",
                        help='load model path')
    parser.add_argument('--padid', type=int, default=0,
                        help='pad id')
    args = parser.parse_args()

    print(args)

    dl = data_loader.Data_Loader({'dir_name': args.datapath, 'padid': args.padid})
    all_samples = dl.item
    print(all_samples.shape)
    items = dl.item_dict
    print("len(items)",len(items))

    # Randomly shuffle data
    np.random.seed(10)
    shuffle_indices = np.random.permutation(np.arange(len(all_samples)))
    all_samples = all_samples[shuffle_indices]

    # Split train/test set
    dev_sample_index = -1 * int(args.tt_percentage * float(len(all_samples)))
    train_set, valid_set = all_samples[:dev_sample_index], all_samples[dev_sample_index:]

    random.seed(10)
    ratio = args.data_ratio
    train_set_len = len(train_set)
    train_index_set = set(list(range(train_set_len)))

    if ratio == 0.2:
        train_ratio = int(ratio * float(train_set_len))
        real_train_index_set = random.sample(list(train_index_set), train_ratio)
        real_train_set = train_set[real_train_index_set]
        train_set = np.array(real_train_set)
        print("real train len", len(train_set))
    elif ratio == 0.4:
        last_ratio = ratio - 0.2
        last_train_ratio = int(last_ratio * float(train_set_len))
        last_train_index_set = random.sample(list(train_index_set), last_train_ratio)
        last_train_set = train_set[last_train_index_set]

        remain_train_index_set = train_index_set - set(last_train_index_set)
        remain_len = len(remain_train_index_set)
        new_train_index_set = random.sample(list(remain_train_index_set), int(1.0/4.0 * float(remain_len)))
        new_train_set = train_set[new_train_index_set]

        real_train_set = np.concatenate((last_train_set, new_train_set), axis=0)
        train_set = np.array(real_train_set)
        print("real train len", len(train_set))
    elif ratio == 0.6:
        last_last_ratio = ratio - 0.2 - 0.2
        last_last_train_ratio = int(last_last_ratio * float(train_set_len))
        last_last_train_index_set = random.sample(list(train_index_set), last_last_train_ratio)
        last_last_train_set = train_set[last_last_train_index_set]

        remain_train_index_set = train_index_set - set(last_last_train_index_set)
        remain_len = len(remain_train_index_set)
        last_train_index_set = random.sample(list(remain_train_index_set), int(1.0/4.0 * float(remain_len)))
        last_train_set = train_set[last_train_index_set]
        real_train_set = np.concatenate((last_last_train_set, last_train_set), axis=0)

        remain_train_index_set = remain_train_index_set - set(last_train_index_set)
        remain_len = len(remain_train_index_set)
        new_train_index_set = random.sample(list(remain_train_index_set), int(1.0/3.0 * float(remain_len)))
        new_train_set = train_set[new_train_index_set]

        real_train_set = np.concatenate((real_train_set, new_train_set), axis=0)
        train_set = np.array(real_train_set)
        print("real train len", len(train_set))
    elif ratio == 0.8:
        last_last_ratio = ratio - 0.2 - 0.2 - 0.2
        last_last_train_ratio = int(last_last_ratio * float(train_set_len))
        last_last_train_index_set = random.sample(list(train_index_set), last_last_train_ratio)
        last_last_train_set = train_set[last_last_train_index_set]

        remain_train_index_set = train_index_set - set(last_last_train_index_set)
        remain_len = len(remain_train_index_set)
        last_train_index_set = random.sample(list(remain_train_index_set), int(1.0/4.0 * float(remain_len)))
        last_train_set = train_set[last_train_index_set]
        real_train_set = np.concatenate((last_last_train_set, last_train_set), axis=0)

        remain_train_index_set = remain_train_index_set - set(last_train_index_set)
        remain_len = len(remain_train_index_set)
        new_train_index_set = random.sample(list(remain_train_index_set), int(1.0/3.0 * float(remain_len)))
        new_train_set = train_set[new_train_index_set]
        real_train_set = np.concatenate((real_train_set, new_train_set), axis=0)

        remain_train_index_set = remain_train_index_set - set(new_train_index_set)
        remain_len = len(remain_train_index_set)
        new_train_index_set = random.sample(list(remain_train_index_set), int(1.0/2.0 * float(remain_len)))
        new_train_set = train_set[new_train_index_set]

        real_train_set = np.concatenate((real_train_set, new_train_set), axis=0)
        train_set = np.array(real_train_set)
        print("real train len", len(train_set))
    elif ratio == 1:
        train_set = np.array(train_set)
        print("real train len", len(train_set))
    else:
        train_ratio = int(ratio * float(train_set_len))
        real_train_index_set = random.sample(list(train_index_set), train_ratio)
        real_train_set = train_set[real_train_index_set]
        train_set = np.array(real_train_set)
        print("real train len", len(train_set))

    model_para = {
        'item_size': len(items),
        'dilated_channels': 64,
        'dilations': [1,4]*args.dilation_count,
        'step': args.step,
        'kernel_size': 3,
        'learning_rate': args.learning_rate,
        'L2': args.L2,
        'batch_size': 256,
        'load_model': args.load_model,
        'model_path': args.model_path,
        'method': args.method
    }

    print(model_para)

    if not os.path.exists(args.save_dir):
        os.makedirs(args.save_dir)

    itemrec = generator_deep.NextItNet_Decoder(model_para)
    itemrec.train_graph()
    optimizer = tf.train.AdamOptimizer(model_para['learning_rate'], beta1=args.beta1).minimize(itemrec.loss)
    itemrec.predict_graph(reuse=True)

    gpu_options = tf.GPUOptions(allow_growth=True)
    sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
    saver = tf.train.Saver(max_to_keep=1)

    init = tf.global_variables_initializer()
    sess.run(init)

    numIters = 1
    max_mrr = 0
    break_stick = 0
    early_stop = 0

    while(1):
        if break_stick == 1:
            break

        batch_no = 0
        batch_size = model_para['batch_size']

        while (batch_no + 1) * batch_size < train_set.shape[0]:

            start = time.time()

            item_batch = train_set[batch_no * batch_size: (batch_no + 1) * batch_size, :]
            _, loss = sess.run(
                [optimizer, itemrec.loss],
                feed_dict={
                    itemrec.itemseq_input: item_batch
                })

            end = time.time()

            if numIters % args.eval_iter == 0:
                print("-------------------------------------------------------train")
                print("LOSS: {}\tBATCH_NO: {}\t STEP:{}\t total_batches:{}".format(
                    loss, batch_no, numIters, train_set.shape[0] / batch_size))
                print("TIME FOR BATCH", end - start)
                print("TIME FOR EPOCH (mins)", (end - start) * (train_set.shape[0] / batch_size) / 60.0)

            batch_no += 1

            if numIters % args.eval_iter == 0:
                print("-------------------------------------------------------test")
                batch_no_test = 0
                batch_size_test = batch_size*1
                curr_preds_5 = []
                rec_preds_5 = []
                ndcg_preds_5 = []
                curr_preds_10 = []
                rec_preds_10 = []
                ndcg_preds_10 = []
                while (batch_no_test + 1) * batch_size_test < valid_set.shape[0]:
                    item_batch = valid_set[batch_no_test * batch_size_test: (batch_no_test + 1) * batch_size_test, :]
                    [probs_10, probs_5] = sess.run(
                        [itemrec.top_10, itemrec.top_5],
                        feed_dict={
                            itemrec.input_predict: item_batch
                        })
                    for bi in range(batch_size_test):
                        pred_items_10 = probs_10[1][bi][-1]
                        pred_items_5 = probs_5[1][bi][-1]

                        true_item = item_batch[bi][-1]
                        predictmap_5 = {ch : i for i, ch in enumerate(pred_items_5)}
                        pred_items_10 = {ch: i for i, ch in enumerate(pred_items_10)}

                        rank_5 = predictmap_5.get(true_item)
                        rank_10 = pred_items_10.get(true_item)
                        if rank_5 == None:
                            curr_preds_5.append(0.0)
                            rec_preds_5.append(0.0)
                            ndcg_preds_5.append(0.0)
                        else:
                            MRR_5 = 1.0/(rank_5+1)
                            Rec_5 = 1.0
                            ndcg_5 = 1.0 / math.log(rank_5 + 2, 2)
                            curr_preds_5.append(MRR_5)
                            rec_preds_5.append(Rec_5)
                            ndcg_preds_5.append(ndcg_5)
                        if rank_10 == None:
                            curr_preds_10.append(0.0)
                            rec_preds_10.append(0.0)
                            ndcg_preds_10.append(0.0)
                        else:
                            MRR_10 = 1.0/(rank_10+1)
                            Rec_10 = 1.0
                            ndcg_10 = 1.0 / math.log(rank_10 + 2, 2)
                            curr_preds_10.append(MRR_10)
                            rec_preds_10.append(Rec_10)
                            ndcg_preds_10.append(ndcg_10)

                    batch_no_test += 1

                mrr = sum(curr_preds_5) / float(len(curr_preds_5))
                mrr_10 = sum(curr_preds_10) / float(len(curr_preds_10))
                hit = sum(rec_preds_5) / float(len(rec_preds_5))
                hit_10 = sum(rec_preds_10) / float(len(rec_preds_10))
                ndcg = sum(ndcg_preds_5) / float(len(ndcg_preds_5))
                ndcg_10 = sum(ndcg_preds_10) / float(len(ndcg_preds_10))

                if mrr > max_mrr:
                    max_mrr = mrr

                    print("Save model!  mrr_5:", mrr)
                    print("Save model!  mrr_10:", mrr_10)
                    print("Save model!  hit_5:", hit)
                    print("Save model!  hit_10:", hit_10)
                    print("Save model!  ndcg_5:", ndcg)
                    print("Save model!  ndcg_10:", ndcg_10)
                    early_stop = 0
                    saver.save(sess, args.save_dir + "/{}_{}_{}_{}.ckpt".format(args.dilation_count, args.learning_rate, args.data_ratio, args.step))
                else:
                    print("mrr_5:", mrr)
                    print("mrr_10:", mrr_10)
                    print("hit_5:", hit)
                    print("hit_10:", hit_10)
                    print("ndcg_5:", ndcg)
                    print("ndcg_10:", ndcg_10)
                    early_stop += 1

            if numIters >= model_para['step']:
                break_stick = 1
                break
            if early_stop >= args.early_stop:
                break_stick = 1
                print("early stop!")
                break

            numIters += 1

if __name__ == '__main__':
    main()


# This application will read roster data in JSON format, parse the file,
# and then produce an SQLite database that contains a User, Course,
# and Member table and populate the tables from the data file.
#
# Store the role column in the Member table.
#
# Run the following SQL command:
# SELECT hex(User.name || Course.title || Member.role )
#     AS X
#     FROM User
#     JOIN Member
#     JOIN Course
#     ON User.id = Member.user_id
#     AND Member.course_id = Course.id
#     ORDER BY X
#
# Find the first row in the resulting record set and enter the long string
# that looks like 53656C696E613333
#
# (Hint: starts with 416)

import json
import sqlite3

db_connection = sqlite3.connect('data-samples/rosterdb.sqlite')
a_cursor = db_connection.cursor()

a_cursor.executescript('''
DROP TABLE IF EXISTS User;
DROP TABLE IF EXISTS Course;
DROP TABLE IF EXISTS Member;

CREATE TABLE User (
    id     INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
    name   TEXT UNIQUE
);
CREATE TABLE Course (
    id     INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
    title  TEXT UNIQUE
);
CREATE TABLE Member (
    user_id     INTEGER,
    course_id   INTEGER,
    role        INTEGER,
    PRIMARY KEY (user_id, course_id)
);
''')

requested_file = 'data-samples/roster_data.json'
extracted_content = open(requested_file).read()
the_content = json.loads(extracted_content)

for an_item in the_content:
    name = an_item[0]
    title = an_item[1]
    role = an_item[2]

    a_cursor.execute('''INSERT OR IGNORE INTO User (name)
        VALUES ( ? )''', (name, ) )
    a_cursor.execute('SELECT id FROM User WHERE name = ? ', (name, ) )
    user_id = a_cursor.fetchone()[0]

    a_cursor.execute('''INSERT OR IGNORE INTO Course (title)
        VALUES ( ? )''', (title, ) )
    a_cursor.execute('SELECT id FROM Course WHERE title = ? ', (title, ) )
    course_id = a_cursor.fetchone()[0]

    a_cursor.execute('''INSERT OR REPLACE INTO Member (user_id, course_id, role)
        VALUES ( ?, ?, ? )''', (user_id, course_id, role) )

    db_connection.commit()

    #
    #


from typing import Dict
from .descriptors import DescriptorsAPI, obj_yaml_request_builder, yaml_response_handler, accept_yaml_or_json_header_request_builder
from .resource_api_base import ResourceAPIBase, ListAPIMeta, ReadAPIMeta, CreateAPIMeta, UpdateAPIMeta, DeleteAPIMeta

class DescriptorTemplatesAPI(DescriptorsAPI):
    endpoint = 'api/catalog/descriptorTemplates'

    def __init__(self, base_client: 'TNCOClient'):
        if base_client.kami_address is not None:
            self.override_address = base_client.kami_address
        else:
            self.override_address = None
        super().__init__(base_client)

    def render(self, template_name: str, render_request: Dict) -> Dict:
        if render_request is None:
            render_request = {}
        endpoint = self._render_endpoint(template_name)
        request = obj_yaml_request_builder(render_request)
        request['headers']['Accept'] = 'application/yaml'
        request['method'] = 'POST'
        request['endpoint'] = endpoint
        if self.override_address is not None:
            request['override_address'] = self.override_address
        response = self.base_client.make_request(**request)
        return yaml_response_handler(response)

    def render_raw(self, template_name: str, render_request: Dict) -> str:
        if render_request is None:
            render_request = {}
        endpoint = self._render_raw_endpoint(template_name)
        request = obj_yaml_request_builder(render_request)
        request['headers']['Accept'] = 'text/plain'
        request['method'] = 'POST'
        request['endpoint'] = endpoint
        if self.override_address is not None:
            request['override_address'] = self.override_address
        response = self.base_client.make_request(**request)
        return response.text

    def _render_endpoint(self, template_name: str) -> str:
        return f'{self.endpoint}/{template_name}/render'

    def _render_raw_endpoint(self, template_name: str) -> str:
        return f'{self.endpoint}/{template_name}/render-raw'


t=int(input())
from fractions import Fraction
for _ in range(t):

    n=int(input())
    if(n==1):
        print(1,1)
    else:

        if(n%2==0):
            m=n//2
            c=m-1

        else:
            m=(n-1)//2
            c=m
        k=n-1
        print(1,10**(k-c))


#!/usr/bin/env python

import argparse

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '-i',
        '--csv',
        help="Provide your tab-delimited file.",
        required=True)
    parser.add_argument(
        '-o',
        '--outfile',
        help="Output file for merged csv filtered for freq <=0.1",
        default="default")
    args = parser.parse_args()
    tmpStr = ""
    if(args.outfile == "default"):
        tmpStr += args.csv.split('/')[-1].split('.')[0] + ".10pctfilt.csv"

    outfile = open(tmpStr, 'w')

    # You could encase this whole thing in a loop that would take all
    # arguments (sys.argv[1], sys.argv[2], etc) each as a file to read in and
    # then write them all out to the same file like
    thiscsv = open(args.csv, 'r')  # this would become thiscsv = open(args.csv, 'r')
    header = "Filename\t" + thiscsv.readline()
    outfile.write(header)

    for line in thiscsv:
        if line.strip().split('\t')[19] == '-':
            outfile.write(
                args.csv +
                '\t' +
                line)  # add the file args.csv as the first column
        elif float(line.strip().split('\t')[19]) <= 0.1:
            outfile.write(args.csv + '\t' + line)

    # Have the outer loop end here

    thiscsv.close()
    outfile.close()
    return

if(__name__ == "__main__"):
    main()


"""
Script to find field values that haven't been persisted on objects.

    bin/instance run find_fields_with_non_persisted_default_values.py

This script logs a detailed CSV report and a summary to var/log/, and displays
some progress info and stats on STDERR/STDOUT.
"""

from App.config import getConfiguration
from collections import Counter
from datetime import datetime
from opengever.base.default_values import get_persisted_value_for_field
from opengever.maintenance.debughelpers import setup_app
from opengever.maintenance.debughelpers import setup_option_parser
from opengever.maintenance.debughelpers import setup_plone
from plone import api
from plone.dexterity.utils import iterSchemataForType
from zope.component import getUtility
from zope.intid.interfaces import IIntIds
from zope.schema import getFieldsInOrder
import logging
import os
import sys
import transaction

logger = logging.getLogger('check_default_values')
logging.getLogger().setLevel(logging.INFO)
for handler in logging.getLogger().handlers:
    handler.setLevel(logging.INFO)

class NonPersistedValueFinder(object):

    CSV_HEADER = "intid;portal_type;path;created;missing_fields"
    SCHEMA_CACHE = {}
    FIELD_CACHE = {}

    def __init__(self):
        self.catalog = api.portal.get_tool('portal_catalog')
        self.intids = getUtility(IIntIds)

        self.stats = Counter()
        self.stats['by_field'] = Counter()

        ts = datetime.today().strftime('%Y-%m-%d_%H-%M-%S')
        self.csv_log_path = self.get_logfile_path(
            'find-nonpersistent-values-%s.csv' % ts)
        self.summary_log_path = self.get_logfile_path(
            'find-nonpersistent-values-summary-%s.log' % ts)

    def run(self):
        logger.info("Checking for non-persisted values...\n\n")

        all_brains = self.catalog.unrestrictedSearchResults()
        total = len(all_brains)

        with open(self.csv_log_path, 'w') as self.csv_log:
            with open(self.summary_log_path, 'w') as self.summary_log:
                self.csv_log.write(self.CSV_HEADER + '\n')

                for i, brain in enumerate(all_brains):
                    obj = brain.getObject()
                    missing_fields = self.check_for_missing_fields(obj)
                    self.update_stats(missing_fields)

                    if missing_fields:
                        self.write_csv_row(obj, missing_fields)

                    if i % 100 == 0:
                        logger.info("Progress: %s of %s objects\n" % (i, total))

                self.display_stats()

    def fields_with_default(self, schema):
        fields = []
        for name, field in getFieldsInOrder(schema):
            if field.__dict__.get('defaultFactory', field.__dict__.get('default')) is None:
                continue
            fields.append(field)
        return fields

    def check_for_missing_fields(self, obj):
        missing_fields = []
        portal_type = obj.portal_type

        if portal_type not in self.SCHEMA_CACHE:
            self.SCHEMA_CACHE[portal_type] = list(iterSchemataForType(portal_type))
        schemas = self.SCHEMA_CACHE[portal_type]

        for schema in schemas:
            if schema.__identifier__ not in self.FIELD_CACHE:
                self.FIELD_CACHE[schema.__identifier__] = self.fields_with_default(schema)
            fields = self.FIELD_CACHE[schema.__identifier__]

            for field in fields:
                name = field.getName()

                if name == 'changeNote':
                    # The changeNote field from p.a.versioningbehavior
                    # is a "fake" field - it never gets persisted, but
                    # written to request annotations instead
                    continue

                if name == 'reference_number':
                    # reference_number is a special field. It never gets
                    # set directly, but instead acts as a computed field
                    # for all intents and purposes.
                    continue

                try:
                    get_persisted_value_for_field(obj, field)
                except AttributeError:
                    missing_fields.append((schema.__identifier__, name))

        missing_fields.sort()
        return missing_fields

    def write_csv_row(self, obj, missing_fields):
        created = str(obj.created())
        intid = self.intids.queryId(obj)
        row = [
            str(intid),
            obj.portal_type,
            '/'.join(obj.getPhysicalPath()),
            created,
            str([f[1] for f in missing_fields]),
        ]
        self.csv_log.write(';'.join(row) + '\n')

    def update_stats(self, missing_fields):
        if missing_fields:
            self.stats['missing'] += 1

            for schema_name, field_name in missing_fields:
                self.stats['by_field'][(schema_name, field_name)] += 1
        else:
            self.stats['ok'] += 1

    def display_stats(self):

        def log(line):
            logger.info(line)
            self.summary_log.write(line)

        log("\n")

        log("Missing (by field):\n")
        stats_by_field = sorted(self.stats['by_field'].items())
        for (schema_name, field_name), count in stats_by_field:
            dotted_name = '.'.join((schema_name, field_name))
            log("  %-120s %s\n" % (dotted_name, count))

        log("\n")

        log("Summary (by object):\n")
        log("Missing: %s\n" % self.stats['missing'])
        log("OK: %s\n" % self.stats['ok'])

        log("\n")
        log("Detailed CSV report written to %s\n" % self.csv_log_path)
        log("Summary written to %s\n" % self.summary_log_path)

    def get_logfile_path(self, filename):
        log_dir = self.get_logdir()
        return os.path.join(log_dir, filename)

    def get_logdir(self):
        """Determine the log directory.
        This will be derived from Zope2's EventLog location, in order to not
        have to figure out the path to var/log/ ourselves.
        """
        zconf = getConfiguration()
        eventlog = getattr(zconf, 'eventlog', None)

        if eventlog is None:
            logger.error('')
            logger.error(
                "Couldn't find eventlog configuration in order to determine "
                "logfile location - aborting!")
            logger.error('')
            sys.exit(1)

        handler_factories = eventlog.handler_factories
        eventlog_path = handler_factories[0].section.path
        assert eventlog_path.endswith('.log')
        log_dir = os.path.dirname(eventlog_path)
        return log_dir

if __name__ == '__main__':
    app = setup_app()

    parser = setup_option_parser()
    (options, args) = parser.parse_args()

    plone = setup_plone(app, options)

    transaction.doom()

    finder = NonPersistedValueFinder()
    finder.run()


# _*_ coding:utf-8 _*_

from django.conf.urls import url, include
from views import OrgListView, AddAskView, OrgHomeView, OrgCourseView, OrgDescView, OrgTeacherView
from views import AddFavView, TeacherListView, TeacherDetailView
urlpatterns = [
    url(r'^list/$', OrgListView.as_view(), name='orglist'),
    url(r'^addask/$', AddAskView.as_view(), name='addask'),
    url(r'^home/(?P<org_id>\d+)/$', OrgHomeView.as_view(), name='home'),
    url(r'^course/(?P<org_id>\d+)/$', OrgCourseView.as_view(), name='course'),
    url(r'^desc/(?P<org_id>\d+)/$', OrgDescView.as_view(), name='desc'),
    url(r'^teacher/(?P<org_id>\d+)/$', OrgTeacherView.as_view(), name='teacher'),
    url(r'^add_fav/$', AddFavView.as_view(), name='add_fav'),
    url(r'^teacher/list/$', TeacherListView.as_view(), name='teacher_list'),
    url(r'^teacherdatail/(?P<teacher_id>\d+)/$', TeacherDetailView.as_view(), name='teacher_detail'),

]


from pint import models, toa , residuals, fitter
import matplotlib.pyplot as plt
from astropy import units
from astropy.visualization import quantity_support
import numpy as np

#%% Set matplotlib general parameters

#Make everything larger for readability on graphs
plt.rcParams['xtick.labelsize'] = 12
plt.rcParams['ytick.labelsize'] = 12
plt.rcParams['font.size'] = 14
#Set same fonts as my LaTeX document
plt.rcParams['font.family'] = 'STIXGeneral'
plt.rcParams['mathtext.fontset'] = 'stix'
#other plotting params
plt.rcParams['axes.grid'] = True
plt.rcParams["figure.figsize"] = [10,10]

#%%

# Import model
model = models.model_builder.get_model('J0332+5434_Grahams_initial_parameters.par')

# Import TOAs
TOAs = toa.get_TOAs('J0332+5434_Grahams_Observations.tim', planets=True)

# Compute pre-fit residuals
prefit_residuals = residuals.Residuals(TOAs, model)

# Plot them, with the jumps
fig3 = plt.figure()
ax3 = plt.gca()
ax3.errorbar(
    TOAs.get_mjds().value,
    prefit_residuals.time_resids.to(units.us),
    yerr=TOAs.get_errors().to(units.us),
    marker='o',markersize=0.5,
    color='black', elinewidth=0.5, capsize=0, linestyle='none')
ax3.vlines(56769, np.min(prefit_residuals.time_resids.to(units.us)),  np.max(prefit_residuals.time_resids.to(units.us)), linewidth=0.5, color='red', label='Jumps')
ax3.vlines(56794, np.min(prefit_residuals.time_resids.to(units.us)),  np.max(prefit_residuals.time_resids.to(units.us)),  linewidth=0.5, color='red')
ax3.vlines(57048, np.min(prefit_residuals.time_resids.to(units.us)),  np.max(prefit_residuals.time_resids.to(units.us)), linewidth=0.5, color='red')
ax3.vlines(57575, np.min(prefit_residuals.time_resids.to(units.us)),  np.max(prefit_residuals.time_resids.to(units.us)),  linewidth=0.5, color='red')
ax3.set_title("%s Pre-fit Timing Residuals" % model.PSR.value)
ax3.set_xlabel("MJD")
ax3.set_ylabel("Residual ($\mu$s)")
plt.legend()
plt.savefig('Pre_fit_residuals_Graham.pdf')
plt.close()

# Perform the fit
WLS_fit = fitter.WLSFitter(TOAs, model)
WLS_fit.set_fitparams('F0','F1', 'RAJ', 'DECJ', 'JUMP1', 'JUMP2', 'JUMP3')
WLS_fit.fit_toas()

fig4 = plt.figure()
ax4 = plt.gca()

# Plot the post-fit residuals
ax4.errorbar(TOAs.get_mjds().value, WLS_fit.resids.time_resids.to(units.us), yerr=TOAs.get_errors().to(units.us), color='black', elinewidth=0.05, capsize=0, marker='o',markersize=0.8, linestyle='none')
ax4.vlines(56769,  np.min(WLS_fit.resids.time_resids.to(units.us)),  np.max(WLS_fit.resids.time_resids.to(units.us)), linewidth=0.5, color='red', label='Jumps')
ax4.vlines(56794, np.min(WLS_fit.resids.time_resids.to(units.us)),  np.max(WLS_fit.resids.time_resids.to(units.us)), linewidth=0.5, color='red')
ax4.vlines(57048, np.min(WLS_fit.resids.time_resids.to(units.us)),  np.max(WLS_fit.resids.time_resids.to(units.us)), linewidth=0.5, color='red')
ax4.vlines(57575, np.min(WLS_fit.resids.time_resids.to(units.us)),  np.max(WLS_fit.resids.time_resids.to(units.us)), linewidth=0.5, color='red')
ax4.set_title("%s Post-Fit Timing Residuals" % model.PSR.value)
ax4.set_xlabel("MJD")
ax4.set_ylabel("Residual ($\mu$s)")
plt.legend()
#plt.savefig('Post_fit_residuals_Graham.pdf')
#plt.close()


import uuid

import oss2
import requests
from django.conf import settings
from django.core.management import BaseCommand
from django.db.models import Q

from car.models import CarModel, Car
from operation.models import Member, ChangeMobileRecord
from order.models import ParkingFee

expired = settings.ALIYUN_OSS_STS_EXPIRED_TIME
access_key = settings.ALIYUN_OSS_ACCESS_KEY_ID
secret_key = settings.ALIYUN_OSS_ACCESS_KEY_SECRET
endpoint = settings.ALIYUN_OSS_ENDPOINT
bucket_name = settings.ALIYUN_OSS_BUCKET
auth = oss2.Auth(access_key, secret_key)
bucket = oss2.Bucket(auth, endpoint, bucket_name)

START_STR = 'https://zerocar.oss'

CORRECT_OSS_PREFIX = 'https://zerocar-pub.oss-cn-shenzhen.aliyuncs.com/'

class Command(BaseCommand):
    help = '修复由于bucket变成私有所造成的脏数据'

    def add_arguments(self, parser):
        parser.add_argument('what', nargs='+', type=str)

    def handle(self, *args, **options):

        what = options['what'][0]
        print(what)
        if what == 'car_model':
            # 迁移车型
            car_model_list = CarModel.objects.filter(img_url__startswith=START_STR)
            print(f'need migreta {what} total==> {car_model_list.count()}')
            for car_model in car_model_list:
                self._update_model(car_model, 'img_url')
        elif what == 'parking_fee':
            # 迁移停车费报销图片
            parking_fee_list = ParkingFee.objects.filter(img_url__startswith=START_STR)
            print(f'need migreta {what} total==> {parking_fee_list.count()}')
            for parking_fee in parking_fee_list:
                self._update_model(parking_fee, 'img_url')

        elif what == 'car':
            # 迁移车
            car_list = Car.objects.filter(
                Q(img_url__startswith=START_STR) | Q(card_imgurl__startswith=START_STR)
                | Q(drlicense_imgurl__startswith=START_STR)
                | Q(business_imgurl__startswith=START_STR)
                | Q(traffic_imgurl__startswith=START_STR))
            print(f'need migreta {what} total==> {car_list.count()}')
            for car in car_list:
                for field in [
                        'img_url', 'card_imgurl', 'drlicense_imgurl', 'business_imgurl',
                        'traffic_imgurl'
                ]:
                    self._update_model(car, field)

        elif what == 'member':
            # 迁移用户
            member_list = Member.objects.filter(
                Q(card_front_imgurl__startswith=START_STR)
                | Q(card_back_imgurl__startswith=START_STR)
                | Q(drlicense_imgurl__startswith=START_STR))
            print(f'need migreta {what} total==> {member_list.count()}')
            for member in member_list:
                for field in ['card_front_imgurl', 'card_back_imgurl', 'drlicense_imgurl']:
                    self._update_model(member, field)

        elif what == 'change_mobile':
            # 迁移用户
            change_list = ChangeMobileRecord.objects.filter(
                Q(card_front_imgurl__startswith=START_STR)
                | Q(card_back_imgurl__startswith=START_STR)
                | Q(hand_card_imgurl__startswith=START_STR))
            print(f'need migreta {what} total==> {change_list.count()}')
            for c in change_list:
                for field in ['card_front_imgurl', 'card_back_imgurl', 'hand_card_imgurl']:
                    self._update_model(c, field)
        return 'done'

    def _upload_oss(self, file_name):
        # 上传至oss
        file_key = str(uuid.uuid5(uuid.NAMESPACE_URL, file_name))
        file_input = requests.get(file_name)
        res = bucket.put_object(file_key, file_input.content)
        if res.status == 200:
            oss_url = CORRECT_OSS_PREFIX + file_key
            return True, oss_url
        return False, None

    def _update_model(self, obj, img_field):
        # 更新数据表的img相关字段
        img_url = getattr(obj, img_field)
        if not img_url:
            return False
        if not img_url.startswith(START_STR):
            return False
        ok, oss_url = self._upload_oss(img_url)
        if ok:
            setattr(obj, img_field, oss_url)
            obj.save()
            print(f'{obj.pk} => {img_field} => {img_url}  fix success')
            return True
        print(f'{obj.pk} => {img_field} => {img_url}  fix fail')
        return False