"""
April 2018
Simulates the trajectory implementing a CZ gate.

June 2018
Included noise in the simulation.
"""
import time
import numpy as np
import qutip as qtp
from pycqed.measurement import detector_functions as det
from scipy.interpolate import interp1d
from pycqed.measurement.waveform_control_CC import waveform as wf
import scipy
import matplotlib.pyplot as plt
#np.set_printoptions(threshold=np.inf)

# operators
b = qtp.tensor(qtp.destroy(3), qtp.qeye(3))  # LSB is static qubit
a = qtp.tensor(qtp.qeye(3), qtp.destroy(3))
n_q0 = a.dag() * a
n_q1 = b.dag() * b
H_c=n_q0

'''alpha_q0 = -275e6 * 2*np.pi
alpha_q1 = -310e6 * 2*np.pi
w_q0 = 5.11e9 * 2*np.pi  # Higher frequency qubit (fluxing) qubit
w_q1 = 4.10e9 * 2*np.pi  # Lower frequency

J = 3.07e6 * 2 * np.pi  # coupling strength

# caracteristic timescales for jump operators
T1_q0=34e-6
T1_q1=42e-6
Tphi_q0_ket0toket0=0    # here useless parameters
Tphi_q0_ket1toket1=0
Tphi_q0_ket2toket2=0
Tphi_q1_ket0toket0=0
Tphi_q1_ket1toket1=0
T2_q0=23e-6   # these two are the coherence times for q0 and q1 as qubits
T2_q1=23e-6
Tphi_q0_sigmaZ_01=1/(-1/(2*T1_q0)+1/T2_q0)     # extracting Tphi which is not the Tphi above
Tphi_q0_sigmaZ_12=Tphi_q0_sigmaZ_01           # we will assume for the moment that the pure decoherence
                                              # is caused by wiggles in the frequency, which cause
                                              # a fluctuation half as large for 02 wrt 01 and 12
                                              # (ignoring the anharmonicity)
Tphi_q0_sigmaZ_02=Tphi_q0_sigmaZ_01/2
Tphi_q1_sigmaZ_01=1/(-1/(2*T1_q1)+1/T2_q1)'''

#scalefactor=1e6

# Hamiltonian
def coupled_transmons_hamiltonian(w_q0, w_q1, alpha_q0, alpha_q1, J):
    """
    Hamiltonian of two coupled anharmonic transmons.
    Because the intention is to tune one qubit into resonance with the other,
    the number of levels is limited.
        q1 -> static qubit, 3-levels
        q0 -> fluxing qubit, 3-levels

    intended avoided crossing:
        11 <-> 02     (q1 is the first qubit and q0 the second one)

    N.B. the frequency of q0 is expected to be larger than that of q1
        w_q0 > w_q1
        and the anharmonicities alpha negative
    """

    #psi02_11=qtp.tensor(qtp.basis(3,0),qtp.basis(3,2))*qtp.tensor(qtp.basis(3,1),qtp.basis(3,1)).dag()

    H_0 = w_q0 * n_q0 + w_q1 * n_q1 +  \
        1/2*alpha_q0*(a.dag()*a.dag()*a*a) + 1/2*alpha_q1*(b.dag()*b.dag()*b*b) + \
        J * (a.dag() + a) * (b + b.dag()) #np.sqrt(2)*J*(psi02_11+psi02_11.dag())

    return H_0

#H_0 = coupled_transmons_hamiltonian(w_q0=w_q0, w_q1=w_q1, alpha_q0=alpha_q0,alpha_q1=alpha_q1,J=J)

# target in the case with no noise
# note that the Hilbert space is H_q1 /otimes H_q0
# so the ordering of basis states below is 00,01,02,10,11,12,20,21,22
U_target = qtp.Qobj([[1, 0, 0, 0, 0, 0, 0, 0, 0],
                     [0, 1, 0, 0, 0, 0, 0, 0, 0],
                     [0, 0, -1, 0, 0, 0, 0, 0, 0],
                     [0, 0, 0, 1, 0, 0, 0, 0, 0],
                     [0, 0, 0, 0, -1, 0, 0, 0, 0],
                     [0, 0, 0, 0, 0, 1, 0, 0, 0],
                     [0, 0, 0, 0, 0, 0, 1, 0, 0],
                     [0, 0, 0, 0, 0, 0, 0, 1, 0],
                     [0, 0, 0, 0, 0, 0, 0, 0, 1]],
                    type='oper',
                    dims=[[3, 3], [3, 3]])
#U_target._type = 'oper'
U_target_diffdims = qtp.Qobj([[1, 0, 0, 0, 0, 0, 0, 0, 0],
                     [0, 1, 0, 0, 0, 0, 0, 0, 0],
                     [0, 0, -1, 0, 0, 0, 0, 0, 0],
                     [0, 0, 0, 1, 0, 0, 0, 0, 0],
                     [0, 0, 0, 0, -1, 0, 0, 0, 0],
                     [0, 0, 0, 0, 0, 1, 0, 0, 0],
                     [0, 0, 0, 0, 0, 0, 1, 0, 0],
                     [0, 0, 0, 0, 0, 0, 0, 1, 0],
                     [0, 0, 0, 0, 0, 0, 0, 0, 1]],
                    type='oper',
                    dims=[[9], [9]])     # otherwise average_gate_fidelity doesn't work

# if there is noise the target is the corresponding superoperator
U_super_target = qtp.to_super(U_target)

'''
remember that qutip uses the Liouville (matrix) representation for superoperators,
with column stacking.
This means that
rho_{xy,x'y'}=rho[3*x+y,3*x'+y']
rho_{xy,x'y'}=operator_to_vector(rho)[3*x+y+27*x'+9*y']				VERIFY
where xy is the row and x'y' is the column
'''

def plot(x_plot_vec,y_plot_vec,title='No title',xlabel='No xlabel',ylabel='No ylabel',legend_labels=list(),yscale='linear'):

    if isinstance(y_plot_vec,list):
        y_length=len(y_plot_vec)
    else:
        y_length=np.size(y_plot_vec)

    if legend_labels==[]:
        legend_labels=np.arange(y_length)

    for i in range(y_length):

        if isinstance(y_plot_vec[i],list):
            y_plot_vec[i]=np.array(y_plot_vec[i])
        if isinstance(legend_labels[i],int):
            legend_labels[i]=str(legend_labels[i])

        if len(x_plot_vec)==1:
            if isinstance(x_plot_vec[0],list):
                x_plot_vec[0]=np.array(x_plot_vec[0])
            plt.plot(x_plot_vec[0], y_plot_vec[i], label=legend_labels[i])
        else:
            if isinstance(x_plot_vec[i],list):
                x_plot_vec[i]=np.array(x_plot_vec[i])
            plt.plot(x_plot_vec[i], y_plot_vec[i], label=legend_labels[i])

    plt.legend()
    plt.title(title)
    plt.xlabel(xlabel)
    plt.ylabel(ylabel)
    plt.yscale(yscale)
    plt.show()

def jump_operators(T1_q0,T1_q1,Tphi_q0_ket0toket0,Tphi_q0_ket1toket1,Tphi_q0_ket2toket2,Tphi_q1_ket0toket0,Tphi_q1_ket1toket1,
					Tphi_q0_sigmaZ_01,Tphi_q0_sigmaZ_12,Tphi_q0_sigmaZ_02,Tphi_q1_sigmaZ_01,Tphi_q1_sigmaZ_12,Tphi_q1_sigmaZ_02):
    # time independent case
	c_ops=[]
	if T1_q0 != 0:
		c_ops.append(np.sqrt(1/T1_q0)*a)
	if T1_q1 != 0:
		c_ops.append(np.sqrt(1/T1_q1)*b)
	if Tphi_q0_ket0toket0 != 0:
		collapse=qtp.tensor(qtp.qeye(3),qtp.ket2dm(qtp.basis(3,0)))
		c_ops.append(np.sqrt(1/Tphi_q0_ket0toket0)*collapse)
	if Tphi_q0_ket1toket1 != 0:
		collapse=qtp.tensor(qtp.qeye(3),qtp.ket2dm(qtp.basis(3,1)))
		c_ops.append(np.sqrt(1/Tphi_q0_ket1toket1)*collapse)
	if Tphi_q0_ket2toket2 != 0:
		collapse=qtp.tensor(qtp.qeye(3),qtp.ket2dm(qtp.basis(3,2)))
		c_ops.append(np.sqrt(1/Tphi_q0_ket2toket2)*collapse)
	if Tphi_q1_ket0toket0 != 0:
		collapse=qtp.tensor(qtp.ket2dm(qtp.basis(3,0)),qtp.qeye(3))
		c_ops.append(np.sqrt(1/Tphi_q1_ket0toket0)*collapse)
	if Tphi_q1_ket1toket1 != 0:
		collapse=qtp.tensor(qtp.ket2dm(qtp.basis(3,1)),qtp.qeye(3))
		c_ops.append(np.sqrt(1/Tphi_q1_ket1toket1)*collapse)
	if Tphi_q0_sigmaZ_01 != 0:
		sigmaZinqutrit = qtp.Qobj([[1,0,0],
									[0,-1,0],
									[0,0,0]])
		collapse=qtp.tensor(qtp.qeye(3),sigmaZinqutrit)
		c_ops.append(np.sqrt(1/(2*Tphi_q0_sigmaZ_01))*collapse)
	if Tphi_q0_sigmaZ_12 != 0:
		sigmaZinqutrit = qtp.Qobj([[0,0,0],
									[0,1,0],
									[0,0,-1]])
		collapse=qtp.tensor(qtp.qeye(3),sigmaZinqutrit)
		c_ops.append(np.sqrt(1/(2*Tphi_q0_sigmaZ_12))*collapse)
	if Tphi_q0_sigmaZ_02 != 0:
		sigmaZinqutrit = qtp.Qobj([[1,0,0],
									[0,0,0],
									[0,0,-1]])
		collapse=qtp.tensor(qtp.qeye(3),sigmaZinqutrit)
		c_ops.append(np.sqrt(1/(2*Tphi_q0_sigmaZ_02))*collapse)
	if Tphi_q1_sigmaZ_01 != 0:
		sigmaZinqutrit = qtp.Qobj([[1,0,0],
									[0,-1,0],
									[0,0,0]])
		collapse=qtp.tensor(sigmaZinqutrit,qtp.qeye(3))
		c_ops.append(np.sqrt(1/(2*Tphi_q1_sigmaZ_01))*collapse)
	if Tphi_q1_sigmaZ_12 != 0:
		sigmaZinqutrit = qtp.Qobj([[0,0,0],
									[0,1,0],
									[0,0,-1]])
		collapse=qtp.tensor(sigmaZinqutrit,qtp.qeye(3))
		c_ops.append(np.sqrt(1/(2*Tphi_q1_sigmaZ_12))*collapse)
	if Tphi_q1_sigmaZ_02 != 0:
		sigmaZinqutrit = qtp.Qobj([[1,0,0],
									[0,0,0],
									[0,0,-1]])
		collapse=qtp.tensor(sigmaZinqutrit,qtp.qeye(3))
		c_ops.append(np.sqrt(1/(2*Tphi_q1_sigmaZ_02))*collapse)
	return c_ops

#c_ops=jump_operators(T1_q0,T1_q1,Tphi_q0_ket0toket0,Tphi_q0_ket1toket1,Tphi_q0_ket2toket2,Tphi_q1_ket0toket0,Tphi_q1_ket1toket1,
#					 Tphi_q0_sigmaZ_01,Tphi_q0_sigmaZ_12,Tphi_q0_sigmaZ_02,Tphi_q1_sigmaZ_01)

def c_ops_interpolating(T1_q0,T1_q1,Tphi01_q0_vec,Tphi01_q1):
    # case where the pure decoherence for qubit q0 is time dependent, or better pulse-amplitude dependent
    c_ops=[]

    if T1_q0 != 0:
        c_ops.append(np.sqrt(1/T1_q0)*a)

    if T1_q1 != 0:
        c_ops.append(np.sqrt(1/T1_q1)*b)

    if Tphi01_q1 != 0:                                 # we automatically put also the decoherence for 12 and 02
        sigmaZinqutrit = qtp.Qobj([[1,0,0],
                                    [0,-1,0],
                                    [0,0,0]])
        collapse=qtp.tensor(sigmaZinqutrit,qtp.qeye(3))
        c_ops.append(collapse*np.sqrt(1/(2*Tphi01_q1)))

        Tphi12_q1=Tphi01_q1
        sigmaZinqutrit = qtp.Qobj([[0,0,0],
                                    [0,1,0],
                                    [0,0,-1]])
        collapse=qtp.tensor(sigmaZinqutrit,qtp.qeye(3))
        c_ops.append(collapse*np.sqrt(1/(2*Tphi12_q1)))

        Tphi02_q1=Tphi01_q1/2
        sigmaZinqutrit = qtp.Qobj([[1,0,0],
                                    [0,0,0],
                                    [0,0,-1]])
        collapse=qtp.tensor(sigmaZinqutrit,qtp.qeye(3))
        c_ops.append(collapse*np.sqrt(1/(2*Tphi02_q1)))

    if Tphi01_q0_vec != []:                                 # we automatically put also the decoherence for 12 and 02
        sigmaZinqutrit = qtp.Qobj([[1,0,0],
                                    [0,-1,0],
                                    [0,0,0]])
        collapse=qtp.tensor(qtp.qeye(3),sigmaZinqutrit)
        c_ops.append([collapse,np.sqrt(1/(2*Tphi01_q0_vec))])

        Tphi12_q0_vec=Tphi01_q0_vec
        sigmaZinqutrit = qtp.Qobj([[0,0,0],
                                    [0,1,0],
                                    [0,0,-1]])
        collapse=qtp.tensor(qtp.qeye(3),sigmaZinqutrit)
        c_ops.append([collapse,np.sqrt(1/(2*Tphi12_q0_vec))])

        Tphi02_q0_vec=Tphi01_q0_vec/2
        sigmaZinqutrit = qtp.Qobj([[1,0,0],
                                    [0,0,0],
                                    [0,0,-1]])
        collapse=qtp.tensor(qtp.qeye(3),sigmaZinqutrit)
        c_ops.append([collapse,np.sqrt(1/(2*Tphi02_q0_vec))])

    return c_ops

def rotating_frame_transformation(U, t: float,
                                  w_q0: float=0, w_q1: float =0):
    """
    Transforms the frame of the unitary according to
        U' = U_{RF}*U*U_{RF}^dag
    with
        U_{RF} = e^{-i w_q0 a^dag a t } otimes e^{-i w_q1 b^dag b t }

    Args:
        U (QObj): Unitary to be transformed
        t (float): time at which to transform
        w_q0 (float): freq of frame for q0
        w_q1 (float): freq of frame for q1

    """
    U_RF = (1j*w_q0*n_q0*t).expm() * (1j*w_q1*n_q1*t).expm()

    U_prime = U_RF * U
    """ U_RF only on one side because that's the operator that
    satisfies the Schroedinger equation in the interaction picture.
    Anyway we won't use this function.
    In case we would need to rotate in the new picture the jump operators as well !
    """
    return U_prime

def phases_from_superoperator(U):
    """
    Returns the phases from the unitary or superoperator U
    """
    if U.type=='oper':
        phi_00 = np.rad2deg(np.angle(U[0, 0]))  # expected to equal 0 because of our
        # choice for the energy, not because of rotating frame
        phi_01 = np.rad2deg(np.angle(U[1, 1]))
        phi_10 = np.rad2deg(np.angle(U[3, 3]))
        phi_11 = np.rad2deg(np.angle(U[4, 4]))
        phi_02 = np.rad2deg(np.angle(U[2, 2]))  # used only for avgatefid_superoperator_phasecorrected
        phi_20 = np.rad2deg(np.angle(U[6, 6]))

        phi_cond = (phi_11 - phi_01 - phi_10 + phi_00) % 360
        # notice the + even if it is irrelevant

        return phi_00, phi_01, phi_10, phi_11, phi_02, phi_20, phi_cond
    elif U.type=='super':
        phi_00 = 0   # we set it to 0 arbitrarily but it is actually not knowable
        phi_01 = np.rad2deg(np.angle(U[1, 1]))   # actually phi_01-phi_00
        phi_10 = np.rad2deg(np.angle(U[3, 3]))
        phi_11 = np.rad2deg(np.angle(U[4, 4]))
        phi_02 = np.rad2deg(np.angle(U[2, 2]))
        phi_20 = np.rad2deg(np.angle(U[6, 6]))

        phi_cond = (phi_11 - phi_01 - phi_10 + phi_00) % 360  # still the right formula
        # independently from phi_00

        return phi_00, phi_01, phi_10, phi_11, phi_02, phi_20, phi_cond
    # !!! check that this is a good formula for superoperators: there is a lot of redundancy
    #     there if the evolution is unitary, but not necessarily if it's noisy!

def pro_avfid_superoperator_compsubspace(U,L1):
    """
    Average process (gate) fidelity in the qubit computational subspace for two qutrits
    Leakage has to be taken into account, see Woods & Gambetta
    """

    if U.type=='oper':
        inner = U.dag()*U_target
        part_idx = [0, 1, 3, 4]  # only computational subspace
        ptrace = 0
        for i in part_idx:
            ptrace += inner[i, i]
        dim = 4  # 2 qubits comp subspace

        return np.real(((np.abs(ptrace))**2+dim*(1-L1))/(dim*(dim+1)))

    elif U.type=='super':
        kraus_form = qtp.to_kraus(U)
        dim=4 # 2 qubits in the computational subspace
        part_idx = [0, 1, 3, 4]  # only computational subspace
        psum=0
        for A_k in kraus_form:
        	ptrace = 0
        	inner = U_target_diffdims.dag()*A_k # otherwise dimension mismatch
        	for i in part_idx:
        		ptrace += inner[i, i]
        	psum += (np.abs(ptrace))**2

        return np.real((dim*(1-L1) + psum) / (dim*(dim + 1)))

def pro_avfid_superoperator_compsubspace_phasecorrected(U,L1,phases):
    """
    Average process (gate) fidelity in the qubit computational subspace for two qutrits
    Leakage has to be taken into account, see Woods & Gambetta
    The phase is corrected with Z rotations considering both transmons as qubits
    """

    Ucorrection = qtp.Qobj([[np.exp(-1j*np.deg2rad(phases[0])), 0, 0, 0, 0, 0, 0, 0, 0],
                     [0, np.exp(-1j*np.deg2rad(phases[1])), 0, 0, 0, 0, 0, 0, 0],
                     [0, 0, np.exp(-1j*np.deg2rad(phases[0])), 0, 0, 0, 0, 0, 0],
                     [0, 0, 0, np.exp(-1j*np.deg2rad(phases[2])), 0, 0, 0, 0, 0],
                     [0, 0, 0, 0, np.exp(-1j*np.deg2rad(phases[3]-phases[-1])), 0, 0, 0, 0],
                     [0, 0, 0, 0, 0, np.exp(-1j*np.deg2rad(phases[2])), 0, 0, 0],
                     [0, 0, 0, 0, 0, 0, np.exp(-1j*np.deg2rad(phases[0])), 0, 0],
                     [0, 0, 0, 0, 0, 0, 0, np.exp(-1j*np.deg2rad(phases[1])), 0],
                     [0, 0, 0, 0, 0, 0, 0, 0, np.exp(-1j*np.deg2rad(phases[0]))]],
                    type='oper',
                    dims=[[3, 3], [3, 3]])

    if U.type=='oper':
        U=Ucorrection*U
        inner = U.dag()*U_target
        part_idx = [0, 1, 3, 4]  # only computational subspace
        ptrace = 0
        for i in part_idx:
            ptrace += inner[i, i]
        dim = 4  # 2 qubits comp subspace

        return np.real(((np.abs(ptrace))**2+dim*(1-L1))/(dim*(dim+1)))

    elif U.type=='super':
        U=qtp.to_super(Ucorrection)*U
        kraus_form = qtp.to_kraus(U)
        dim=4 # 2 qubits in the computational subspace
        part_idx = [0, 1, 3, 4]  # only computational subspace
        psum=0
        for A_k in kraus_form:
        	ptrace = 0
        	inner = U_target_diffdims.dag()*A_k # otherwise dimension mismatch
        	for i in part_idx:
        		ptrace += inner[i, i]
        	psum += (np.abs(ptrace))**2

        return np.real((dim*(1-L1) + psum) / (dim*(dim + 1)))

def leakage_from_superoperator(U):
    if U.type=='oper':
        """
        Calculates leakage by summing over all in and output states in the
        computational subspace.
            L1 = 1- 1/2^{number computational qubits} sum_i sum_j abs(|<phi_i|U|phi_j>|)**2
        """
        sump = 0
        for i in range(4):
            for j in range(4):
                bra_i = qtp.tensor(qtp.ket([i//2], dim=[3]),
                                   qtp.ket([i % 2], dim=[3])).dag()
                ket_j = qtp.tensor(qtp.ket([j//2], dim=[3]),
                                   qtp.ket([j % 2], dim=[3]))
                p = np.abs((bra_i*U*ket_j).data[0, 0])**2
                sump += p
        sump /= 4  # divide by dimension of comp subspace
        L1 = 1-sump
        return L1
    elif U.type=='super':
        """
        Calculates leakage by summing over all in and output states in the
        computational subspace.
            L1 = 1- 1/2^{number computational qubits} sum_i sum_j Tr(rho_{x'y'}C_U(rho_{xy}))
            where C is U in the channel representation
        """
        sump = 0
        for i in range(4):
            for j in range(4):
                ket_i = qtp.tensor(qtp.ket([i//2], dim=[3]),
                                   qtp.ket([i % 2], dim=[3])) #notice it's a ket
                rho_i=qtp.operator_to_vector(qtp.ket2dm(ket_i))
                ket_j = qtp.tensor(qtp.ket([j//2], dim=[3]),
                                   qtp.ket([j % 2], dim=[3]))
                rho_j=qtp.operator_to_vector(qtp.ket2dm(ket_j))
                p = (rho_i.dag()*U*rho_j).data[0, 0]
                sump += p
        sump /= 4  # divide by dimension of comp subspace
        sump=np.real(sump)
        L1 = 1-sump
        return L1

def seepage_from_superoperator(U):
    """
    Calculates seepage by summing over all in and output states outside the
    computational subspace.
        L1 = 1- 1/2^{number non-computational states} sum_i sum_j abs(|<phi_i|U|phi_j>|)**2
    """
    if U.type=='oper':
        sump = 0
        for i_list in [[0,2],[1,2],[2,0],[2,1],[2,2]]:
            for j_list in [[0,2],[1,2],[2,0],[2,1],[2,2]]:
                bra_i = qtp.tensor(qtp.ket([i_list[0]], dim=[3]),
                                   qtp.ket([i_list[1]], dim=[3])).dag()
                ket_j = qtp.tensor(qtp.ket([j_list[0]], dim=[3]),
                                   qtp.ket([j_list[1]], dim=[3]))
                p = np.abs((bra_i*U*ket_j).data[0, 0])**2  # could be sped up
                sump += p
        sump /= 5  # divide by number of non-computational states
        L1 = 1-sump
        return L1
    elif U.type=='super':
        sump = 0
        for i_list in [[0,2],[1,2],[2,0],[2,1],[2,2]]:
            for j_list in [[0,2],[1,2],[2,0],[2,1],[2,2]]:
                ket_i = qtp.tensor(qtp.ket([i_list[0]], dim=[3]),
                                   qtp.ket([i_list[1]], dim=[3]))
                rho_i=qtp.operator_to_vector(qtp.ket2dm(ket_i))
                ket_j = qtp.tensor(qtp.ket([j_list[0]], dim=[3]),
                                   qtp.ket([j_list[1]], dim=[3]))
                rho_j=qtp.operator_to_vector(qtp.ket2dm(ket_j))
                p = (rho_i.dag()*U*rho_j).data[0, 0]
                sump += p
        sump /= 5  # divide by number of non-computational states
        sump=np.real(sump)
        L1 = 1-sump
        return L1

def pro_avfid_superoperator(U):
    """
    Average process (gate) fidelity in the whole space for two qutrits
    """
    if U.type=='oper':
        ptrace = np.abs((U.dag()*U_target).tr())**2
        dim = 9  # dimension of the whole space
        return np.real((ptrace+dim)/(dim*(dim+1)))

    elif U.type=='super':
        return np.real(qtp.average_gate_fidelity(U,target=U_target_diffdims))

def pro_avfid_superoperator_phasecorrected(U,phases):
    """
    Average process (gate) fidelity in the whole space for a qubit and qutrit
    Qubit Z rotation and qutrit "Z" rotations are applied, taking into account the anharmonicity as well
    """
    Ucorrection = qtp.Qobj([[np.exp(-1j*np.deg2rad(phases[0])), 0, 0, 0, 0, 0, 0, 0, 0],
                     [0, np.exp(-1j*np.deg2rad(phases[1])), 0, 0, 0, 0, 0, 0, 0],
                     [0, 0, np.exp(-1j*np.deg2rad(phases[4]-phases[-1])), 0, 0, 0, 0, 0, 0],
                     [0, 0, 0, np.exp(-1j*np.deg2rad(phases[2])), 0, 0, 0, 0, 0],
                     [0, 0, 0, 0, np.exp(-1j*np.deg2rad(phases[3]-phases[-1])), 0, 0, 0, 0],
                     [0, 0, 0, 0, 0, np.exp(-1j*np.deg2rad(phases[4]-phases[-1]+phases[2]-phases[0])), 0, 0, 0],
                     [0, 0, 0, 0, 0, 0, np.exp(-1j*np.deg2rad(phases[5])), 0, 0],
                     [0, 0, 0, 0, 0, 0, 0, np.exp(-1j*np.deg2rad(phases[5]+phases[1]-phases[0])), 0],
                     [0, 0, 0, 0, 0, 0, 0, 0, np.exp(-1j*np.deg2rad(phases[4]-phases[-1]+phases[5]-phases[0]))]],
                    type='oper',
                    dims=[[3, 3], [3, 3]])

    if U.type=='oper':
        U=Ucorrection*U
        ptrace = np.abs((U.dag()*U_target).tr())**2
        dim = 9  # dimension of the whole space
        return np.real((ptrace+dim)/(dim*(dim+1)))

    elif U.type=='super':
        U=qtp.to_super(Ucorrection)*U
        return np.real(qtp.average_gate_fidelity(U,target=U_target_diffdims))

'''
# benchmark for phase correction
gamma=2j*np.pi/3
alpha=2j*np.pi
beta=2j*np.pi/2
phi=2j*np.pi/4
delta=2j*np.pi/7
Uphases= qtp.Qobj([[np.exp(gamma), 0, 0, 0, 0, 0],
                     [0, np.exp(gamma+beta), 0, 0, 0, 0],
                     [0, 0, np.exp(gamma+delta+phi), 0, 0, 0],
                     [0, 0, 0, np.exp(gamma+alpha), 0, 0],
                     [0, 0, 0, 0, np.exp(gamma+alpha+beta+phi), 0],
                     [0, 0, 0, 0, 0, np.exp(gamma+alpha+delta)]],
                    type='oper',
                    dims=[[2, 3], [2, 3]])
print(Uphases)
phases=phases_from_superoperator(Uphases)
print(phases)
print(pro_avfid_superoperator_phasecorrected(Uphases,phases))
print(pro_avfid_superoperator_compsubspace_phasecorrected(Uphases,
	  leakage_from_superoperator(Uphases),phases))
'''

#tlist = np.arange(0, 240e-9, 1/2.4e9)

def matrix_change_of_variables(H_0):
    eigs,eigvectors=H_0.eigenstates()

    eigvectors_ordered_according2basis = []
    eigvectors_ordered_according2basis.append(eigvectors[0].full())   # 00 state
    eigvectors_ordered_according2basis.append(eigvectors[2].full())   # 01 state
    eigvectors_ordered_according2basis.append(eigvectors[5].full())   # 02 state
    eigvectors_ordered_according2basis.append(eigvectors[1].full())   # 10 state
    eigvectors_ordered_according2basis.append(eigvectors[4].full())   # 11 state
    eigvectors_ordered_according2basis.append(eigvectors[7].full())   # 12 state
    eigvectors_ordered_according2basis.append(eigvectors[3].full())   # 20 state
    eigvectors_ordered_according2basis.append(eigvectors[6].full())   # 21 state
    eigvectors_ordered_according2basis.append(eigvectors[8].full())   # 22 state

    S=np.hstack(eigvectors_ordered_according2basis)
    return S

def time_evolution(sim_step,eps_vec,H_0,c_ops,initial_propagator):
    '''scalefactor=1e6
    tlist_sim=tlist_sim*scalefactor
    eps_vec=eps_vec/scalefactor
    H_0=H_0/scalefactor
    if c_ops!=[]:       # c_ops is a list of either operators or lists where the first element is
                                    # an operator and the second one is a list of the (time-dependent) coefficients
        for c in range(len(c_ops)):
            if isinstance(c_ops[c],list):
                c_ops[c][1]=c_ops[c][1]/np.sqrt(scalefactor)
            else:
                c_ops[c]=c_ops[c]/np.sqrt(scalefactor)'''

    exp_L_total=initial_propagator

    #S = qtp.Qobj(matrix_change_of_variables(H_0),dims=[[3, 3], [3, 3]])
    '''
    if isinstance(tlist_sim,list):
        length_tlist=len(tlist_sim)
    else:
    	length_tlist=np.size(tlist_sim)'''
    for eps in eps_vec:
        H=H_0+eps*H_c    #(eps_vec[i+1]+eps_vec[i])/2
        #H=S*H*S.dag()
        c_ops_temp=[]
        if c_ops != []:
            for c in range(len(c_ops)):
                if isinstance(c_ops[c],list):
                   c_ops_temp.append(c_ops[c][0]*c_ops[c][1][i])
                else:
                   c_ops_temp.append(c_ops[c])
            liouville_exp_t=(qtp.liouvillian(H,c_ops_temp)*sim_step).expm()
        else:
        	liouville_exp_t=(-1j*H*sim_step).expm()
        exp_L_total=liouville_exp_t*exp_L_total
    return exp_L_total

def simulate_quantities_of_interest_superoperator(H_0, tlist, c_ops, eps_vec,
                                    sim_step,
                                    verbose: bool=True):
    """
    Calculates the quantities of interest from the propagator U

    Args:
        H_0 (Qobj): static hamiltonian, see "coupled_transmons_hamiltonian"
            for the expected form of the Hamiltonian.
        tlist (array): times in s, describes the x component of the
            trajectory to simulate
        c-ops (list of Qobj): list of jump operators, time-independent at the momennt
        eps_vec(array): detuning describes the y-component of the trajectory
            to simulate.

    Returns
        phi_cond (float):   conditional phase (deg)
        L1      (float):    leakage
        L2      (float):    seepage
        avgatefid (float):  average gate fidelity in full space
        avgatefid_compsubspace (float):  average gate fidelity only in the computational subspace

    """

    scalefactor=1  # otherwise qtp.propagator in parallel mode doesn't work
    # time is multiplied by scalefactor and frequency is divided by it
    tlist=tlist*scalefactor
    eps_vec=eps_vec/scalefactor
    sim_step=sim_step*scalefactor
    H_0=H_0/scalefactor
    if c_ops!=[]:       # c_ops is a list of either operators or lists where the first element is
                                    # an operator and the second one is a list of the (time-dependent) coefficients
        for c in range(len(c_ops)):
            if isinstance(c_ops[c],list):
                c_ops[c][1]=c_ops[c][1]/np.sqrt(scalefactor)
            else:
                c_ops[c]=c_ops[c]/np.sqrt(scalefactor)

    '''								# step of 1/sampling_rate=1/2.4e9=0.4 ns seems good by itself
    sim_step_new=sim_step*2

    eps_interp = interp1d(tlist, eps_vec, fill_value='extrapolate')
    tlist_new = (np.linspace(0, np.max(tlist), 576/2))
    eps_vec_new=eps_interp(tlist_new)

    c_ops_new=[]
    for c in range(len(c_ops)):
        if isinstance(c_ops[c],list):
            c_ops_interp=interp1d(tlist,c_ops[c][1], fill_value='extrapolate')
            c_ops_new.append([c_ops[c][0],c_ops_interp(tlist_new)])
        else:
            c_ops_new.append(c_ops[c])

    # function only exists to wrap
    #def eps_t(t, args=None):
    #    return eps_interp(t)
    print(len(eps_vec),len(eps_vec_new))

    t0 = time.time()

    exp_L_total_new=1
    for i in range(len(tlist_new)):
        H=H_0+eps_vec_new[i]*H_c
        c_ops_temp=[]
        for c in range(len(c_ops_new)):
            if isinstance(c_ops_new[c],list):
                c_ops_temp.append(c_ops_new[c][0]*c_ops_new[c][1][i])
            else:
                c_ops_temp.append(c_ops_new[c])
        liouville_exp_t=(qtp.liouvillian(H,c_ops_temp)*sim_step_new).expm()
        exp_L_total_new=liouville_exp_t*exp_L_total_new

    #exp_L_oneway=(qtp.liouvillian(H_0,c_ops)*240e-3).expm()

    t1 = time.time()
    print('\n alternative propagator_new',t1-t0)
    '''

    t0 = time.time()

    exp_L_total=1
    for i in range(len(tlist)):
        H=H_0+eps_vec[i]*H_c
        c_ops_temp=[]
        for c in range(len(c_ops)):
            if isinstance(c_ops[c],list):
                c_ops_temp.append(c_ops[c][0]*c_ops[c][1][i])
            else:
                c_ops_temp.append(c_ops[c])
        liouville_exp_t=(qtp.liouvillian(H,c_ops_temp)*sim_step).expm()
        exp_L_total=liouville_exp_t*exp_L_total

    #exp_L_oneway=(qtp.liouvillian(H_0,c_ops)*240e-3).expm()

    t1 = time.time()
    #print('\n alternative propagator',t1-t0)

    '''						# qutip propagator not used anymore because it takes too much time
    t0 = time.time()
    if c_ops==[]:
    	nstepsmax=1000
    else:
    	nstepsmax=100000
    H_t = [H_0, [H_c, eps_vec]]
    U_t = qtp.propagator(H_t, tlist, c_ops, parallel=True, options=qtp.Options(nsteps=nstepsmax))   # returns unitary 'oper' if c_ops=[], otherwise 'super'
    t1 = time.time()
    print('/n propagator',t1-t0)
    if verbose:
        print('simulation took {:.2f}s'.format(t1-t0))
    '''

    U_final = exp_L_total
    phases = phases_from_superoperator(U_final)
    phi_cond = phases[-1]
    L1 = leakage_from_superoperator(U_final)
    L2 = seepage_from_superoperator(U_final)
    avgatefid = pro_avfid_superoperator_phasecorrected(U_final,phases)
    avgatefid_compsubspace = pro_avfid_superoperator_compsubspace_phasecorrected(U_final,L1,phases)     # leakage has to be taken into account, see Woods & Gambetta
    #print('avgatefid_compsubspace',avgatefid_compsubspace)

    '''
    U_final = exp_L_total_new
    phases2 = phases_from_superoperator(U_final)
    phi_cond2 = phases2[-1]
    L12 = leakage_from_superoperator(U_final)
    L22 = seepage_from_superoperator(U_final)
    avgatefid2 = pro_avfid_superoperator_phasecorrected(U_final,phases2)
    avgatefid_compsubspace2 = pro_avfid_superoperator_compsubspace_phasecorrected(U_final,L12,phases2)

    print(phi_cond-phi_cond2,phi_cond)
    print(L1-L12,L1)
    print(L2-L22,L2)
    print(avgatefid-avgatefid2,avgatefid)
    print(avgatefid_compsubspace-avgatefid_compsubspace2,avgatefid_compsubspace)
    '''

    return {'phi_cond': phi_cond, 'L1': L1, 'L2': L2, 'avgatefid_pc': avgatefid, 'avgatefid_compsubspace_pc': avgatefid_compsubspace}

def simulate_quantities_of_interest_superoperator2(U_final):
    """
    Calculates the quantities of interest from the propagator U_final

    Args:
        U_final = propagator (either unitary or superoperator)

    Returns
        phi_cond (float):   conditional phase (deg)
        L1      (float):    leakage
        L2      (float):    seepage
        avgatefid (float):  average gate fidelity in full space
        avgatefid_compsubspace (float):  average gate fidelity only in the computational subspace

    """

    phases = phases_from_superoperator(U_final)
    phi_cond = phases[-1]
    L1 = leakage_from_superoperator(U_final)
    L2 = seepage_from_superoperator(U_final)
    avgatefid = pro_avfid_superoperator_phasecorrected(U_final,phases)
    avgatefid_compsubspace = pro_avfid_superoperator_compsubspace_phasecorrected(U_final,L1,phases)     # leakage has to be taken into account, see Woods & Gambetta

    return {'phi_cond': phi_cond, 'L1': L1, 'L2': L2, 'avgatefid_pc': avgatefid, 'avgatefid_compsubspace_pc': avgatefid_compsubspace}

### functions for FAQUAD

def gap_and_eigenvectors(H_0,Omega):     # returns gap between 11 and 02 and the corresponding eigenvectors
    H=H_0+Omega*H_c
    eigs,eigvectors=H.eigenstates()  # they're ordered from smallest to highest
    E_11=eigs[4]
    E_02=eigs[5]
    gap=np.abs(E_02-E_11)
    return gap, eigvectors[4], eigvectors[5]

def integrand_for_ctilde(H_0,Omega):
	dH_over_dOmega=H_c
	gap,psi11,psi02=gap_and_eigenvectors(H_0,Omega)
	scalprod=np.abs(psi11.dag().overlap(dH_over_dOmega*psi02))
	return scalprod/gap**2

def fix_ctilde(H_0,Omega_initial,Omega_final):
	result=scipy.integrate.quad(lambda Omega: integrand_for_ctilde(H_0,Omega), Omega_initial, Omega_final,epsabs=1e-14)
	return result

def create_epsvecFAQUAD(rampup_forward,rampup_backwards,nsimsteps_rampup,nsimsteps_interaction):
	t_list_rampup=np.arange(0,nsimsteps_rampup+1,1)
	t_list_rampup=t_list_rampup/nsimsteps_rampup
	eps_rampupforward=rampup_forward(t_list_rampup)

	eps_interaction=np.full(nsimsteps_interaction-1,rampup_forward([1])[0])

	eps_rampupbackwards=rampup_backwards(t_list_rampup)

	return np.concatenate((eps_rampupforward,eps_interaction,eps_rampupbackwards),axis=0)

def create_epsvecFAQUADandtlist(rampup_forward,rampup_backwards,nsimsteps_rampup,nsimsteps_interaction):
	tlist_sim=rampup_forward[0]*nsimsteps_rampup
	tlist_sim=np.concatenate((tlist_sim,np.array([nsimsteps_interaction+tlist_sim[-1]])),axis=0)
	tlist_sim=np.concatenate((tlist_sim,np.delete(rampup_backwards[0],0)*nsimsteps_rampup+tlist_sim[-1]),axis=0)

	epsvecFAQUAD=rampup_forward[1]
	epsvecFAQUAD=np.concatenate((epsvecFAQUAD,np.array([rampup_forward[1][-1]])),axis=0)
	epsvecFAQUAD=np.concatenate((epsvecFAQUAD,np.delete(rampup_backwards[1],0)),axis=0)

	return epsvecFAQUAD,tlist_sim

'''
Omega_initial=0
Omega_final=w_q1-alpha_q0-w_q0
ctilde=fix_ctilde(H_0,Omega_initial,Omega_final)

s_span=(0,1)
Omega0=np.array([0,])
def rhs_diffequ(t,Omega):
    dH_over_dOmega=H_c
    gap,psi11,psi02=gap_and_eigenvectors(H_0,Omega[0])
    scalprod=np.abs((psi11.dag()*dH_over_dOmega*psi02).data[0,0])
    if scalprod==0:
    	scalprod=1e-10
    return np.array([ctilde[0]*gap**2/scalprod,])
solution=scipy.integrate.solve_ivp(fun=rhs_diffequ, t_span=s_span, y0=Omega0)
print(solution)
print(Omega_final)

x_plot = solution.t
y_plot_a = solution.y[0]
import matplotlib.pyplot as plt
plt.plot(x_plot, y_plot_a, label='FAQUAD')
plt.legend()
plt.xlabel("s")
plt.ylabel("Omega")
plt.show()
'''

def compute_adiabatic_infidelities(U_final,eigvectors,eigvectors_H0):
	dim=len(eigvectors)

	columns=[]
	for i in range(dim):
		ket_i=eigvectors_H0[i]			# ordering of eigenstates is 00,10,01,20,11,02,21,12,22
		phi_i=U_final*ket_i
		columns.append(phi_i)

	adiab_infid=[]
	for i in range(dim):
		adiab_infid.append(1-np.abs(columns[i].dag().overlap(eigvectors[i]))**2)

	av_adiab_infid=0
	for i in range(dim):
		av_adiab_infid+=adiab_infid[i]
	av_adiab_infid=av_adiab_infid/dim

	adiab_infid.append(av_adiab_infid)
	return adiab_infid

def variance(H,ket):
	H_squared=np.abs(ket.dag().overlap(H*H*ket))
	av_value_squared=np.abs((ket.dag().overlap(H*ket)))**2
	return np.sqrt(H_squared/av_value_squared-1)

def compute_variances(H,U_final,eigvectors_H0):
    dim=len(eigvectors_H0)

    columns=[]
    for i in range(dim):
        ket_i=eigvectors_H0[i]			# ordering of eigenstates is 00,10,01,20,11,02,21,12,22
        phi_i=U_final*ket_i
        columns.append(phi_i)

    variances=[]
    for i in range(dim):
        variances.append(variance(H,columns[i]))

    av_var=0
    for i in range(dim):
        av_var+=variances[i]
        av_var=av_var/dim
    variances.append(av_var)

    return variances

def spectrum(H_0,eps_vec):
	eigenvalues=[[],[],[],[],[],[],[],[],[]]
	for Omega in eps_vec:
		H=H_0+Omega*H_c
		eigs=H.eigenenergies()
		for i in range(len(eigs)):
			eigenvalues[i].append(eigs[i])
	return eigenvalues

def compute_phases(U_final,eigvectors_H0):
	dim=len(eigvectors_H0)

	columns=[]
	for i in range(dim):
		ket_i=eigvectors_H0[i]			# ordering of eigenstates is 00,10,01,20,11,02,21,12,22
		phi_i=U_final*ket_i
		columns.append(phi_i)

	phases=[]
	for i in range(dim):
	    phases.append((np.rad2deg(np.angle(eigvectors_H0[i].dag().overlap(columns[i]))))%360)

	cond_phase=(phases[4]-phases[1]-phases[2]+phases[0])%360
	phases.append(cond_phase)

	return phases

def compute_cz_time(missing_phase,H):
    eigs,eigvectors=H.eigenstates()
    E_00=eigs[0]
    E_10=eigs[1]
    E_01=eigs[2]
    E_11=eigs[4]
    #E_02=eigs[4]
    gap=E_11-E_01-E_10+E_00
    return -missing_phase/360*(2*np.pi)/gap

class CZ_trajectory_superoperator(det.Soft_Detector):
    def __init__(self, H_0, fluxlutman, noise_parameters_CZ):
        """
        Detector for simulating a CZ trajectory.
        Args:
            fluxlutman (instr): an instrument that contains the parameters
                required to generate the waveform for the trajectory.
        """
        super().__init__()
        self.value_names = ['Cost func', 'Cond phase', 'L1', 'L2', 'avgatefid_pc', 'avgatefid_compsubspace_pc']
        self.value_units = ['a.u.', 'deg', '%', '%', '%', '%']
        self.fluxlutman = fluxlutman
        self.H_0 = H_0
        self.noise_parameters_CZ = noise_parameters_CZ

    def acquire_data_point(self, **kw):
        tlist = (np.arange(0, self.fluxlutman.cz_length(),
                           1/self.fluxlutman.sampling_rate()))
        if not self.fluxlutman.czd_double_sided():
            f_pulse = wf.martinis_flux_pulse(
                length=self.fluxlutman.cz_length(),
                lambda_2=self.fluxlutman.cz_lambda_2(),
                lambda_3=self.fluxlutman.cz_lambda_3(),
                theta_f=self.fluxlutman.cz_theta_f(),
                f_01_max=self.fluxlutman.cz_freq_01_max(),
                J2=self.fluxlutman.cz_J2(),
                f_interaction=self.fluxlutman.cz_freq_interaction(),
                sampling_rate=self.fluxlutman.sampling_rate(),
                return_unit='f01')
        else:
            f_pulse = self.get_f_pulse_double_sided()

        # extract base frequency from the Hamiltonian
        w_q0 = np.real(self.H_0[1,1])
        alpha_q0 = np.real(self.H_0[2,2]) - 2*w_q0
        w_q1 = np.real(self.H_0[3,3])
        eps_vec = f_pulse - w_q0

        sim_step=1/self.fluxlutman.sampling_rate()
        J2=self.fluxlutman.cz_J2()
        fundam_speedlimit=np.pi/(2*J2)

        Omega_initial=0
        Omega_final=w_q1-alpha_q0-w_q0

        scalefactor=1
        sim_step=sim_step*scalefactor
        Omega_final=Omega_final/scalefactor
        self.H_0=self.H_0/scalefactor

        ctilde=fix_ctilde(self.H_0,Omega_initial,Omega_final)

        s_span=(0,1)
        Omega0=np.array([Omega_initial,])
        #t_eval=np.linspace(0,1,1001)
        def rhs_diffequ(t,Omega):
            dH_over_dOmega=H_c
            gap,psi11,psi02=gap_and_eigenvectors(self.H_0,Omega[0])
            scalprod=np.abs(psi11.dag().overlap(dH_over_dOmega*psi02))
            if scalprod==0:
                scalprod=1e-10
            return np.array([ctilde[0]*gap**2/scalprod,])
        solution=scipy.integrate.solve_ivp(fun=rhs_diffequ, t_span=s_span, y0=Omega0,rtol=1e-8,atol=1e-14)

        rampup_forward=interp1d(solution.t,solution.y[0], fill_value=(0,solution.y[0][-1]))
        rampup_backwards=interp1d(1-solution.t[::-1],solution.y[0][::-1], fill_value=(solution.y[0][-1],0))
        #rampup_forward=[solution.t,solution.y[0]]
        #rampup_backwards=[1-solution.t[::-1],solution.y[0][::-1]]

        '''
        # BENCHMARK SPECTRUM
        eps_vec_FAQUAD=rampup_forward[1]
        eigenvalues=spectrum(self.H_0,eps_vec_FAQUAD)

        x_plot = rampup_forward[0]
        #x_plot_b = 1-solution.t[::-1]
        for i in [4]:
           y_plot_a = np.array(eigenvalues[5])-np.array(eigenvalues[4])
        #y_plot_b = solution.y[0][::-1]
           plt.plot(x_plot, y_plot_a, label='FAQUAD {}'.format(i))
        #plt.plot(x_plot_b, y_plot_b, label='Geller-Martinis')
        plt.legend()
        plt.xlabel("(Rampup) time (non-rescaled)")
        plt.ylabel("Gap 11-02")
        plt.yscale('log')
        plt.show()
        '''

        '''# BENCHMARK EVOLUTION OF STATES DURING SINGLE PASSAGE
        total_time=np.arange(1,252,50)
        division_of_tlist=1000    # 1000 is good
        tlist_partial=np.linspace(0,1,division_of_tlist)
        eps_vec_FAQUAD_partial=rampup_forward(tlist_partial)
        eigs_H0,eigvectors_H0=self.H_0.eigenstates()
        x_plot = tlist_partial
        c_ops=[]

        for T in total_time:
            U_final=1
            sim_step_temp=sim_step*T/division_of_tlist
            adiabatic_infidelities=[[],[],[],[],[],[],[],[],[],[]]
            variances=[[],[],[],[],[],[],[],[],[]]

            for x_elements in range(np.size(tlist_partial)):

                H=self.H_0+eps_vec_FAQUAD_partial[x_elements]*H_c

                initial_propagator=U_final
                U_final=time_evolution(sim_step_temp,[eps_vec_FAQUAD_partial[x_elements]],self.H_0,c_ops,initial_propagator)
                #U=qtp.propagator(H,sim_step_temp,c_ops)
                #U_final=U*U_final

                eigs,eigvectors=H.eigenstates()
                infids=compute_adiabatic_infidelities(U_final,eigvectors,eigvectors_H0)
                varsH=compute_variances(H,U_final,eigvectors_H0)

                for i in range(9+1):
                    adiabatic_infidelities[i].append(infids[i])
                for i in range(9):
                    variances[i].append(varsH[i])

            y_plot_a = np.array(adiabatic_infidelities[-1])
            plt.plot(x_plot, y_plot_a, label='T: {}'.format(T))

        plt.legend()
        plt.title("Av_infid different total time")
        plt.xlabel("s")
        plt.ylabel("Av_infid")
        plt.yscale('log')
        plt.show()'''

        c_ops=[]

        # BENCHMARK RAMPUP + RAMPDOWN no flat
        H=self.H_0+rampup_forward(np.array([1]))[0]*H_c
        eigs,eigvectors=H.eigenstates()
        eigs_H0,eigvectors_H0=self.H_0.eigenstates()

        subdivisions_of_simstep=20   # 40 is good, 10 is acceptable. No less
        rampuptime=np.arange(1,2502,10)

        adiabatic_infidelities=[[],[],[],[],[],[],[],[],[],[]]
        variances=[[],[],[],[],[],[],[],[],[],[]]

        adiabatic_infidelities_updown=[[],[],[],[],[],[],[],[],[],[]]
        variances_updown=[[],[],[],[],[],[],[],[],[],[]]
        phases_updown=[[],[],[],[],[],[],[],[],[],[]]
        cond_phases=[]
        leakage=[]
        avgateinfidpc=[]

        adiabatic_infidelities_full=[[],[],[],[],[],[],[],[],[],[]]
        variances_full=[[],[],[],[],[],[],[],[],[],[]]
        phases_full=[[],[],[],[],[],[],[],[],[],[]]
        cond_phases_full=[]
        leakage_full=[]
        avgateinfidpc_full=[]

        total_time=[]

        S = qtp.Qobj(matrix_change_of_variables(self.H_0),dims=[[3, 3], [3, 3]])

        print('how many rampup times =',np.size(rampuptime))

        for nsimsteps_rampup in rampuptime:

            t0=time.time()

            tlist_sim=np.linspace(0,1,nsimsteps_rampup*subdivisions_of_simstep)
            eps_vec_FAQUAD=rampup_forward(tlist_sim)
            sim_step_temp=sim_step/subdivisions_of_simstep

            initial_propagator=1
            U_final=time_evolution(sim_step_temp,eps_vec_FAQUAD,self.H_0,c_ops,initial_propagator)
            #H_t = [self.H_0, [H_c, eps_vec_FAQUAD]]
            #U_final=qtp.propagator(H_t,tlist_sim[-1],c_ops,nsteps=10000000000)

            infids=compute_adiabatic_infidelities(U_final,eigvectors,eigvectors_H0)
            varsH=compute_variances(H,U_final,eigvectors_H0)

            #eps_vec_FAQUAD_down=rampup_backwards(tlist_sim)
            #initial_propagator=1
            #U_final_down=time_evolution(sim_step_temp,eps_vec_FAQUAD_down,self.H_0,c_ops,initial_propagator)
            #U_final_updown=U_final_down*U_final

            U_final_updown=U_final.trans()*U_final
            #print(U_final_updown)
            infids_updown=compute_adiabatic_infidelities(U_final_updown,eigvectors_H0,eigvectors_H0)
            varsH_updown=compute_variances(self.H_0,U_final_updown,eigvectors_H0)
            ph_updown=compute_phases(U_final_updown,eigvectors_H0)

            qoi = simulate_quantities_of_interest_superoperator2(S*U_final_updown*S.dag())
            cond_phases.append(qoi['phi_cond'])
            leakage.append(qoi['L1'])
            avgateinfidpc.append(1-qoi['avgatefid_compsubspace_pc'])

            missing_phase=(180-qoi['phi_cond']) % 360
            missing_delta_t=compute_cz_time(missing_phase,H)
            U_flat=(-1j*H*missing_delta_t).expm()
            U_full=U_final.trans()*U_flat*U_final

            infids_full=compute_adiabatic_infidelities(U_full,eigvectors_H0,eigvectors_H0)
            varsH_full=compute_variances(self.H_0,U_full,eigvectors_H0)
            ph_full=compute_phases(U_full,eigvectors_H0)

            for i in range(9+1):
                adiabatic_infidelities[i].append(infids[i])
                adiabatic_infidelities_updown[i].append(infids_updown[i])
                phases_updown[i].append(ph_updown[i])
                adiabatic_infidelities_full[i].append(infids_full[i])
                phases_full[i].append(ph_full[i])
            for i in range(9+1):
                variances[i].append(varsH[i])
                variances_updown[i].append(varsH_updown[i])
                variances_full[i].append(varsH_full[i])

            qoi_full = simulate_quantities_of_interest_superoperator2(S*U_full*S.dag())
            cond_phases_full.append(qoi_full['phi_cond'])
            leakage_full.append(qoi_full['L1'])
            avgateinfidpc_full.append(1-qoi_full['avgatefid_compsubspace_pc'])

            total_time.append(2*nsimsteps_rampup*sim_step+missing_delta_t)

            t1=time.time()
            print("1rampup time =",t1-t0)

        ### only rampup

        x_plot = rampuptime*sim_step*1e9
        #x_plot_b = 1-solution.t[::-1]
        for i in range(10):
           y_plot_a = np.array(adiabatic_infidelities[i])
        #y_plot_b = solution.y[0][::-1]
           plt.plot(x_plot, y_plot_a, label='FAQUAD {}'.format(i))
        #plt.plot(x_plot_b, y_plot_b, label='Geller-Martinis')
        plt.legend()
        plt.title("Rampup only")
        plt.xlabel("Rampup time (ns)")
        plt.ylabel("Infidelity per state")
        plt.yscale('log')
        plt.show()

        x_plot = rampuptime*sim_step*1e9
        #x_plot_b = 1-solution.t[::-1]
        for i in range(1,10,1):
           y_plot_a = np.array(variances[i])
        #y_plot_b = solution.y[0][::-1]
           plt.plot(x_plot, y_plot_a, label='FAQUAD {}'.format(i))
        #plt.plot(x_plot_b, y_plot_b, label='Geller-Martinis')
        plt.legend()
        plt.title("Rampup only")
        plt.xlabel("Rampup time (ns)")
        plt.ylabel("Variance of H_0 over Av_value")
        plt.yscale('log')
        plt.show()

        ### rampup + rampdown

        x_plot = rampuptime*sim_step*1e9
        #x_plot_b = 1-solution.t[::-1]
        for i in range(10):
           y_plot_a = np.array(adiabatic_infidelities_updown[i])
        #y_plot_b = solution.y[0][::-1]
           plt.plot(x_plot, y_plot_a, label='FAQUAD {}'.format(i))
        #plt.plot(x_plot_b, y_plot_b, label='Geller-Martinis')
        plt.legend()
        plt.title("Rampup+rampdown")
        plt.xlabel("Rampup time (ns)")
        plt.ylabel("Infidelity per state")
        plt.yscale('log')
        plt.show()

        x_plot = rampuptime*sim_step*1e9
        #x_plot_b = 1-solution.t[::-1]
        for i in range(1,10,1):
           y_plot_a = np.array(variances_updown[i])
        #y_plot_b = solution.y[0][::-1]
           plt.plot(x_plot, y_plot_a, label='FAQUAD {}'.format(i))
        #plt.plot(x_plot_b, y_plot_b, label='Geller-Martinis')
        plt.legend()
        plt.title("Rampup+rampdown")
        plt.xlabel("Rampup time (ns)")
        plt.ylabel("Variance of H_0 over Av_value")
        plt.yscale('log')
        plt.show()

        ### rampup + flat + rampdown

        x_plot = rampuptime*sim_step*1e9
        #x_plot_b = 1-solution.t[::-1]
        for i in range(10):
           y_plot_a = np.array(adiabatic_infidelities_full[i])
        #y_plot_b = solution.y[0][::-1]
           plt.plot(x_plot, y_plot_a, label='FAQUAD {}'.format(i))
        #plt.plot(x_plot_b, y_plot_b, label='Geller-Martinis')
        plt.legend()
        plt.title("Rampup+flat+rampdown")
        plt.xlabel("Rampup time (ns)")
        plt.ylabel("Infidelity per state")
        plt.yscale('log')
        plt.show()

        x_plot = rampuptime*sim_step*1e9
        #x_plot_b = 1-solution.t[::-1]
        for i in range(1,10,1):
           y_plot_a = np.array(variances_full[i])
        #y_plot_b = solution.y[0][::-1]
           plt.plot(x_plot, y_plot_a, label='FAQUAD {}'.format(i))
        #plt.plot(x_plot_b, y_plot_b, label='Geller-Martinis')
        plt.legend()
        plt.title("Rampup+flat+rampdown")
        plt.xlabel("Rampup time (ns)")
        plt.ylabel("Variance of H_0 over Av_value")
        plt.yscale('log')
        plt.show()

        ### comparison

        plot(x_plot_vec=[rampuptime*sim_step*1e9],
                  y_plot_vec=[adiabatic_infidelities[9],adiabatic_infidelities_updown[9],adiabatic_infidelities_full[9]],
                  title='Comparison infidelities',
                  xlabel='Rampup time (ns)',ylabel='Infidelity per state',legend_labels=['up','updown','full'],yscale='log')

        plot(x_plot_vec=[rampuptime*sim_step*1e9],
                  y_plot_vec=[variances[9],variances_updown[9],variances_full[9]],
                  title='Comparison variances',
                  xlabel='Rampup time (ns)',ylabel='Variance of H_0 over Av_value',legend_labels=['up','updown','full'],yscale='log')

        '''x_plot = rampuptime*sim_step*1e9
        #x_plot_b = 1-solution.t[::-1]
        for i in range(10):
           y_plot_a = np.array(phases_updown[i])
        #y_plot_b = solution.y[0][::-1]
           plt.plot(x_plot, y_plot_a, label='FAQUAD {}'.format(i))
        #plt.plot(x_plot_b, y_plot_b, label='Geller-Martinis')
        plt.legend()
        plt.title("Rampup+rampdown no flat")
        plt.xlabel("Rampup time (ns)")
        plt.ylabel("Phases acquired")
        #plt.yscale('log')
        plt.show()'''

        #----------------------------------------------

        plot(x_plot_vec=[rampuptime*sim_step*1e9],
                  y_plot_vec=[cond_phases,cond_phases_full],
                  title='Conditional phase',
                  xlabel='Rampup time (ns)',ylabel='Cond_phase (deg)',legend_labels=['updown','full'])

        plot(x_plot_vec=[rampuptime*sim_step*1e9],
                  y_plot_vec=[np.array(total_time)*1e9],
                  title='Total pulse time = 2*rampuptime + time_at_the_top',
                  xlabel='Rampup time (ns)',ylabel='Total time (ns)',legend_labels=['full'])

        plot(x_plot_vec=[rampuptime*sim_step*1e9],
                  y_plot_vec=[leakage,leakage_full],
                  title='Leakage',
                  xlabel='Rampup time (ns)',ylabel='Leakage',legend_labels=['updown','full'],yscale='log')

        plot(x_plot_vec=[rampuptime*sim_step*1e9],
                  y_plot_vec=[avgateinfidpc,avgateinfidpc_full],
                  title='Average gate infidelity',
                  xlabel='Rampup time (ns)',ylabel='Av_gate_infid_comp_pc',legend_labels=['updown','full'],yscale='log')

        '''
        rampuptime=np.arange(1,1001,10)
        min_infidelity=[]
        for nsimsteps_rampup in rampuptime:
            infidelities=[]
            for nsimsteps_interaction in range(1,600,2):  #275-nsimsteps_rampup*2-25,275-nsimsteps_rampup*2+25,1
                eps_vec_FAQUAD,tlist_sim=create_epsvecFAQUADandtlist(rampup_forward,rampup_backwards,nsimsteps_rampup,nsimsteps_interaction)
                #tlist=np.arange(0,nsimsteps_rampup*2+nsimsteps_interaction+1,1)
                tlist_sim=tlist_sim*sim_step

                c_ops=[]

                initial_propagator=1
                U_final=time_evolution(tlist_sim,eps_vec_FAQUAD,H_0,c_ops,initial_propagator)

                qoi = simulate_quantities_of_interest_superoperator2(U_final=U_final)

                infidelities.append(1-qoi['avgatefid_compsubspace_pc'])
            #print(infidelities)

            x_plot = tlist_sim*1e9
            #x_plot_b = 1-solution.t[::-1]
            y_plot_a = eps_vec_FAQUAD
            #y_plot_b = solution.y[0][::-1]
            plt.plot(x_plot, y_plot_a, label='FAQUAD')
            #plt.plot(x_plot_b, y_plot_b, label='Geller-Martinis')
            plt.legend()
            plt.xlabel("ns")
            plt.ylabel("FAQUAD")
            #plt.yscale('log')
            plt.show()

            infidelities=np.array(infidelities)
            min_infidelity.append(np.amin(infidelities))
        min_infidelity=np.array(min_infidelity)
        print(np.amin(min_infidelity))
        '''

        T1_q0 = self.noise_parameters_CZ.T1_q0()
        T1_q1 = self.noise_parameters_CZ.T1_q1()
        T2_q0_sweetspot = self.noise_parameters_CZ.T2_q0_sweetspot()
        T2_q0_interaction_point = self.noise_parameters_CZ.T2_q0_interaction_point()
        T2_q1 = self.noise_parameters_CZ.T2_q1()

        def Tphi_from_T1andT2(T1,T2):
            return 1/(-1/(2*T1)+1/T2)

        if T2_q0_sweetspot != 0:
            Tphi01_q0_sweetspot=Tphi_from_T1andT2(T1_q0,T2_q0_sweetspot)
        else:
            Tphi01_q0_sweetspot=0
        if T2_q0_interaction_point != 0:
            Tphi01_q0_interaction_point=Tphi_from_T1andT2(T1_q0,T2_q0_interaction_point)
        else:
            Tphi01_q0_interaction_point=0
        # Tphi01=Tphi12=2*Tphi02
        if T2_q1 != 0:
            Tphi01_q1 = Tphi_from_T1andT2(T1_q1,T2_q1)
        else:
            Tphi01_q1=0

        def omega_prime(omega):                                   # derivative of f_pulse
            '''
            frequency is w = w_0 * cos(phi_e/2)    where phi_e is the external flux through the SQUID.
            So the derivative wrt phi_e is
                 w_prime = - w_0/2 sin(phi_e/2) = - w_0/2 * sqrt(1-cos(phi_e/2)**2) = - w_0/2 * sqrt(1-(w/w_0)**2)
            Note: no need to know what phi_e is.
            '''
            return np.abs((w_q0/2)*np.sqrt(1-(omega/w_q0)**2))    # we actually return the absolute value because it's the only one who matters later

        if Tphi01_q0_interaction_point != 0:       # mode where the pure dephazing is amplitude-dependent
            w_min = np.nanmin(f_pulse)
            omega_prime_min = omega_prime(w_min)

            f_pulse=np.clip(f_pulse,0,w_q0)
            f_pulse_prime = omega_prime(f_pulse)
            Tphi01_q0_vec = Tphi01_q0_sweetspot - f_pulse_prime/omega_prime_min*(Tphi01_q0_sweetspot-Tphi01_q0_interaction_point)
                     # we interpolate Tphi from the sweetspot to the interaction point (=worst point in terms of Tphi)
                     # by weighting depending on the derivative of f_pulse compared to the derivative at the interaction point
            c_ops = c_ops_interpolating(T1_q0,T1_q1,Tphi01_q0_vec,Tphi01_q1)
        else:                                       # mode where the collapse operators are time-independent, and possibly are 0
            c_ops=jump_operators(T1_q0,T1_q1,0,0,0,0,0,
                    Tphi01_q0_sweetspot,Tphi01_q0_sweetspot,Tphi01_q0_sweetspot/2,Tphi01_q1,Tphi01_q1,Tphi01_q1/2)

        qoi = simulate_quantities_of_interest_superoperator(
            H_0=self.H_0,
            tlist=tlist, c_ops=c_ops, eps_vec=eps_vec,
            sim_step=sim_step, verbose=False)

        cost_func_val = -np.log10(1-qoi['avgatefid_compsubspace_pc'])   # new cost function: infidelity
        #np.abs(qoi['phi_cond']-180) + qoi['L1']*100 * 5
        return cost_func_val, qoi['phi_cond'], qoi['L1']*100, qoi['L2']*100, qoi['avgatefid_pc']*100, qoi['avgatefid_compsubspace_pc']*100

    def get_f_pulse_double_sided(self):
        half_CZ_A = wf.martinis_flux_pulse(
            length=self.fluxlutman.cz_length()*self.fluxlutman.czd_length_ratio(),
            lambda_2=self.fluxlutman.cz_lambda_2(),
            lambda_3=self.fluxlutman.cz_lambda_3(),
            theta_f=self.fluxlutman.cz_theta_f(),
            f_01_max=self.fluxlutman.cz_freq_01_max(),
            J2=self.fluxlutman.cz_J2(),
            # E_c=self.fluxlutman.cz_E_c(),
            f_interaction=self.fluxlutman.cz_freq_interaction(),
            sampling_rate=self.fluxlutman.sampling_rate(),
            return_unit='f01')

        # Generate the second CZ pulse. If the params are np.nan, default
        # to the main parameter
        if not np.isnan(self.fluxlutman.czd_theta_f()):
            d_theta_f = self.fluxlutman.czd_theta_f()
        else:
            d_theta_f = self.fluxlutman.cz_theta_f()

        if not np.isnan(self.fluxlutman.czd_lambda_2()):
            d_lambda_2 = self.fluxlutman.czd_lambda_2()
        else:
            d_lambda_2 = self.fluxlutman.cz_lambda_2()
        if not np.isnan(self.fluxlutman.czd_lambda_3()):
            d_lambda_3 = self.fluxlutman.czd_lambda_3()
        else:
            d_lambda_3 = self.fluxlutman.cz_lambda_3()

        half_CZ_B = wf.martinis_flux_pulse(
            length=self.fluxlutman.cz_length()*(1-self.fluxlutman.czd_length_ratio()),
            lambda_2=d_lambda_2,
            lambda_3=d_lambda_3,
            theta_f=d_theta_f,
            f_01_max=self.fluxlutman.cz_freq_01_max(),
            J2=self.fluxlutman.cz_J2(),
            f_interaction=self.fluxlutman.cz_freq_interaction(),
            sampling_rate=self.fluxlutman.sampling_rate(),
            return_unit='f01')

        # N.B. No amp scaling and offset present
        f_pulse = np.concatenate([half_CZ_A, half_CZ_B])
        return f_pulse


"""
Interactive syllable segmentation

1) Relevant audio?
2) Boundary?
3) Good clusters?
"""

__author__ = "Jack Goffinet"
__date__ = "March-April 2019"

import os
import numpy as np
import h5py
import joblib
from tqdm import tqdm
from skimage.transform import resize, downscale_local_mean
from scipy.io import wavfile, loadmat
from scipy.signal import stft, welch

from time import sleep

# import matplotlib
# matplotlib.use('qt4agg')
# matplotlib.use('TkAgg')
import matplotlib.pyplot as plt
from matplotlib.widgets import RectangleSelector
# plt.switch_backend('agg')

import umap

from bokeh.plotting import figure, output_file, show, ColumnDataSource
from bokeh.models import HoverTool
from bokeh.models.glyphs import ImageURL

from .preprocessing import time_from_filename

FS = 44100
SPEC_THRESH = 1.0 # -8.5 homer, 4.0 mouse, -9 marmoset, -3.5 zebra finch
EPSILON = 1e-9

DOWNSAMPLE_TUPLE = (1,1) # marmoset (1,2)

MIN_FREQ, MAX_FREQ = 300, 12e3 # 40k-110k mouse, 350-13k marmoset

PSD_DUR, SPEC_DUR = 0.5, 0.5 # 0.1 mouse, 0.5 marmoset, 0.5 zebra finch

CENTER_T = 0.5 # how far into the song are we looking?
SONG_DURATION = 2.0 # how long is the song?

X1, Y1, X2, Y2 = 0, 0, 0, 0
LINES = []

def segment(load_dirs):
	""" Main method """
	global X1, Y1, X2, Y2
	# Collect filenames.
	filenames = []
	for load_dir in load_dirs:
		filenames += [os.path.join(load_dir,i) for i in os.listdir(load_dir) if i[-4:] in ['.wav', '.mat']]

	"""
	run_num = 0
	while True:
		run_num += 1
		# Collect PSDs and spectrograms.
		psds, specs = [], []
		np.random.seed(42)
		psd_f_ind, spec_f_ind = None, None
		print("Collecting data...")

		# from multiprocessing import Pool
		# from itertools import repeat
		# with Pool(min(3, os.cpu_count()-1)) as pool:
		# 	pool.starmap(get_rand_psds_specs, zip())
		# # quit()

		for i in tqdm(range(5000)): #10000

			flag = True
			while flag:
				filename = filenames[np.random.randint(len(filenames))]
				psd, spec, psd_f_ind, spec_f_ind = get_rand_psd_spec(filename, psd_dur=PSD_DUR, spec_dur=SPEC_DUR, psd_f_ind=psd_f_ind, spec_f_ind=spec_f_ind)
				if run_num > 1:
					flag = not psd_in_region(spec, transform, boundaries)
				else:
					flag = False
			# print("psd", psd.shape)
			# print("spec", spec.shape)
			# print(np.min(spec), np.max(spec))
			# plt.imshow(spec, aspect='auto', origin='lower')
			# plt.savefig('temp.pdf')
			# plt.close('all')
			# sleep(2)
			# if i == 10:
			# 	quit()
			psds.append(psd)
			specs.append(spec)
		psds = np.array(psds)
		np.save('psds.npy', psds)
		specs = np.array(specs)
		np.save('specs.npy', specs)
		# UMAP the PSDs.
		print("Dimensionality reduction...")
		transform = umap.UMAP(n_components=2, n_neighbors=20, min_dist=0.1, metric='euclidean', random_state=42)
		embedding = transform.fit_transform(specs.reshape(len(specs), -1))
		joblib.dump(transform, 'transform_'+str(run_num)+'.sav')
		perm = np.random.permutation(len(embedding))
		embedding, specs = embedding[perm], specs[perm]
		np.save('embedding.npy', embedding)

		# Make an HTML mouse-over plot.
		print("Making HTML...")
		make_html_plot(embedding, specs, output_dir='temp2', num_imgs=400)
		embedding = np.load('embedding.npy')

		# Non-interactive:
		plt.title("Select relevant audio")
		plt.scatter(embedding[:,0], embedding[:,1], c='b', alpha=0.1, s=0.7)
		plt.savefig('temp.pdf')
		plt.close('all')
		x1 = float(input("x1: "))
		x2 = float(input("x2: "))
		y1 = float(input("y1: "))
		y2 = float(input("y2: "))
		x1s, x2s, y1s, y2s = [x1], [x2], [y1], [y2]

		boundaries = {'x1s':x1s, 'y1s':y1s, 'x2s':x2s, 'y2s':y2s}
		np.save('boundaries.npy', boundaries)

		if input("q for quit ") == 'q':
			quit()

	"""

	"""
	# Plot an interactive selcting tool.
	print("Interactive plot...")
	x1s, y1s, x2s, y2s = [], [], [], []
	colors = np.array(['b'] * len(embedding))
	X1 = 0
	while X1 is not None:
		print("here")
		X1 = None
		for i, embed in enumerate(embedding):
			if colors[i] == 'r':
				continue
			for x1, y1, x2, y2 in zip(x1s, y1s, x2s, y2s):
				if embed[0] < max(x1, x2) and embed[0] > min(x1, x2) and \
						embed[1] < max(y1, y2) and embed[1] > min(y1, y2):
					colors[i] = 'r'
		fig, current_ax = plt.subplots()
		toggle_selector.RS = RectangleSelector(current_ax, line_select_callback,
				drawtype='box', useblit=True,
				button=[1, 3],  # don't use middle button
				minspanx=5, minspany=5,
				spancoords='pixels',
				interactive=True)
		plt.title("Select relevant audio")
		plt.connect('key_press_event', toggle_selector)
		print("making plot 1")
		plt.scatter(embedding[:,0], embedding[:,1], c=colors, alpha=0.1, s=0.7)
		plt.show()
		sleep(20)
		print("making plot 2")
		if X1 is not None:
			x1s.append(X1)
			y1s.append(Y1)
			x2s.append(X2)
			y2s.append(Y2)
		"""

	"""
	boundaries = {'x1s':x1s, 'y1s':y1s, 'x2s':x2s, 'y2s':y2s}
	np.save('boundaries.npy', boundaries)
	"""

	"""
	PART II
	-------
	"""

	"""
	transform = joblib.load('transform_'+str(run_num)+'.sav')
	boundaries = np.load('boundaries.npy').item()

	# Find segmentation boundaries.
	specs = []
	np.random.seed(42)
	psd_f_ind, spec_f_ind = None, None
	print("Collecting data...")

	# # Collect non-random spectrograms.
	# delta_t = 0.025
	# fs, audio = wavfile.read('001.wav')
	# t = 0.0
	# f_ind = None
	# specs = []
	# print("len(audio)", len(audio)/fs)
	# while t + SPEC_DUR + 0.01 < min(len(audio)/fs, 2000):
	# 	i1, i2 = int(fs*t), int(fs*(t+SPEC_DUR))
	# 	spec, f_ind = get_spec(audio[i1:i2], f_ind=f_ind, min_freq=MIN_FREQ, max_freq=MAX_FREQ)
	# 	specs.append(spec)
	# 	t += delta_t
	# print("f_ind", f_ind)

	# Collect random spectrograms.
	for i in tqdm(range(5000)): # 100000
		flag = True
		while flag:
			filename = filenames[np.random.randint(len(filenames))]
			psd, spec, psd_f_ind, spec_f_ind = get_rand_psd_spec(filename, psd_dur=PSD_DUR, spec_dur=SPEC_DUR, psd_f_ind=psd_f_ind, spec_f_ind=spec_f_ind)
			flag = not psd_in_region(spec, transform, boundaries)
		specs.append(spec)
		if (i + 1) % 500 == 0:
			np.save('specs.npy', np.array(specs))
	print("spec_f_ind:", spec_f_ind)

	specs = np.array(specs)
	np.save('specs.npy', specs)

	print('specs', specs.shape)
	# weighted_specs = np.copy(specs)
	# for i in range(len(specs)):
	# 	for j in range(len(specs[i])):
	# 		weighted_specs[i,j,:specs.shape[2]//2] *= np.linspace(0,1,specs.shape[2]//2)
	# 		weighted_specs[i,j,specs.shape[2]//2:] *= np.linspace(1,0,specs.shape[2]-specs.shape[2]//2)
	print("Dimensionality reduction...")
	transform_final = umap.UMAP(n_components=2, n_neighbors=20, min_dist=0.1, metric='euclidean', random_state=42)
	print("specs shape:", specs.shape)
	embedding = transform_final.fit_transform(specs.reshape(len(specs), -1))
	joblib.dump(transform_final, 'transform_2.sav')
	perm = np.random.permutation(len(embedding))
	embedding, specs = embedding[perm], specs[perm]
	np.save('embedding_2.npy', embedding)

	# Make an HTML mouse-over plot.
	print("Making HTML...")
	marked_specs = get_marked_specs(specs)
	make_html_plot(embedding, marked_specs, output_dir='temp3')

	# Non-interactive:
	plt.scatter(embedding[:,0], embedding[:,1], c='b', alpha=0.1, s=0.7)
	plt.savefig('temp.pdf')
	plt.close('all')
	center_x = float(input("center x: "))
	center_y = float(input("center y: "))
	radius = float(input("radius: "))
	radii = [radius]
	centers = [[center_x, center_y]]
	"""

	"""
	# Plot an interactive selcting tool.
	print("Interactive plot...")
	x1s, y1s, x2s, y2s = [], [], [], []
	radii = []
	centers = []
	X1 = 0
	while X1 is not None:
		X1 = None
		fig, current_ax = plt.subplots()
		toggle_selector.RS = RectangleSelector(current_ax, line_select_callback,
				drawtype='line', useblit=True,
				button=[1, 3],  # don't use middle button
				minspanx=0, minspany=0,
				spancoords='pixels',
				interactive=True)
		plt.title("Select relevant audio")
		plt.connect('key_press_event', toggle_selector)
		plt.scatter(embedding[:,0], embedding[:,1], c='b', alpha=0.1, s=0.7)
		for x1, y1, x2, y2, center, radius in zip(x1s, y1s, x2s, y2s, centers, radii):
			plt.plot([x1,x2], [y1,y2], c='r')
			circle = plt.Circle(tuple(center), radius, color='r', fill=False)
			current_ax.add_artist(circle)
		plt.show()
		if X1 is not None:
			x1s.append(X1)
			y1s.append(Y1)
			x2s.append(X2)
			y2s.append(Y2)
			centers.append([(X1+X2)/2,(Y1+Y2)/2])
			radii.append(0.5 * ((X1-X2)**2+(Y1-Y2)**2)**0.5)
	"""

	# TEMP!
	centers = [[12.5,7]]
	radii = [2]

	seg_stuff = {'radii':radii, 'centers':np.array(centers)}
	np.save('seg_stuff.npy', seg_stuff)

	# Parse through all the files, collecting
	seg_stuff = np.load('seg_stuff.npy').item()
	radius = seg_stuff['radii'][0]
	center = seg_stuff['centers'][0]

	transform = joblib.load('transform_2.sav')

	# Segment a file.
	segment_audio('data/raw/bird_data/79', 'temp_save', transform, center, radius)
	quit()
	fs, audio = wavfile.read('data/raw/bird_data/79/red291_43566.27672737_4_11_7_41_12.wav')
	print(filenames[0])
	temp = segment_audio_chunk(audio, transform, center, radius)
	print(temp)
	quit()

def segment_audio(load_dir, save_dir, transform, center, radius):
	"""Save all the song bouts."""
	load_filenames = [os.path.join(load_dir,i) for i in os.listdir(load_dir) if i[-4:] in ['.wav', '.mat']]
	# load_filenames = ['data/raw/bird_data/79/red291_43566.27672737_4_11_7_41_12.wav']
	audio_segs, filenames, file_times, times = [], [], [], []
	num_audio_frames = int(SONG_DURATION*FS)
	write_file_num = 0
	songs_per_file = 10
	max_num_songs = 2000
	for load_filename in load_filenames:
		print(load_filename, len(audio_segs))
		filename_time = time_from_filename(load_filename)
		start_t = 0.0
		center_ts = []
		audio = get_audio(load_filename)
		while True:
			result = segment_audio_chunk(audio[int(FS*start_t):], transform, center, radius)
			if result == -1:
				continue
			if result is None:
				break
			center_ts.append(result + start_t)
			start_t += result + SONG_DURATION / 2
		for center_t in center_ts:
			t1 = int((center_t - CENTER_T)*FS)
			t2 = t1 + num_audio_frames
			audio_segs.append(audio[t1:t2])
			filenames.append(load_filename)
			file_times.append(center_t - CENTER_T)
			times.append(filename_time + center_t - CENTER_T)
		while len(audio_segs) >= songs_per_file:
			save_filename = os.path.join(save_dir, str(write_file_num).zfill(2)+'.hdf5')
			wavfile.write(os.path.join(save_dir, 'temp.wav'), FS, np.concatenate(audio_segs, axis=0))
			with h5py.File(save_filename, "w") as f:
				f.create_dataset('audio', data=np.array(audio_segs[:songs_per_file]))
				f.create_dataset('filename', data=np.array(filenames[:songs_per_file]).astype('S'))
				f.create_dataset('file_time', data=np.array(file_times[:songs_per_file]))
				f.create_dataset('time', data=np.array(times[:songs_per_file]))
			audio_segs = audio_segs[songs_per_file:]
			filenames = filenames[songs_per_file:]
			file_times = file_times[songs_per_file:]
			times = times[songs_per_file:]
			write_file_num += 1
			if write_file_num * songs_per_file >= max_num_songs:
				return

def segment_audio_chunk(audio, transform, center, radius):
	r2 = radius*radius
	f_ind = None
	STEP = 0.01
	SMALL_STEP = 0.002
	t = CENTER_T
	while t + SPEC_DUR + 0.01 < len(audio)/FS:
		embed, f_ind = get_embed_from_time(audio, t, f_ind, transform)
		current_r2 = np.sum(np.power(embed-center,2))
		# print(t, current_r2)
		if current_r2 > r2:
			t += STEP
			continue
		# Step until radius increases.
		prev_r2 = current_r2
		while True:
			t += SMALL_STEP
			embed, f_ind = get_embed_from_time(audio, t, f_ind, transform)
			current_r2 = np.sum(np.power(embed-center,2))
			# print(t, current_r2, "inside loop")
			if current_r2 > prev_r2:
				t -= SMALL_STEP
				# print(t)
				# print(CENTER_T)
				# print(t-CENTER_T+SONG_DURATION)
				# print(len(audio)/FS)
				if t - CENTER_T + SONG_DURATION < len(audio)/FS:
					return t
				return None
			prev_r2 = current_r2
	return None

def get_embed_from_time(audio, t, f_ind, transform):
	i1, i2 = int(t*FS), int((t+SPEC_DUR)*FS)
	spec, f_ind = get_spec(audio[i1:i2], f_ind=f_ind)
	return transform.transform(spec.reshape(1,-1)).flatten(), f_ind

def get_rand_psd_spec(filename, psd_dur=0.2, spec_dur=0.3, psd_f_ind=None, \
		spec_f_ind=None, return_middle_sample=False):
	audio = get_audio(filename)
	psd_samples, spec_samples = int(psd_dur * FS), int(spec_dur * FS)
	max_samples = max(psd_samples, spec_samples)
	start = np.random.randint(len(audio) - max_samples)
	delta = (max_samples - psd_samples) // 2
	psd, psd_f_ind = get_psd(audio[start+delta:start+delta+psd_samples], f_ind=psd_f_ind)
	delta = (max_samples - spec_samples) // 2
	spec, spec_f_ind = get_spec(audio[start+delta:start+delta+spec_samples], f_ind=spec_f_ind)
	if return_middle_sample:
		return psd, spec, psd_f_ind, spec_f_ind, start+psd_samples//2
	return psd, spec, psd_f_ind, spec_f_ind

def get_rand_segmented_spec(filename, lines, transform, boundaries, \
			psd_dur=0.2, spec_dur=0.3, psd_spec_ind=None, spec_f_ind=None, \
			max_syll_dur=1.5):
	"""

	Returns
	-------
	spec : numpy array or None
		spectrogram of a single syllable or None if none is found.
	"""
	# Get a psd
	psd, _, psd_f_ind, spec_d_ind, mid = get_rand_psd_spec(filename, \
			psd_dur=psd_dur, spec_dur=spec_dur, psd_spec_ind=psd_spec_ind, \
			spec_f_ind=spec_f_ind, return_middle_sample=True)
	# If there's no nearby birdsong, give up.
	if not psd_in_region(psd, transform, boundaries):
		return None
	# Otherwise, find the nearest syllable.
	max_syll_samples = int(max_syll_dur * FS)
	# First search left, looking for local minima in distances to segmenting lines.
	distances = {} # NOTE: TEMP to see how stable these are.
	for i in range(max(0,mid-max_syll_samples), min(len(audio)-1, mid+max_syll_samples), 1):
			distances[i] = get_distance(filename) # NOTE: HERE

def get_spec(audio, min_freq=MIN_FREQ, max_freq=MAX_FREQ, f_ind=None, fs=FS):
	"""Get a spectrogram."""
	# Get the spectrogram.
	f, t, Zxx = stft(audio, fs=fs)
	# Cut out the frequencies we don't care about.
	if f_ind is None:
		f_ind = np.searchsorted(f, [min_freq, max_freq])
	f = f[f_ind[0]:f_ind[1]]
	Zxx = Zxx[f_ind[0]:f_ind[1]]
	# Set a threshold & normalize.
	Zxx = np.log((np.abs(Zxx) + EPSILON))
	Zxx -= SPEC_THRESH
	Zxx[Zxx < 0.0] = 0.0
	Zxx /= np.max(Zxx) + EPSILON
	Zxx = downscale_local_mean(Zxx, DOWNSAMPLE_TUPLE)
	return Zxx, f_ind

def get_psd(audio, min_freq=MIN_FREQ, max_freq=MAX_FREQ, f_ind=None):
	"""Get a power spectral density."""
	# Get the PSD.
	f, Pxx = welch(audio, fs=FS)
	# Cut out the frequencies we don't care about.
	if f_ind is None:
		f_ind = np.searchsorted(f, [min_freq, max_freq])
	f = f[f_ind[0]:f_ind[1]]
	Pxx = Pxx[f_ind[0]:f_ind[1]]
	return Pxx, f_ind

def make_html_plot(embedding, specs, output_dir='temp', num_imgs=2000, title=""):
	"""Make an HTML tooltip mouse-over plot."""
	# Write the tooltip images.
	if not os.path.exists(output_dir):
		os.makedirs(output_dir)
	num_imgs = min(len(specs), num_imgs)
	for i in tqdm(range(num_imgs)):
		save_image(specs[i], os.path.join(output_dir, str(i)+'.jpg'))
	output_file(os.path.join(output_dir,"main.html"))
	source = ColumnDataSource(
			data=dict(
				x=embedding[:num_imgs,0],
				y=embedding[:num_imgs,1],
				imgs = ['./'+str(i)+'.jpg' for i in range(num_imgs)],
			)
		)
	source2 = ColumnDataSource(
			data=dict(
				x=embedding[num_imgs:,0],
				y=embedding[num_imgs:,1],
			)
		)
	p = figure(plot_width=800, plot_height=600, title=title)
	p.scatter('x', 'y', size=3, fill_color='blue', fill_alpha=0.1, source=source2)
	tooltip_points = p.scatter('x', 'y', size=5, fill_color='red', source=source)
	hover = HoverTool(
			renderers=[tooltip_points],
			tooltips="""

			"""
		)
	p.add_tools(hover)
	p.title.align = "center"
	p.title.text_font_size = "25px"
	p.axis.visible = False
	p.xgrid.visible = False
	p.ygrid.visible = False
	show(p)

def save_image(data, filename):
	"""https://fengl.org/2014/07/09/matplotlib-savefig-without-borderframe/"""
	sizes = np.shape(data)
	height = float(sizes[0])
	width = float(sizes[1])
	fig = plt.figure()
	fig.set_size_inches(width/height, 1, forward=False)
	ax = plt.Axes(fig, [0., 0., 1., 1.])
	ax.set_axis_off()
	fig.add_axes(ax)
	ax.imshow(data, cmap='viridis', origin='lower')
	plt.savefig(filename, dpi=height)
	plt.close('all')

def line_select_callback(eclick, erelease):
	"""https://matplotlib.org/examples/widgets/rectangle_selector.html"""
	# eclick and erelease are the press and release events
	global X1, Y1, X2, Y2
	X1, Y1 = eclick.xdata, eclick.ydata
	X2, Y2 = erelease.xdata, erelease.ydata

def toggle_selector(event):
	"""https://matplotlib.org/examples/widgets/rectangle_selector.html"""
	print(' Key pressed.')
	if event.key in ['Q', 'q'] and toggle_selector.RS.active:
		print(' RectangleSelector deactivated.')
		toggle_selector.RS.set_active(False)
	if event.key in ['A', 'a'] and not toggle_selector.RS.active:
		print(' RectangleSelector activated.')
		toggle_selector.RS.set_active(True)

def get_audio(filename):
	"""Retrieve audio from a given file."""
	if filename[-4:] == '.wav':
		fs, audio = wavfile.read(filename)
	elif filename[-4:] == '.mat':
		temp = loadmat(filename)
		audio = temp['spike2Chunk'].reshape(-1)
		fs = temp['fs']
	else:
		raise NotImplementedError
	assert fs == FS, "found "+str(fs)+", required " + str(FS)
	return audio

def psd_in_region(psd, transform, b):
	"""Does this PSD contain relevant audio?"""
	embed = transform.transform(psd.reshape(1,-1)).flatten()
	for x1, y1, x2, y2 in zip(b['x1s'], b['y1s'], b['x2s'], b['y2s']):
		if embed[0]>min(x1,x2) and embed[0]<max(x1,x2) and embed[1]>min(y1,y2) and embed[1]<max(y1,y2):
			return True
	return False

def get_marked_specs(specs):
	"""Mark the center in time."""
	marked_specs = np.zeros(shape=(specs.shape[0], specs.shape[1]+4, specs.shape[2]))
	marked_specs[:,2:-2,:] = specs[:,:,:]
	marked_specs[:,:2,specs.shape[2]//2] = 1.2
	marked_specs[:,-2:,specs.shape[2]//2] = 1.2
	return marked_specs

def onclick(event):
	if event.button == 3:
		global LINES
		x = event.xdata
		y = event.ydata
		if len(LINES) == 0 or len(LINES[-1]) % 4 == 0:
			LINES.append([x,y])
		else:
			LINES[-1].append(x,y)
			x1, y1, x2, y2 = LINES[-1]
			plt.plot([x1,x1], [y1,y2], '-')
			radius = 0.5 * ((x1-x2)**2 + (y1-y2)**2)**0.5
			circle = plt.Circle((0.5*(x1+x2), 0.5*(y1+y2)), radius, color='r', alpha=0.3)
			plt.gca().add_artist(circle)
			plt.show()

if __name__ == '__main__':
	# load_dirs = ['human/']
	load_dirs = ['data/raw/bird_data/79']
	segment(load_dirs)

###


import pandas as pd
from bs4 import BeautifulSoup
import requests
import zipfile
import io
import re
import numpy as np

pd.set_option('float_format', '{:.6f}'.format)

survey = "https://drive.google.com/uc?export=download&id=0B6ZlG_Eygdj-c1kzcmUxN05VUXM"
data = "survey_results_public.csv"
schema = "survey_results_schema.csv"
populations = "https://en.wikipedia.org/wiki/List_of_countries_and_dependencies_by_population"

survey = requests.get(survey)
#download zip in memory
zp = zipfile.ZipFile(io.BytesIO(survey.content))

data = pd.read_csv(zp.open(data), index_col = 0)
schema = pd.read_csv(zp.open(schema))
#1

print(data.iloc[:, :1].head())
print(schema.head())
#I used index_col only to match output in assignment, it's uncomfortable for me to use
#therefore, reseting it
data = data.reset_index()
respondents = data["Respondent"].size
questions = schema["Question"].size
print("Number of respondents:", respondents)
print("Number of questions:", questions)
print()

#2
countries = (data[["Respondent", "Country"]]
       .groupby("Country").size()
       .sort_values(ascending = False))
print(countries.head(10))
print()
print(countries.head(10)/respondents)
print()

#3
populations = requests.get(populations)
soup = BeautifulSoup(populations.text, "html.parser")
populations  = soup.find('table', attrs = {'class' : 'wikitable sortable'})

table = []
headers = []

rows = populations.find_all('tr')
for th in populations.find_all('th'):
    headers.append(th.getText().strip().replace('\n',''))
for tr in populations.find_all('tr'):
    row = []
    for td in tr.find_all('td'):
        #deleting text between square brackets, on wiki there are notes for some countries in []
        td = re.sub(r'\[.*?\]', '', td.text)
        row.append(' '.join(td.split()))
    table.append(row)

populations = pd.DataFrame(table, columns = headers)
#from wiki downloaded table leaving only country and population
populations = populations.iloc[1:, 1:3]
populations.columns = ['Country', 'Population']
#change population from string to int
populations["Population"] = populations["Population"].str.replace(',', '').apply(int)
countries100 = countries[countries > 100]
#getting numeric index
countries100 = countries100.reset_index()
countries100.columns = ['Country', 'Developers']
#merging 2 datasets on 'Country' column
merged = pd.merge(countries100, populations, on = 'Country')
merged['Ratio'] = merged['Developers'].apply(int)/merged['Population']
#Adjust data to similar output in home assignment pdf
merged = merged.set_index('Country')
merged.columns = ['Country' , 'Population', 'Ratio']
del merged.index.name
print(merged.sort_values('Ratio', ascending = False).head(10))
print()
#4
versionControl = (data[["Respondent", "VersionControl"]]
       .groupby("VersionControl").size()
       .sort_values(ascending = False))
print(versionControl)
print()
#5
#removing white spaces in language string
dummyLanguages = data["HaveWorkedLanguage"].str.replace(r'\s', '')
#splitting language string into columns, with corresponding binary variables
dummyLanguages = dummyLanguages.str.get_dummies(sep = ';')
#all languages can be retrivied as array of columns after splitting
languages = list(dummyLanguages.columns)
print(languages)
print()
#6
print(dummyLanguages[dummyLanguages == 1]
      .count()
      .sort_values(ascending = False)
      .head(10))
print()
#7
#Counting how many users for each language in countries
dummyLanguages["Country"] = data["Country"]
countryPopular = (dummyLanguages
                  .groupby("Country")
                  .sum())
#Finding maximum value for language in row
countryPopular["Total"] = countryPopular.max(axis = 1)
#and corresponding column for that value
countryPopular["Popular"] = countryPopular.idxmax(axis = 1)
print(countryPopular[["Total", "Popular"]]
      .sort_values("Total", ascending = False)
      .head(10))
print()
print(countryPopular[["Total", "Popular"]]
      .loc[countryPopular["Popular"] != "JavaScript"]
      .sort_values("Total", ascending = False)
      .head(5))
print()
#let's find countries with highest copy paste rate, and countries which provide
#above average salary to copy pasters
cp = data[["Country", "StackOverflowCopiedCode", "Salary"]]
#get countries with at least 100 people in survey for more meaningful results
cp = cp[cp["Country"].isin(list(countries100["Country"]))]
#get list of unique answers on question about code copying, and select where person
#copied code at least once a day or once a week
indicators = list(cp.groupby("StackOverflowCopiedCode").max().index)[:2]
cp = cp[cp["StackOverflowCopiedCode"].isin(indicators)]
#group by country and calculating how many people copied code once a day or week, and their
#average salaries
cp = (cp.groupby('Country')
      .agg({'StackOverflowCopiedCode' : np.size, 'Salary' : np.mean}))
#reseting indices for proper division
cp = cp.reset_index()
countries100 = countries100.sort_values("Country").reset_index(drop = True)
cp["copypaste rate"] = (cp["StackOverflowCopiedCode"]/
                                countries100["Developers"])
#shows countries with highest level of copy paste in their code
print(cp.sort_values("copypaste rate", ascending = False).head(10))
print()
#countries where copypasters can earn salary higher than average in survey
print(cp[cp["Salary"] >= data["Salary"].mean()].head(10))
print()
#in which countries copy pasters earn salary higher than country average?
countryAvg = (data[['Country', 'Salary']]
              .groupby('Country', as_index = False)
              .mean())
countryAvg.columns = ['Country', 'Average Salary']
cp = pd.merge(cp, countryAvg, on = 'Country')
print(cp[cp["Salary"] > cp["Average Salary"]]
      [["Country","Salary","Average Salary"]]
      .head(10))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
'''
本示例代码仅作参考。如果您是业务调用，建议对body体也做加密。
注意：本示例用到的鉴权方式不适用于一部分产品（语音识别、人脸识别等）。请查阅您使用产品的API文档，如果鉴权方式基于Access Key（包括Access Key ID（AK）和Secret Access Key（SK）），
且最终认证字符串为bce-auth-v{version}/{accessKeyId}/{timestamp}/{expirationPeriodInSeconds}/{signedHeaders}/{signature}，则适用本示例。
此外，部分产品在AK/SK加密之外，还会要求额外的认证，例如百度信息流推广API需要您将百度推广账号的信息填到body体里。请注意查阅文档。

'''

import hashlib
import hmac
import urllib.parse
import time

# 1.AK/SK、host、method、URL绝对路径、querystring

AK = "c02376d42a05434c98*************"
SK = "08ead63a52f84ad****************"
host = "iotdm.gz.baidubce.com"
method = "GET"
query = ""
deviceName = "wu_t_shadow"
URI = "/v3/iot/management/device/"

# 2.x-bce-date
x_bce_date = time.gmtime()
x_bce_date = time.strftime('%Y-%m-%dT%H:%M:%SZ',x_bce_date)
#x_bce_date = "2020-03-20T13:33:41Z"
# 3.header和signedHeaders
header = {
        "Host":host,
        "Content-type":"application/json; charset=utf-8"
        }
signedHeaders = "content-type;host;x-bce-date"
# 4.认证字符串前缀
authStringPrefix = "bce-auth-v1" + "/" +AK + "/" + x_bce_date + "/" + "1800"
# 5.生成CanonicalRequest
#5.1生成CanonicalURI
CanonicalURI = urllib.parse.quote(URI)        # windows下为urllib.parse.quote，Linux下为urllib.quote
#5.2生成CanonicalQueryString
CanonicalQueryString = query           # 如果您调用的接口的query比较复杂的话，需要做额外处理
#5.3生成CanonicalHeaders
result = []
for key,value in header.items():
    tempStr = str(urllib.parse.quote(key.lower(),safe="")) + ":" + str(urllib.parse.quote(value,safe=""))
    result.append(tempStr)
result.sort()
CanonicalHeaders = "\n".join(result)
print("--------CanonicalHeaders---------" + CanonicalHeaders)
#5.4拼接得到CanonicalRequest
CanonicalRequest = method + "\n" + CanonicalURI + deviceName + "\n" + CanonicalQueryString +"\n" + "host:" + host
# 6.生成signingKey
signingKey = hmac.new(SK.encode('utf-8'),authStringPrefix.encode('utf-8'),hashlib.sha256)
print("---------signingKey------" + signingKey.hexdigest())
# 7.生成Signature
Signature = hmac.new((signingKey.hexdigest()).encode('utf-8'),CanonicalRequest.encode('utf-8'),hashlib.sha256)
print(signingKey.hexdigest().encode('utf-8'))
print(CanonicalRequest.encode('utf-8'))
print("---------Signature------" + Signature.hexdigest())

# 8.生成Authorization并放到header里
header['Authorization'] = authStringPrefix + "/host/" + Signature.hexdigest()
# 9.发送API请求并接受响应
print(header)
import requests
import json

body={
      "name" : "QQQQQQ"
      }

url = "http://"+ host + URI + deviceName
print(url)
r = requests.get(url,headers = header,data=json.dumps(body))

print(r.text)


#-*- coding: utf-8 -*-
import requests
from bs4 import BeautifulSoup

url = "https://www.advokatsamfundet.se/Behover-du-advokat/Sok-advokat/Sokresultat/?positions=102001"
response = requests.get(url)
#print(response.text)
soup = BeautifulSoup(response.text)
divs = soup.find_all('div', {'class': "col-1"})
for div in divs:
	link = div.find("a")
	#print(link)
	if link:
		href = link["href"]
		sub_url = "https://www.advokatsamfundet.se" + href
		print(sub_url)

"""
print(sub_url)

	link = soup.find("div", {'class': "lead"})
	#else:
	#	description = ""
	print(mp_name.text)
	if link:
		description = link.find("p")
		print(description.text)
"""
"""

	link =
	url = "https://www.advokatsamfundet.se/" + link

def slugify(argument):
	slug = argument.lower()\
		.replace(" ", "-")\
		.replace("ö", "o")\
		.replace("ä", "a")\
		.replace("å", "a")\
		.replace("Å", "a")\
		.replace("é", "e")\
		.replace("Ö", "o")\
		.replace("ü", "u")
	return slug

for argument in miljopartiet.mps:
	changed_name = slugify(argument)
	url = "http://www.mp.se/om/" + changed_name
	# = response.text
	response = requests.get(url)
	soup = BeautifulSoup.BeautifulSoup(response.text)
#response.text innehåller html-koden
	mp_name = soup.find("h1")
	#mp_uppdrag = soup.find('div', {'class': "lead"})
	mp_email = soup.find('a', {'class': "email"})
	mp_mobile = soup.find('a', {'class': "tel"})
	div_tagg = soup.find("div", {'class': "lead"})
	#else:
	#	description = ""
	print(mp_name.text)
	if div_tagg:
		description = div_tagg.find("p")
		print(description.text)
	#print(mp_uppdrag.text)
	if mp_email:
		print(mp_email.text)
	if mp_mobile:
		print(mp_mobile.text)
	#print(slugify(argument))

	"""


import math
import random
import sys

import numpy as np
import pygame

# RGB
RED = (255, 0, 0)
BLUE = (0, 0, 255)
YELLOW = (255, 255, 0)

BLACK = (0, 0, 0)

ROW_COUNT = 6
COLUMN_COUNT = 6

PLAYER = 0
SKY_NET = 1  # AI actually XD,just had fun

PLAYER_PIECE = 1
AI_PIECE = 2

WINDOW_LENGTH = 4
EMPTY = 0

def create_board():
    board = np.zeros((ROW_COUNT, COLUMN_COUNT))
    return board

def drop_piece(board, row, col, piece):
    board[row][col] = piece

def is_valid_location(board, col):
    return board[ROW_COUNT - 1][col] == 0

def get_valid_row(board, col):
    for r in range(ROW_COUNT):
        if board[r][col] == 0:
            return r

def print_board(board):
    print(np.flip(board, 0))

pygame.init()

SQUARE_SIZE = 50

width = COLUMN_COUNT * SQUARE_SIZE
height = (ROW_COUNT + 1) * SQUARE_SIZE

size = (width, height)

RADIUS = int(SQUARE_SIZE / 2 - 5)

screen = pygame.display.set_mode(size)

def wining_move(board, piece):
    # Check horizontal location to win
    for c in range(COLUMN_COUNT - 3):
        for r in range(ROW_COUNT):
            if board[r][c] == piece and board[r][c + 1] == piece and \
                    board[r][c + 2] == piece and board[r][c + 3] == piece:
                return True

    # Check vertical location to win
    for c in range(COLUMN_COUNT):
        for r in range(ROW_COUNT - 3):
            if board[r][c] == piece and board[r + 1][c] == piece and \
                    board[r + 2][c] == piece and board[r + 3][c] == piece:
                return True
    # Check for Positive diagonal
    for c in range(COLUMN_COUNT - 3):
        for r in range(ROW_COUNT - 3):
            if board[r][c] == piece and board[r + 1][c + 1] == piece and \
                    board[r + 2][c + 2] == piece and board[r + 3][c + 3] == piece:
                return True

    # Check for Negative diagonal
    for c in range(COLUMN_COUNT - 3):
        for r in range(3, ROW_COUNT):
            if board[r][c] == piece and board[r - 1][c + 1] == piece and \
                    board[r - 2][c + 2] == piece and board[r - 3][c + 3] == piece:
                return True

def evaluate(window, piece):
    score = 0
    opp_piece = PLAYER_PIECE
    if piece == PLAYER_PIECE:
        opp_piece = AI_PIECE
    # 4 in a row
    if window.count(piece) == 4:
        score += 100
    # 3 in a row
    elif window.count(piece) == 3 and window.count(EMPTY) == 1:
        score += 5
    elif window.count(piece) == 2 and window.count(EMPTY) == 2:
        score += 2

    if window.count(opp_piece) == 3 and window.count(EMPTY) == 1:
        score -= 4

    return score

def score_position(board, piece):
    score = 0
    # Score center column
    center_array = [int(i) for i in list(board[:, COLUMN_COUNT // 2])]
    center_count = center_array.count(piece)
    score += center_count * 3

    # Score Horizontal
    for r in range(ROW_COUNT):
        row_array = [int(i) for i in list(board[r, :])]
        for c in range(COLUMN_COUNT - 3):
            window = row_array[c:c + WINDOW_LENGTH]
            # 4 in a row
            score += evaluate(window, piece)
    # Score Vertical
    for c in range(COLUMN_COUNT):
        column_array = [int(i) for i in list(board[:, c])]
        for r in range(ROW_COUNT - 3):
            window = column_array[r:r + WINDOW_LENGTH]
            score += evaluate(window, piece)
    # Score positive diagonal
    for r in range(ROW_COUNT - 3):
        for c in range(COLUMN_COUNT - 3):
            window = [board[r + i][c + i] for i in range(WINDOW_LENGTH)]
            score += evaluate(window, piece)
    # Score negative diagonal
    for r in range(ROW_COUNT - 3):
        for c in range(COLUMN_COUNT - 3):
            window = [board[r + 3 - i][c + i] for i in range(WINDOW_LENGTH)]
            score += evaluate(window, piece)

    return score

def get_valid_locations(board):
    valid_locations = []
    for col in range(COLUMN_COUNT):
        if is_valid_location(board, col):
            valid_locations.append(col)
    return valid_locations

def pick_best_move(board, piece):
    valid_locations = get_valid_locations(board)
    best_score = -10000
    best_col = random.choice(valid_locations)
    for column in valid_locations:
        row = get_valid_row(board, column)
        # we make copy to take new memory location, to any modifications do not change main board
        temp_board = board.copy()
        drop_piece(temp_board, row, column, piece)
        score = score_position(temp_board, piece)
        if score > best_score:
            best_score = score
            best_col = column

    return best_col

def is_terminal_node(board):
    return wining_move(board, PLAYER_PIECE) or wining_move(board, AI_PIECE) or len(get_valid_locations(board)) == 0

def minimax(board, depth, maximizing_player, alpha, beta):
    valid_locations = get_valid_locations(board)
    is_terminal = is_terminal_node(board)
    if depth == 0 or is_terminal:
        if is_terminal:
            if wining_move(board, AI_PIECE):
                return None, 9999999999999999
            elif wining_move(board, PLAYER_PIECE):
                return None, -9999999999999999
            else:  # no more valid moves
                return None, 0
        else:
            return None, score_position(board, AI_PIECE)
    if maximizing_player:
        value = -math.inf
        column = random.choice(valid_locations)
        for col in valid_locations:
            row = get_valid_row(board, col)
            b_copy = board.copy()
            drop_piece(b_copy, row, col, AI_PIECE)
            new_score = minimax(b_copy, depth - 1, False, alpha, beta)[1]
            if new_score > value:
                value = new_score
                column = col
            alpha = max(alpha, value)
            if alpha > beta:
                break
        return column, value
    else:  # minimizing player
        value = math.inf
        column = random.choice(valid_locations)
        for col in valid_locations:
            row = get_valid_row(board, col)
            b_copy = board.copy()
            drop_piece(b_copy, row, col, PLAYER_PIECE)
            new_score = minimax(b_copy, depth - 1, True, alpha, beta)[1]
            if new_score < value:
                value = new_score
                column = col
            beta = min(beta,value)
            if beta <= alpha:
                break
        return column, value

def draw_board(board):
    for c in range(COLUMN_COUNT):
        for r in range(ROW_COUNT):
            # rect(Surface, color, Rect,width = 0)
            pygame.draw.rect(screen, BLUE, (c * SQUARE_SIZE, r * SQUARE_SIZE + SQUARE_SIZE, SQUARE_SIZE, SQUARE_SIZE))
            # circle(Surface, color, pos, radius, width=0)
            pygame.draw.circle(screen,
                               BLACK,
                               (int(c * SQUARE_SIZE + SQUARE_SIZE / 2),
                                int(r * SQUARE_SIZE + SQUARE_SIZE + SQUARE_SIZE / 2)),
                               RADIUS)
    for c in range(COLUMN_COUNT):
        for r in range(ROW_COUNT):
            if board[r][c] == PLAYER_PIECE:
                pygame.draw.circle(
                    screen,
                    RED,
                    (int(c * SQUARE_SIZE + SQUARE_SIZE / 2),
                     height - int(r * SQUARE_SIZE + SQUARE_SIZE / 2)),
                    RADIUS)
            elif board[r][c] == AI_PIECE:
                pygame.draw.circle(
                    screen,
                    YELLOW,
                    (int(c * SQUARE_SIZE + SQUARE_SIZE / 2),
                     height - int(r * SQUARE_SIZE + SQUARE_SIZE / 2)),
                    RADIUS)
    pygame.display.update()

board = create_board()
print_board(board)
draw_board(board)
pygame.display.update()

my_font = pygame.font.SysFont("monospace", 75)

game_over = False
turn = random.randint(PLAYER, SKY_NET)

while not game_over:

    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            sys.exit()

        if event.type == pygame.MOUSEMOTION:
            pygame.draw.rect(screen, BLACK, (0, 0, width, SQUARE_SIZE))
            pos_x = event.pos[0]
            if turn == PLAYER:
                pygame.draw.circle(screen, RED, (pos_x, int(SQUARE_SIZE / 2)), RADIUS)
        pygame.display.update()

        # MouseButtonDown give us position
        if event.type == pygame.MOUSEBUTTONDOWN:
            pygame.draw.rect(screen, BLACK, (0, 0, width, SQUARE_SIZE))
            # Ask for PLayer 1 input
            if turn == PLAYER:
                pos_x = event.pos[0]
                col = int(math.floor(pos_x / SQUARE_SIZE))

                if is_valid_location(board, col):
                    row = get_valid_row(board, col)
                    drop_piece(board, row, col, PLAYER_PIECE)

                    if wining_move(board, PLAYER_PIECE):
                        label = my_font.render("Player 1 wins", 1, RED)
                        screen.blit(label, (40, 10))
                        print("Player 1 wins")
                        game_over = True

                    turn += 1
                    turn = turn % 2

    # Ask for PLayer 2 input
    if turn == SKY_NET and not game_over:

        #col = random.randint(0, COLUMN_COUNT - 1)
        #col = pick_best_move(board, AI_PIECE)
        col, minimax_score = minimax(board, 4, True, -math.inf, math.inf)
        if is_valid_location(board, col):
            # pygame.time.wait(500)
            row = get_valid_row(board, col)
            drop_piece(board, row, col, AI_PIECE)

            if wining_move(board, AI_PIECE):
                label = my_font.render("Player 2 wins", 7, YELLOW)
                screen.blit(label, (40, 10))
                print("Player 2 wins")
                game_over = True

            turn += 1
            turn = turn % 2

            print_board(board)
            draw_board(board)

    if game_over:
        pygame.time.wait(3000)


#core python example 4-9' book ranking by Screenscraper

from atexit import register
from re import compile
from threading import Thread
from time import ctime
from urllib.request import *

REGEX=compile('#([\d,]+) in Books')     #正则表达式匹配书籍
AMZN='http://amazon.com/dp/'        #Amazon基本连接
ISBNS={     #书籍isbn码及书名
	'0132269937':'Core python programing ',
	'0132356139':'Python Web Development with Django',
	'0137143419':'Python Fundamentals',
}

def getRanking(isbn):
	page=urlopen('%s%s'%(AMZN,isbn))
	data=page.read()
	page.close()
	return REGEX.findall(data)[0]

def _showRanking(isbn):
	print('- %r ranked %s'%(ISBNS[isbn],getRanking(isbn)))

def _main():
	print('At ',ctime(),'on Amazon...')
	for isbn in ISBNS:
		# _showRanking(isbn)
		Thread(target=_showRanking, args=(isbn,)).start()

@register
def _atexit():
	print('all DONE at: ',ctime())

if __name__=='__main__':
	_main()


#!/usr/bin/env python
#
# Copyright (c) 2015 Intel Corporation.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of works must retain the original copyright notice, this
#   list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the original copyright
#   notice, this list of conditions and the following disclaimer in the
#   documentation and/or other materials provided with the distribution.
# * Neither the name of Intel Corporation nor the names of its contributors
#   may be used to endorse or promote products derived from this work without
#   specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY INTEL CORPORATION "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL INTEL CORPORATION BE LIABLE FOR ANY DIRECT,
# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors:
#         Hongjuan, Wang<hongjuanx.wang@intel.com>

import unittest
import os, sys, commands
import comm

class TestPackertoolsFunctions(unittest.TestCase):
  def test_permission_duplicate(self):
      targetDir = "testapp/permission_field_duplicate_tests"
      manifestPath = comm.ConstPath + "/../" + targetDir + "/manifest.json"
      comm.perm(targetDir,manifestPath, self)

  def test_permission_lowercase(self):
      targetDir = "testapp/permission_field_lowercase_tests"
      manifestPath = comm.ConstPath + "/../" + targetDir + "/manifest.json"
      comm.perm(targetDir,manifestPath, self)

  def test_permission_more2(self):
      targetDir = "testapp/permission_field_more2_tests"
      manifestPath = comm.ConstPath + "/../" + targetDir + "/manifest.json"
      comm.perm(targetDir,manifestPath, self)

  def test_permission_uppercase(self):
      targetDir = "testapp/permission_field_uppercase_tests"
      manifestPath = comm.ConstPath + "/../" + targetDir + "/manifest.json"
      comm.perm(targetDir,manifestPath, self)

  def test_permission_duplicate2(self):
      comm.setUp()
      cmd = "python %smake_apk.py --package=org.xwalk.permission --name=example --project-dir=permission --arch=%s --mode=%s --app-url=http://www.intel.com --permission=Contacts --permission=contacts --permission=CONTACTS" % (comm.Pck_Tools, comm.ARCH, comm.MODE)
      num = 2
      comm.perm2(num, cmd, self)

  def test_permission_value3(self):
      comm.setUp()
      cmd = "python %smake_apk.py --package=org.xwalk.permission --name=example --project-dir=permission --arch=%s --mode=%s --app-url=http://www.intel.com --permission=Geolocation --permission=Messaging --permission=Contacts" % (comm.Pck_Tools, comm.ARCH, comm.MODE)
      num = 3
      comm.perm2(num, cmd, self)

  def test_permission_value4(self):
      comm.setUp()
      cmd = "python %smake_apk.py --package=org.xwalk.permission --name=example --project-dir=permission --arch=%s --mode=%s --app-url=http://www.intel.com --permission=Contacts --permission=Geolocation --permission=Messaging --permission=Contacts:Geolocation:Messaging" % (comm.Pck_Tools, comm.ARCH, comm.MODE)
      num = 4
      comm.perm2(num, cmd, self)

if __name__ == '__main__':
    unittest.main()


import difflib
import nltk

def showdiff(fr, to):
    differ = difflib.Differ()
    html=""

    for entry in differ.compare(nltk.sent_tokenize(fr), nltk.sent_tokenize(to)):
        if entry[0]=="+":
            html+="<tt style='white-space: pre;background-color:lightgreen'>"+entry+"</tt>
"
        elif entry[0]=="-":
            html+="<tt style='white-space: pre;background-color:lightcoral'>"+entry+"</tt>
"
        else:
            html+="<tt style='white-space: pre;'>"+entry+"</tt>
"

    return html