'''
This program is a rudimentary music transcriber. It uses fast Fourier transform to extract the frequency
spectrum from a moving time window. The Harmonic Sum Spectrum is used to identify pitches and generate
a chromagram. A second derivative peak detection algorithm is used to detect note onset and duration.
Note starts and end times are aligned based on overlapping and adjacent notes. Note duration is then assigned
to the closest matching assignable note duration. The inaccuracies in the results illustrate challenges in
music transcription such as octave errors, interference from harmonics, note decay overlap, and beat detection.
'''

import numpy as np
from scipy.io import wavfile
import matplotlib.pyplot as plt
from sklearn import linear_model
from sklearn.metrics import r2_score
import abjad as ab
from tkinter import *
from tkinter.ttk import *

# Graphic User Interface for Parameter Settings

loopactive=True
window = Tk(className=' Settings')
frame = Frame(window,width=1200, height=800)
frame.pack(fill=BOTH, expand=True )

filelabel = Label(frame,text='File Path and Name')
fileentry = Entry(frame,width=80)

headinglabel1 = Label(frame,text='PITCH RECOGNITION')

intervallabel = Label(frame,text='Moving FFT Window Size (ms)')
intervalentry = Entry(frame,width=5)
intervalentry.insert(0,300)

steplabel = Label(frame,text='Moving FFT Step Size (ms)')
stepentry = Entry(frame,width=5)
stepentry.insert(0,10)

upfreqlabel = Label(frame,text='Upper Frequency (Hz)')
upfreqentry = Entry(frame,width=5)
upfreqentry.insert(0,8000)

lowfreqlabel = Label(frame,text='Lower Frequency (Hz)')
lowfreqentry = Entry(frame,width=5)
lowfreqentry.insert(0,20)

maxharmlabel = Label(frame,text='Maximum Number of Harmonics')
maxharmentry = Entry(frame,width=5)
maxharmentry.insert(0,20)

thresholdlabel = Label(frame,text='HSS Threshold (Fraction of Maximum FFT Coefficient for Exclusion)')
thresholdentry = Entry(frame,width=5)
thresholdentry.insert(0,0.04)

minthresholdlabel = Label(frame,text='Minimum Absolute HSS Threshold')
minthresholdentry = Entry(frame,width=8)
minthresholdentry.insert(0,50000)

detectionlabel = Label(frame,text='Minimum Absolute Threshold for Undertone Subtraction')
detectionentry = Entry(frame,width=8)
detectionentry.insert(0,50000)

adjustmentlabel = Label(frame,text='Tuning Adjustment (Half-Tones)')
adjustmententry = Entry(frame,width=5)
adjustmententry.insert(0,0.1)

headinglabel2 = Label(frame,text='TRANSCRIPTION REGION')

analysisstartlabel = Label(frame,text='Start Transcription (points)')
analysisstartentry = Entry(frame,width=5)
analysisstartentry.insert(0,0)

analysisstoplabel = Label(frame,text='End Transcription (points)')
analysisstopentry = Entry(frame,width=5)
analysisstopentry.insert(0,1000)

analysissteplabel = Label(frame,text='Transcription Step Size (points)')
analysisstepentry = Entry(frame,width=5)
analysisstepentry.insert(0,1)

headinglabel3 = Label(frame,text='NOTE DETECTION')

onsetthresholdlabel = Label(frame,text='Note Onset Absolute Threshold')
onsetthresholdentry = Entry(frame,width=5)
onsetthresholdentry.insert(0,1000000)

onsetfractionlabel = Label(frame,text='Note Onset Relative Threshold (Fraction of Maximum)')
onsetfractionentry = Entry(frame,width=5)
onsetfractionentry.insert(0,0.5)

extremawindow1label = Label(frame,text='Extrema Detection Window (transcription steps)')
extremawindow1entry = Entry(frame,width=5)
extremawindow1entry.insert(0,10)

firstderivwindowlabel = Label(frame,text='First Derivative Window (transcription steps)')
firstderivwindowentry = Entry(frame,width=5)
firstderivwindowentry.insert(0,10)

secondderivwindowlabel = Label(frame,text='Second Derivative Window (transcription steps)')
secondderivwindowentry = Entry(frame,width=5)
secondderivwindowentry.insert(0,10)

extremawindow2label = Label(frame,text='Derivative Extrema Detection Window (transcription steps)')
extremawindow2entry = Entry(frame,width=5)
extremawindow2entry.insert(0,10)

maxmatchwindowlabel = Label(frame,text='Signal & Second Derivative Match Window (transcription steps)')
maxmatchwindowentry = Entry(frame,width=5)
maxmatchwindowentry.insert(0,15)

headinglabel4 = Label(frame,text='RHYTHM ALIGNMENT')

shortthresholdlabel = Label(frame,text='Threshold for Short Note Exclusion (transcription steps)')
shortthresholdentry = Entry(frame,width=5)
shortthresholdentry.insert(0,5)

alignthresholdlabel = Label(frame,text='Discrepancy Threshold for Track Alignment (transcription steps)')
alignthresholdentry = Entry(frame,width=5)
alignthresholdentry.insert(0,5)

alignendsthresholdlabel = Label(frame,text='Threshold for Note End-Note Start Alignment (transcription steps)')
alignendsthresholdentry = Entry(frame,width=5)
alignendsthresholdentry.insert(0,10)

durationbiaslabel = Label(frame,text='Even Note Duration Bias (32nd note)')
durationbiasentry = Entry(frame,width=5)
durationbiasentry.insert(0,0.25)

subdivisionslabel = Label(frame,text='Subdivisions for Tempo Determination')
subdivisionsentry = Entry(frame,width=5)
subdivisionsentry.insert(0,1)

evaluationwindowlabel = Label(frame,text='Tempo Window for Evaluation (40/FFT step)')
evaluationwindowentry = Entry(frame,width=5)
evaluationwindowentry.insert(0,10)

restadjustmentgainlabel = Label(frame,text='Threshold Improvement for Length Adjustment')
restadjustmentgainentry = Entry(frame,width=5)
restadjustmentgainentry.insert(0,5)

adjustmentwindowlabel = Label(frame,text='Allowable Window for Length Adjustment')
adjustmentwindowentry = Entry(frame,width=5)
adjustmentwindowentry.insert(0,10)

headinglabel5 = Label(frame,text='OUTPUT FORMAT')

analysisstaveslabel = Label(frame,text='Output Staves')
stavesanalyzed= StringVar()
analysisstavescombo = Combobox(frame, textvariable=stavesanalyzed)
analysisstavescombo['values']=['Treble','Bass','Treble+Bass']
stavesanalyzed.set('Treble+Bass')

# OK button escapes loop

def done():
    global loopactive
    window.update_idletasks()
    window.withdraw()
    loopactive = False
okbutton =Button(frame, text='OK', command=done)

filelabel.grid(row=0, column = 0, sticky=E)
fileentry.grid(row=0, column = 1, sticky=W)

headinglabel1.grid(row=1, column = 0, sticky=E)

intervallabel.grid(row=2, column = 0, sticky=E)
intervalentry.grid(row=2, column = 1, sticky=W)
steplabel.grid(row=3, column = 0, sticky=E)
stepentry.grid(row=3, column = 1, sticky=W)
upfreqlabel.grid(row=4, column = 0, sticky=E)
upfreqentry.grid(row=4, column = 1, sticky=W)
lowfreqlabel.grid(row=5, column = 0, sticky=E)
lowfreqentry.grid(row=5, column = 1, sticky=W)
maxharmlabel.grid(row=6, column = 0, sticky=E)
maxharmentry.grid(row=6, column = 1, sticky=W)
thresholdlabel.grid(row=7, column = 0, sticky=E)
thresholdentry.grid(row=7, column = 1, sticky=W)
minthresholdlabel.grid(row=8, column = 0, sticky=E)
minthresholdentry.grid(row=8, column = 1, sticky=W)
detectionlabel.grid(row=9, column = 0, sticky=E)
detectionentry.grid(row=9, column = 1, sticky=W)
adjustmentlabel.grid(row=10, column = 0, sticky=E)
adjustmententry.grid(row=10, column = 1, sticky=W)

headinglabel2.grid(row=11, column = 0, sticky=E)

analysisstartlabel.grid(row=12, column = 0, sticky=E)
analysisstartentry.grid(row=12, column = 1, sticky=W)
analysisstoplabel.grid(row=13, column = 0, sticky=E)
analysisstopentry.grid(row=13, column = 1, sticky=W)
analysissteplabel.grid(row=14, column = 0, sticky=E)
analysisstepentry.grid(row=14, column = 1, sticky=W)

headinglabel3.grid(row=15, column = 0, sticky=E)

onsetthresholdlabel.grid(row=16, column = 0, sticky=E)
onsetthresholdentry.grid(row=16, column = 1, sticky=W)
onsetfractionlabel.grid(row=17, column = 0, sticky=E)
onsetfractionentry.grid(row=17, column = 1, sticky=W)
extremawindow1label.grid(row=18, column = 0, sticky=E)
extremawindow1entry.grid(row=18, column = 1, sticky=W)
firstderivwindowlabel.grid(row=19, column = 0, sticky=E)
firstderivwindowentry.grid(row=19, column = 1, sticky=W)
secondderivwindowlabel.grid(row=20, column = 0, sticky=E)
secondderivwindowentry.grid(row=20, column = 1, sticky=W)
extremawindow2label.grid(row=21, column = 0, sticky=E)
extremawindow2entry.grid(row=21, column = 1, sticky=W)
maxmatchwindowlabel.grid(row=22, column = 0, sticky=E)
maxmatchwindowentry.grid(row=22, column = 1, sticky=W)

headinglabel4.grid(row=23, column = 0, sticky=E)

shortthresholdlabel.grid(row=24, column = 0, sticky=E)
shortthresholdentry.grid(row=24, column = 1, sticky=W)
alignthresholdlabel.grid(row=25, column = 0, sticky=E)
alignthresholdentry.grid(row=25, column = 1, sticky=W)
alignendsthresholdlabel.grid(row=26, column = 0, sticky=E)
alignendsthresholdentry.grid(row=26, column = 1, sticky=W)
durationbiaslabel.grid(row=27, column = 0, sticky=E)
durationbiasentry.grid(row=27, column = 1, sticky=W)
subdivisionslabel.grid(row=28, column = 0, sticky=E)
subdivisionsentry.grid(row=28, column = 1, sticky=W)
evaluationwindowlabel.grid(row=29, column = 0, sticky=E)
evaluationwindowentry.grid(row=29, column = 1, sticky=W)
restadjustmentgainlabel.grid(row=30, column = 0, sticky=E)
restadjustmentgainentry.grid(row=30, column = 1, sticky=W)
adjustmentwindowlabel.grid(row=31, column = 0, sticky=E)
adjustmentwindowentry.grid(row=31, column = 1, sticky=W)

headinglabel5.grid(row=32, column = 0, sticky=E)

analysisstaveslabel.grid(row=33, column = 0, sticky=E)
analysisstavescombo.grid(row=33, column = 1, sticky=W)

okbutton.grid(row=34, column = 1, sticky = W)

while loopactive:
    window.update()

# PARAMETER SETTING

# Import specified .wav file
file =fileentry.get()

# Define moving window parameters
interval = int(intervalentry.get()) # Time interval for fft in ms
step = int(stepentry.get()) # Step size in ms
upperfreq = int(upfreqentry.get())
lowerfreq = int(lowfreqentry.get())
maxharm = int(maxharmentry.get())
threshold = float(thresholdentry.get())
minthreshold = int(minthresholdentry.get())
detection = int(detectionentry.get())
adjustment = float(adjustmententry.get()) # Adjustment for out of tune instrument or alternative tuning

# Analysis Region
analysisstart = int(analysisstartentry.get())
analysisstop = int(analysisstopentry.get())
analysisstep = int(analysisstepentry.get())

# Note Detection Parameters
onsetthreshold = int(onsetthresholdentry.get())
onsetfraction =float(onsetfractionentry.get())
extremawindow1 = int(extremawindow1entry.get())
firstderivwindow = int(firstderivwindowentry.get())
secondderivwindow = int(secondderivwindowentry.get())
extremawindow2 = int(extremawindow2entry.get())
maxmatchwindow = int(maxmatchwindowentry.get())

# Rhythm Alignment Parameters
shortthreshold = int(shortthresholdentry.get())
alignthreshold = int(alignthresholdentry.get())
alignendsthreshold = int(alignendsthresholdentry.get())
durationbias = float(durationbiasentry.get())
subdivisions = int(subdivisionsentry.get())
evaluationwindow = int(evaluationwindowentry.get())
restadjustmentgain = int(restadjustmentgainentry.get())
adjustmentwindow = int(adjustmentwindowentry.get())
analysisstaves = stavesanalyzed.get()

window.destroy()

# Read Audio File
rate, audio = wavfile.read(file)

# Calculation of Other Basic Parameters
intervalpts = int(interval*rate/1000) # Time interval in data points
steppts = int(step*rate/1000) # Step in data points
numfreq = int((upperfreq/rate)*intervalpts)

# Average stereo channels
if audio.shape[1]>1:
    audio = np.mean(audio,axis=1)

# Frequency axis points
freq = (np.arange(int(intervalpts/2))/interval*1000)[:numfreq]

# Fast Fourier Transform function
def amplitudes(steps):
    segment = audio[steps*steppts:steps*steppts+intervalpts:1]
    return abs(np.fft.rfft(segment))[:int(intervalpts/2)][:numfreq]

# Harmonic Product Spectrum(input result of fft)
def hps(spectrum):
    hpspectrum = (spectrum[:]-max(minthreshold,threshold*max(spectrum)))/max(spectrum) # Filter and normalize
    hpspectrum+=1 # To ensure no signals are decreased during multiplication
    for i in range(0,numfreq):
        if hpspectrum[i] <= 1: # So signals below threshold have no effect
            hpspectrum[i]=1
    for i in range(1+int((lowerfreq/rate)*intervalpts),numfreq//2):
        if hpspectrum[i] > 1:
            for j in range(1, min(maxharm, numfreq // i)): # Product of harmonic signals
                hpspectrum[i] *= hpspectrum[i * j]
    hpspectrum -=1
    if max(hpspectrum)!=0: # Rescale
        hpspectrum = hpspectrum*max(spectrum)/max(hpspectrum)
    return hpspectrum

# Harmonic Sum Spectrum(input result of fft)
def hss(spectrum):
    hsspectrum = (spectrum[:]-max(minthreshold,threshold*max(spectrum)))/max(spectrum) # Filter and normalize
    for i in range(0,numfreq):
        if hsspectrum[i] <= 0: # So signals below threshold have no effect
            hsspectrum[i]=0
    for i in range(1+int((lowerfreq/rate)*intervalpts),numfreq//2):
        if hsspectrum[i] > 0:
            for j in range(1, min(maxharm, numfreq // i)): # Sum of harmonic signals
                hsspectrum[i] += hsspectrum[i * j]
    if max(hsspectrum)!=0: # Rescale
        hsspectrum = hsspectrum*max(spectrum)/max(hsspectrum)
    return hsspectrum

# Detection Spectrum (input result of fft)
def fhss(spectrum): # Assumes a note is made up of a uniform distribution of harmonics
    harmsum = hss(spectrum)
    dspec = np.zeros(numfreq) # Difference spectrum (subtracting contributions from harmonics of lower notes)
    for i in range(3,numfreq): # Ignore lowest frequencies
        harmonics = 1/np.arange(2,min(i,maxharm)) # Harmonic series
        undertones = np.zeros(len(harmonics))
        for j in range(0,min(i,maxharm)-2):
            undertones[j]=harmsum[int(round(i*harmonics[j]))] # Returns lower note harmonics interfering with note
        hthreshold = np.dot(harmonics,undertones) # Theoretical contribution to signal from harmonics of lower notes
        if harmsum[i]>=detection and harmsum[i]>= hthreshold:
            dspec[i]=(harmsum[i]-hthreshold)/harmsum[i]
    return dspec*harmsum

# Conversion of frequency to linear pitch space
def pitch(x):
    if x == 0:
        return -88
    else:
        return int(round(69 + adjustment + 12 * np.log2(x / 440))) # MIDI note numbering scale

vpitch= np.vectorize(pitch) # Vectorize conversion function
tone = vpitch(freq) # Conversion of frequency axis to pitch
tones = np.arange(-88,1+int(round(69 + 12 * np.log2(upperfreq / 440))))

# Function that bins and sums values according to pitch
def amplitude(steps):
    amp = np.zeros(89+int(round(69 + 12 * np.log2(upperfreq / 440))))
    x = fhss(amplitudes(steps))
    for i in range(0, len(tone)):
        amp[tone[i]+88] += x[i]
    return amp

# Moving Average
def movingavg(x, n):
    a = np.cumsum(x,axis=1)
    return (a-np.concatenate((np.zeros((len(a),n)),a[:,:-n]),axis=1))/n

# First Derivative
def deriv(x,n):
    rshiftx = np.concatenate((np.zeros((len(x),n)),x[:,:-n]),axis=1)
    lshiftx = np.concatenate((x[:,n:],np.zeros((len(x), n))), axis=1)
    return lshiftx-rshiftx

# Second Derivative
def deriv2(x,n,m):
    return deriv(deriv(x,n),m)

# Extrema detection
def extrema(x,n):
    a = np.zeros_like(x)
    for i in range(0,x.shape[0]):
        for j in range(n,x.shape[1]-n):
            counter = 0
            for k in range(1,n+1):
                if x[i,j]>= x[i,j-k] and x[i,j]>=x[i,j+k]: # For maxima
                    counter +=1
                elif x[i,j]<= x[i,j-k] and x[i,j]<=x[i,j+k]: # For minima
                    counter -=1
            if counter == n:
                a[i,j]=1
            elif counter == -1*n:
                a[i,j]=-1
    return a

# Threshold activation (0 if below threshold, 1 if above), to be used on chromagram
def thresh(x,limit,fraction):
    a = np.zeros_like(x)
    maxarray = x.max(axis=0)
    meanarray = np.mean(x,axis=0)
    for i in range(0, x.shape[0]):
        for j in range(0, x.shape[1]):
            if x[i,j] >= limit and (x[i,j]/maxarray[j]>=fraction or x[i,j]/meanarray[j]>=25):
                a[i,j] = 1
    return a

# Boundary, to be used on threshold activation array of chromagram
def boundary(x):
    a = np.zeros_like(x)
    for i in range(0, x.shape[0]):
        for j in range(1, x.shape[1]-1):
            if x[i,j]>0 and x[i,j-1]<=0: # For start
                a[i,j]=1
            elif x[i,j]>0 and x[i,j+1]<=0: # For end
                a[i,j]=-1
    return a

# Match Maxima (x should be d2/dt2 and y should be chromagram extrema array)
def matchmax(x,y,n):
    a = np.zeros_like(x)
    for i in range(0,x.shape[0]):
        for j in range(n,x.shape[1]-n):
            for k in range(0,n+1):
                if y[i,j-k]>0 or y[i,j+k]>0:
                    a[i,j]=1
    return a*x

# Chromagram generation
time = []
chromagramT = []

def chromagraph(start,stop,step): # Generates chromagram
    global time
    global chromogramT
    for i in range(start, stop//step):
        time.append(i * step) # Create list of time values
        x = amplitude(i * step) # Create list of signal values
        chromagramT.append(x)
    chromagram = np.array(chromagramT)
    chromagram = np.transpose(chromagram)
    chromagram = movingavg(chromagram, 40//step) # Smooth via moving average
    time = np.array(time)
    return chromagram

# Function marks the start and end of each note in each tone channel
def findnotes(x): # To be used on a chromagram
    t = thresh(x,onsetthreshold,onsetfraction) # Array entry is 1 if within a note (signal above threshold) and 0 if not
    bounds = boundary(t) # Entry is 1 if at the beginning of the note, -1 if at end, 0 otherwise
    exx = extrema(x,extremawindow1)
    d2x = -1*deriv2(x,firstderivwindow,secondderivwindow) # d2x/dt2 of chromagram
    exd2x = extrema(d2x,extremawindow2) # 1 if d2x/dt2 maximum, -1 if d2x/dt2 minimum
    m = matchmax(exd2x,exx,maxmatchwindow) # 1 if d2x/dt2 maximum matches x(t) maximum
    a = np.zeros_like(x) # Array indicating note start and stop
    for i in range(0, x.shape[0]):
        tracker = 0
        lastend = 0
        firststart = 0
        for j in range(1, x.shape[1] - 1):
            if t[i, j] == 1 and exd2x[i, j] == 1:
                tracker += 1 # Within a note, each d2x/dt2 maximum is counted
                lastend = j
            elif t[i, j] == 0:
                if bounds[i,j-1]==-1: # If at the end of a note
                    if m[i,lastend] == 0 and lastend !=firststart: # If the last d2x/dt2 maximum is not an x(t) maximum
                        a[i,lastend]=-1 # It is set as the note end
                    elif tracker != 0:
                        a[i,j-1]=-1 # Otherwise the note end is set once signal falls below threshold
                lastend = 0
                firststart = 0
                tracker = 0  # Resets to 0 when signal falls below threshold
            if tracker == 1 and exd2x[i, j] == 1: # For first d2x/dt2 maximum in a note
                a[i, j] = 1 # Code for start of a note
                firststart = j
            elif tracker > 1 and m[i, j] > 0: # For subsequent d2x/dt2 maxima that are also x(t) maxima
                a[i, j-1] = -1
                a[i, j] = 1
    return a

# Function creates array of all note starts
def starts(x): # To be used on findnotes() output
    xstarts = np.zeros(x.shape[1])
    for i in range(0, x.shape[1]):
        for j in range(0, x.shape[0]):
            if x[j, i] == 1:
                xstarts[i] = 1
    return xstarts

# Function creates array of all note ends
def ends(x): # To be used on findnotes() output
    xends = np.zeros(x.shape[1])
    for i in range(0, x.shape[1]):
        for j in range(0, x.shape[0]):
            if x[j, i] == -1:
                xends[i] = -1
    return xends

# Function removes notes shorter than threshold value
def removeshort(x,n):
    for i in range(0,x.shape[0]):
        tracker = 0
        laststart= 0
        for j in range(0, x.shape[1]):
            if x[i,j] == 1:
                laststart = j
                tracker = 0
            elif x[i,j] == 0:
                tracker +=1
            elif x[i,j] ==-1:
                laststop = j
                if tracker < n:
                    x[i,laststart]=0
                    x[i,laststop]=0
                tracker = 0
    return x

# Function aligns note start from different channels (perform before splitting staves)
def alignstart(x,n): # Modifies findnotes() output
    xstarts = starts(x)
    zerocount = 0
    clusterlist = []
    for i in range(0,len(xstarts)):
        if xstarts[i]==0:
            zerocount +=1
        else:
            zerocount = 0
            clusterlist.append(i)
        if zerocount > n:
            if clusterlist:
                avg = int(round(sum(clusterlist) / len(clusterlist)))
                for i in clusterlist:
                    for j in range(0, x.shape[0]):
                        if x[j,i]==1:
                            x[j,i]=0
                            x[j, avg] = 1
                clusterlist = []
    return x

# Function aligns note ends with beginning of next note (perform before and after splitting staves)
def alignends(x,n): # Modifies findnotes() output
    xstarts = starts(x)
    for i in range(0,x.shape[0]):
        spacecount = 0
        startcount = False
        note = False
        lastend = None
        for j in range(0,x.shape[1]):
            if xstarts[j]==1 and note: # Aligns note end with next note start
                note = False
                startcount = False
                if spacecount < n and j>0: # Aligns only if space is below a threshold n
                    if lastend and lastend !=j-1:
                        x[i, j - 1] = -1
                        x[i, lastend] = 0
                    elif not lastend:
                        x[i, j - 1] = -1
                spacecount = 0
                lastend = None
            if x[i,j]==1:
                note = True
            if x[i,j]==-1:
                if not note:
                    x[i,j]=0
                else:
                    startcount=True
                    lastend = j
            if startcount and note and xstarts[j]==0:
                spacecount +=1
    endcount = 0
    endlist = []
    xends = ends(x)
    for i in range(0, x.shape[1]): # Aligns ends of different notes to the same time
        if xstarts[i] == 1:
            if xends[i] == -1:
                endcount += 1
                endlist.append(i-1)
            if endcount > 1:
                for j in range(0, x.shape[0]):
                    hasend = False
                    for k in endlist:
                        if x[j, k] == -1:
                            x[j, k] = 0
                            hasend = True
                    if hasend:
                        x[j, max(endlist)] = -1
            endcount = 0
            endlist = []
        elif xends[i] == -1:
            endcount += 1
            endlist.append(i)
    return x

# Combined starts and ends
def showstartend(x):
    start = np.zeros(x.shape[1])
    end = np.zeros(x.shape[1])
    for i in range(0,x.shape[1]):
        for j in range(0,x.shape[0]):
            if x[j,i]==1:
                start[i]+=1
            elif x[j,i]==-1:
                end[i]+=-1
    return (start,end)

# Function creates rest channel (perform after splitting staves and aligning ends)
def rest(x): # To be used on findnotes() output
    a = np.cumsum(x,axis=1)
    b = np.zeros_like(x)
    for i in range(0, x.shape[0]):
        for j in range(1, x.shape[1]):
            if a[i,j]==0 and a[i,j-1]==0:
                b[i,j]=1
    c = np.prod(b,axis=0) # Entry is 1 only if all channels are empty
    d = np.zeros_like(c)
    for j in range(0, x.shape[1]):
        if j == 0:
            d[j] = 0
        elif j == x.shape[1]-1:
            d[j] = -1
        elif c[j] == 1 and c[j-1]==0:
            d[j]=1
        elif c[j]==1 and c[j+1]==0:
            d[j]=-1
    return np.concatenate((x,np.expand_dims(d,axis=0)),axis=0)

# Function turns start and stop into a list of note start times and durations
def tabulate(x): # To be used on findnotes() output with rest channel
    a = starts(x)
    b = ends(x)
    chordlist = []
    currentnotes = []
    notestart = 0
    duration = 0
    for i in range(0,x.shape[1]):
        if a[i]==1:
            if b[i] == -1:
                chordlist.append([notestart, duration, currentnotes])
            notestart = i
            duration = 0
            currentnotes = []
            for j in range(0, x.shape[0]):
                if x[j,i]==1:
                    if j < x.shape[0]-1:
                        currentnotes.append(j)
                    elif j == x.shape[0]-1:
                        currentnotes.append('r')
        elif a[i]==0 and b[i]==0:
            duration +=1
        elif b[i]==-1:
            duration +=1
            chordlist.append([notestart,duration,currentnotes])
    return chordlist

def fillin(chordlist):
    newchordlist = chordlist[:]
    j = 0
    for i in range(len(chordlist)-1):
        if chordlist[i][0]+chordlist[i][1]+1 != chordlist[i+1][0]:
            start = chordlist[i][0]+chordlist[i][1]+1
            end = chordlist[i+1][0]-1
            newchordlist.insert(j+1, [start,end-1-start,['r']])
            j+=1
        j += 1
    return newchordlist

# Allowable note durations
bias = durationbias
durationbins = np.array([[0,1,1.5-bias],
                    [1.5-bias,2,2.5+bias],
                    [2.5+bias,3,3.5-bias],
                    [3.5-bias,4,5+bias],
                    [5+bias,6,7],
                    [7,8,10],
                    [10,12,14],
                    [14,16,20],
                    [20,24,28],
                    [28,32,40],
                    [40,48,56],
                    [56,64,72]])

evendurationbins = np.array([[0,2,3],
                    [3,4,5],
                    [5,6,7],
                    [7,8,10],
                    [10,12,14],
                    [14,16,20],
                    [20,24,28],
                    [28,32,40],
                    [40,48,56],
                    [56,64,72]])

# Function aligns durations in a new column in the chordlist
def subdivide(x1,x2=None,n=1,w=10,guess = None):
    dt = 40/step
    initialtempo = dt
    total = (x1[-1][0] + x1[-1][1])
    cut =  total/n
    cuts = []
    for i in range(0,n): # Splits into n time segments, in case of tempo variability
        if i ==0:
            cuts.append(0)
        else:
            if x2:
                for j in range(int(i * cut), total):
                    if [a for a in x1 if a[0] == j] != [] and [a for a in x2 if a[0] == j] != []:
                        if j not in cuts:
                            cuts.append(j)
                        break
            else:
                for j in range(int(i * cut), total):
                    if [a for a in x1 if a[0] == j] != []:
                        if j not in cuts:
                            cuts.append(j)
                        break
    cuts.append(total+1)
    for i in range(0,n): # Create list of all durations within a time segment
        durationlist = []
        restdurationlist = []
        for x in x1:
            if cuts[i] <= x[0] < cuts[i+1]:
                if x[2]!=['r']:
                    durationlist.append(x[1])
                else:
                    restdurationlist.append(x[1])
        if x2:
            for x in x2:
                if cuts[i] <= x[0] < cuts[i + 1]:
                    if x[2]!=['r']:
                        durationlist.append(x[1])
                    else:
                        restdurationlist.append(x[1])
        def evaluatetempo(): # Evaluate loss function for a tempo
            loss = 0
            bins = dt*durationbins
            for a in restdurationlist:
                loss += (a/dt-round(a/dt))**2
            for a in durationlist:
                for b in bins:
                    if b[0] <= a < b[2]:
                        if b[1] <= 1 or b[1] >= 32: # Penalty for really short and really long notes
                            loss += ((a - b[1]) / max(b[1],dt*6)) ** 2 + 0.1
                        elif b[1]%2 !=0 and b[1]%2 !=0: # Penalty for shorter dotted notes
                            loss += ((a - b[1]) / max(b[1],dt*6)) ** 2 + 0.075
                        elif b[1] >= 16 or b[1]%4 !=0: # Penalty for dotted notes
                            loss += ((a - b[1]) / max(b[1],dt*6)) ** 2 + 0.05
                        else:
                            loss += ((a - b[1]) / max(b[1],dt*6)) ** 2
                        break
            return loss
        minloss = None
        finaltempo = None
        if i == 0: # Evaluate different tempos, and choose the one with minimum loss
            if not guess:
                for j in range(40, 200):
                    dt = j / step
                    tempo = evaluatetempo()
                    if minloss == None or tempo < minloss:
                        minloss = tempo
                        finaltempo = dt
                        initialtempo = j
            else:
                for j in range(int(guess*step-w), int(guess*step+w)):
                    dt = j / step
                    tempo = evaluatetempo()
                    if minloss == None or tempo < minloss:
                        minloss = tempo
                        finaltempo = dt
                        initialtempo = j
        else:
            for j in range(initialtempo-w, initialtempo+w):
                dt = j / step
                tempo = evaluatetempo()
                if minloss == None or tempo < minloss:
                    minloss = tempo
                    finaltempo = dt
                    initialtempo = j
        def bestfitdurations(clist):
            bins = finaltempo * durationbins
            evenbins = finaltempo * evendurationbins
            for a in clist:  # Append new durations onto chord list
                if cuts[i] <= a[0] < cuts[i + 1]:
                    if a[2] != ['r']:
                        for b in bins:
                            if b[0] <= a[1] < b[2]:
                                a.append(b[1] / finaltempo)
                                break
                    else:
                        a.append(round(a[1] / finaltempo))
            oddcount = 0
            for a in clist: # Change unpaired odd durations to even durations
                if cuts[i] <= a[0] < cuts[i + 1]:
                    if a[3] % 2 != 0:
                        oddcount = oddcount % 2
                        oddcount += 1
                    else:
                        oddcount = 0
                    if clist.index(a)<len(clist)-1:
                        if oddcount == 1 and clist[clist.index(a) + 1][3] % 2 == 0:
                            if a[2] != ['r']:
                                for b in evenbins:
                                    if b[0] <= a[1] < b[2]:
                                        a[3] = (b[1] / finaltempo)
                                        break
                            else:
                                a[3] = round(a[1] / finaltempo / 2) * 2
        bestfitdurations(x1)
        if x2:
            bestfitdurations(x2)
        if i == 0:
            firsttempo = finaltempo
    return firsttempo

# Adjusts duration to align with beats
def adjustduration(x,gain=10,window=10): # Input aligned chordlist
    a = []
    for b in x:
        a.append(b[3])
    a = np.array(a)
    def checkalignment(y):
        score = 0
        c = np.cumsum(y)
        for d in c: # Higher score if aligned with beat
            if d%2 == 0:
                score += 1/16
            if d%4 ==0:
                score += 1/8
            if d%8 ==0:
                score += 1/4
            if d%16 ==0 or d%24 ==0:
                score += 1/2
            if d%32 ==0 or d%48 ==0:
                score += 1
        return score
    currentscore = checkalignment(a)
    optimum = a[0]
    for i in range(0,32):
        a[0]=i
        alignment = checkalignment(a)
        if alignment > currentscore:
            currentscore = alignment
            optimum = i
    a[0]=optimum
    for i in range(1,len(a)):
        if x[i][2] != ['r']: # Adjust length of rests if alignment improves more than a threshold (gain)
            currentscore = checkalignment(a)
            shortalignment  = 0
            longalignment  = 0
            for i in range(0,len(durationbins)):
                if a[i] == durationbins[i, 1]:
                    if 9 > i > 2:
                        a[i] = durationbins[i - 1, 1]
                        shortalignment = checkalignment(a)
                    if 8 > i > 0:
                        a[i] = durationbins[i + 1, 1]
                        longalignment = checkalignment(a)
                    if longalignment > shortalignment and longalignment - currentscore > gain:
                        a[i]=durationbins[i + 1, 1]
                    elif longalignment < shortalignment and shortalignment - currentscore > gain:
                        a[i] = durationbins[i - 1, 1]
                    else:
                        a[i]= durationbins[i, 1]
        else: # Adjust length of notes if alignment improves more than a threshold (gain)
            optimum = a[i]
            currentscore = checkalignment(a)
            bestscore = currentscore
            for j in range(-1*window,window+1):
                a[i] += 0.5*j
                alignment  = checkalignment(a)
                if alignment > bestscore:
                    bestscore = alignment
                    optimum = a[i]
                a[i] -= 0.5 * j
            if bestscore - currentscore > gain:
                a[i] = optimum
    newstarts = np.concatenate((np.zeros(1),np.cumsum(a)))
    for i in range(0,len(a)):
        x[i][3]=a[i]
        x[i].append(newstarts[i])

def trackadjust(x,combinedx): # Align treble and bass with adjusted combined chordlist
    x[0][3] = [b[3] for b in combinedx if b[0]==x[0][0]][0]
    for a in x:
        if [b[3] for b in combinedx if b[0]==a[0]]:
            if [b[3] for b in combinedx if b[0] == a[0]][0] > a[3] and a[2]!=['r']:
                a[3] = [b[3] for b in combinedx if b[0] == a[0]][0]
            a.append([b[4] for b in combinedx if b[0]==a[0]][0])
    for a in x:
        if x.index(a)!= len(x)-1 and x.index(a)!=0:
            if not [b[3] for b in combinedx if b[0] == a[0]]:
                a.append(x[x.index(a) - 1][4] + x[x.index(a) - 1][3])
                a[3] = x[x.index(a) + 1][4] - a[4]
            elif a[2] == ['r']:
                a[4]=x[x.index(a) - 1][4] + x[x.index(a) - 1][3]
                a[3] = x[x.index(a) + 1][4] - a[4]
        elif x.index(a)== len(x)-1:
            if not [b[3] for b in combinedx if b[0] == a[0]]:
                a.append(x[x.index(a) - 1][4] + x[x.index(a) - 1][3])
                a[3] = 8
            elif a[2] == ['r']:
                a[4]=x[x.index(a) - 1][4] + x[x.index(a) - 1][3]
                a[3] = 8

# Meter Detection
def meterdetect(x):
    a = []
    for b in x:
        a.append(b[3])
    a = np.array(a)
    score4 = 0
    score3 = 0
    c = np.cumsum(a)
    for d in c:
        if d % 16 == 0:
            score4 += 1/2
        if d % 12 == 0:
            score3 += 1/2
        if d % 32 == 0:
            score4 += 1
        if d % 24 == 0:
            score3 += 1
    if score3/score4 > 1.5:
        return 3
    else:
        return 4

# Scale Basis
majorfilt = [[1,0,1,0,1,1,0,1,0,1,0,1],
 [1,1,0,1,0,1,1,0,1,0,1,0],
 [0,1,1,0,1,0,1,1,0,1,0,1],
 [1,0,1,1,0,1,0,1,1,0,1,0],
 [0,1,0,1,1,0,1,0,1,1,0,1],
 [1,0,1,0,1,1,0,1,0,1,1,0],
 [0,1,0,1,0,1,1,0,1,0,1,1],
 [1,0,1,0,1,0,1,1,0,1,0,1],
 [1,1,0,1,0,1,0,1,1,0,1,0],
 [0,1,1,0,1,0,1,0,1,1,0,1],
 [1,0,1,1,0,1,0,1,0,1,1,0],
 [0,1,0,1,1,0,1,0,1,0,1,1]]

minorfilt = [[1,0,1,1,0,1,0,0.5,0.5,0.5,0.5,1],
 [1,1,0,1,1,0,1,0,0.5,0.5,0.5,0.5],
 [0.5,1,1,0,1,1,0,1,0,0.5,0.5,0.5],
 [0.5,0.5,1,1,0,1,1,0,1,0,0.5,0.5],
 [0.5,0.5,0.5,1,1,0,1,1,0,1,0,0.5],
 [0.5,0.5,0.5,0.5,1,1,0,1,1,0,1,0],
 [0,0.5,0.5,0.5,0.5,1,1,0,1,1,0,1],
 [1,0,0.5,0.5,0.5,0.5,1,1,0,1,1,0],
 [0,1,0,0.5,0.5,0.5,0.5,1,1,0,1,1],
 [1,0,1,0,0.5,0.5,0.5,0.5,1,1,0,1],
 [1,1,0,1,0,0.5,0.5,0.5,0.5,1,1,0],
 [0,1,1,0,1,0,0.5,0.5,0.5,0.5,1,1]]

# Triad Basis
majortriad = [[2,0,0,0,1,0,0,1.5,0,0,0,0],
 [0,2,0,0,0,1,0,0,1.5,0,0,0],
 [0,0,2,0,0,0,1,0,0,1.5,0,0],
 [0,0,0,2,0,0,0,1,0,0,1.5,0],
 [0,0,0,0,2,0,0,0,1,0,0,1.5],
 [1.5,0,0,0,0,2,0,0,0,1,0,0],
 [0,1.5,0,0,0,0,2,0,0,0,1,0],
 [0,0,1.5,0,0,0,0,2,0,0,0,1],
 [1,0,0,1.5,0,0,0,0,2,0,0,0],
 [0,1,0,0,1.5,0,0,0,0,2,0,0],
 [0,0,1,0,0,1.5,0,0,0,0,2,0],
 [0,0,0,1,0,0,1.5,0,0,0,0,2]]

minortriad = [[2,0,0,1,0,0,0,1.5,0,0,0,0],
 [0,2,0,0,1,0,0,0,1.5,0,0,0],
 [0,0,2,0,0,1,0,0,0,1.5,0,0],
 [0,0,0,2,0,0,1,0,0,0,1.5,0],
 [0,0,0,0,2,0,0,1,0,0,0,1.5],
 [1.5,0,0,0,0,2,0,0,1,0,0,0],
 [0,1.5,0,0,0,0,2,0,0,1,0,0],
 [0,0,1.5,0,0,0,0,2,0,0,1,0],
 [0,0,0,1.5,0,0,0,0,2,0,0,1],
 [1,0,0,0,1.5,0,0,0,0,2,0,0],
 [0,1,0,0,0,1.5,0,0,0,0,2,0],
 [0,0,1,0,0,0,1.5,0,0,0,0,2]]

# Note dictionary
tonedict = {0:'C',1:'C#/Db',2:'D',3:'D#/Eb',4:'E',5:'F',6:'F#/Gb',7:'G',8:'G#/Ab',9:'A',10:'A#/Bb',11:'B'}
tonicdict = {0:'c',1:'cs',2:'d',3:'ef',4:'e',5:'f',6:'gf',7:'g',8:'af',9:'a',10:'bf',11:'b'}

# Chromagram average value
def avgval(x):
    a = np.average(x,axis=1)
    a = np.transpose(a)
    if len(a)//12 != 0:
        a = np.concatenate((a,np.zeros(12-len(a)%12)))
    a = np.hsplit(a,len(a)/12)
    b = np.zeros(12)
    for x in a:
        b+=x
    b = np.concatenate((b[4:],b[:4]))
    return b

# Frequency of each tonic note is modeled as a linear combination of uniform distribution of all notes,
# uniform distribution of notes in scale, and triad notes
def findkey(chromagram):
    tonesummary = avgval(chromagram)
    majormax = 0
    majorkey = None
    for i in range(0, 12):
        X = np.transpose(np.vstack((np.ones(12), majorfilt[i], majortriad[i])))
        y = np.transpose(tonesummary)
        model = linear_model.LinearRegression()
        model.fit(X, y)
        ypred = model.predict(X)
        a = r2_score(y, ypred)
        if a > majormax:
            majormax = a
            majorkey = tonedict[i]
            majortonic = tonicdict[i]
    minormax = 0
    minorkey = None
    for i in range(0, 12):
        X = np.transpose(np.vstack((np.ones(12), minorfilt[i], minortriad[i])))
        y = np.transpose(tonesummary)
        model = linear_model.LinearRegression()
        model.fit(X, y)
        ypred = model.predict(X)
        a = r2_score(y, ypred)
        if a > minormax:
            minormax = a
            minorkey = tonedict[i]
            minortonic =tonicdict[i]
    if majormax >= minormax:
        print(majorkey + ' Major')
        return(majortonic,'major')
    else:
        print(minorkey + ' Minor')
        return(minortonic,'minor')

# Analysis Procedure

chromagram = chromagraph(analysisstart,analysisstop,analysisstep)

chromacombined = findnotes(chromagram)

chromacombined = removeshort(chromacombined,shortthreshold)

chromacombined = alignstart(chromacombined,alignthreshold)

# Splits the treble and bass tones
stavesplit = 60+88
chromatop = chromacombined[stavesplit:,:]
chromabottom = chromacombined[:stavesplit,:]

chromacombined = alignends(chromacombined,alignendsthreshold)
chromatop = alignends(chromatop,alignendsthreshold)
chromabottom = alignends(chromabottom,alignendsthreshold)

chromacombined = rest(chromacombined)
chromatop = rest(chromatop)
chromabottom = rest(chromabottom)

combinedlist = tabulate(chromacombined)
toplist = tabulate(chromatop)
bottomlist = tabulate(chromabottom)

for a in toplist:
    for i in range(0,len(a[2])):
        if a[2][i]!='r':
            a[2][i] += stavesplit

combinedlist = fillin(combinedlist)
tempo = subdivide(combinedlist,n=subdivisions,w=evaluationwindow)
subdivide(toplist,bottomlist,n=subdivisions,w=evaluationwindow,guess=tempo)
adjustedcombined = adjustduration(combinedlist,gain=restadjustmentgain,window=adjustmentwindow)

trackadjust(toplist,combinedlist)
trackadjust(bottomlist,combinedlist)

def removezeroes(x):
    for a in x:
        if a[3]==0:
            x.remove(a)
    for a in x: # To remove duplicate entries if any
        if a[3]==0:
            x.remove(a)

removezeroes(combinedlist)
removezeroes(toplist)
removezeroes(bottomlist)

# Plot amplitude of each pitch vs. time
def plotstaff(cgram,clist):
    toneset = set()
    for x in clist:
        if x[2] !=['r']:
            toneset = toneset.union(x[2])
    for x in toneset:
        plt.plot(time,cgram[x])

# Plot note start and end vs. time
def plotnotes(cgram,clist):
    toneset = set()
    for x in clist:
        if x[2] !=['r']:
            toneset = toneset.union(x[2])
    notes = np.cumsum(cgram, axis=1)
    for x in toneset:
        plt.plot(time,10**7*notes[x])

plotstaff(chromagram,combinedlist)
plt.title('Note Plot')
plt.xlabel('Time (ms)')
plt.ylabel('Signal')

# Duration conversion
def convertduration(x):
    x *= 2
    denominator = 64
    while x%2==0:
        x/=2
        denominator/=2
    return (x,denominator)

# Simplify fraction
def simplifyfraction(n,d):
    while n%2==0:
        n/=2
        d/=2
    return (n,d)

# Assignability List
assignable = [1,2,3,4,6,7,8,12,14,15,16,24,28,30,31,32,48,56,60,62,63,64]

# For non-assignable rest durations, split to two assignable durations
def splitduration(x):
    if x not in assignable:
        first = 0
        second = 0
        for a in assignable:
            if x-a in assignable:
                first = a
                second = x-a
                break
        return (first,second)

# Lists Notes and Chords
def listnotes(x): # Input a chord list
    notelist = []
    for a in x:
        if a[2]==['r']:
            d = convertduration(a[3])
            if d[0] not in assignable:
                splitd = splitduration(d[0])
                duration1 = simplifyfraction(splitd[0],d[1])
                duration2 = simplifyfraction(splitd[1],d[1])
                notelist.append(ab.Rest(ab.Duration(*duration1)))
                notelist.append(ab.Rest(ab.Duration(*duration2)))
            else:
                notelist.append(ab.Rest(ab.Duration(*d)))
        elif len(a[2])==1:
            notelist.append(ab.Note(a[2][0]-148,ab.Duration(*convertduration(a[3]))))
        elif len(a[2])>1:
            chordnotes = []
            for b in a[2]:
                chordnotes.append(b-148)
            notelist.append(ab.Chord(chordnotes,ab.Duration(*convertduration(a[3]))))
    return notelist

# Convert chordlists to musical score using Abjad
keysig = findkey(chromagram)
ksignature = ab.KeySignature(*keysig)

timesig = meterdetect(combinedlist)
tsignature = ab.TimeSignature((timesig,4))

def treblebass():
    trebleclef = ab.Clef('treble')
    treblestaff = ab.Staff(listnotes(toplist))
    ab.attach(trebleclef, treblestaff[0])
    ab.attach(ksignature, treblestaff[0])
    ab.attach(tsignature, treblestaff[0])

    bassclef = ab.Clef('bass')
    bassstaff = ab.Staff(listnotes(bottomlist))
    ab.attach(bassclef, bassstaff[0])
    ab.attach(ksignature, bassstaff[0])
    ab.attach(tsignature, bassstaff[0])

    group = ab.StaffGroup([treblestaff, bassstaff])
    ab.show(group)

def treble():
    trebleclef = ab.Clef('treble')
    treblestaff = ab.Staff(listnotes(combinedlist))
    ab.attach(trebleclef, treblestaff[0])
    ab.attach(ksignature, treblestaff[0])
    ab.attach(tsignature, treblestaff[0])
    ab.show(treblestaff)

def bass():
    bassclef = ab.Clef('bass')
    bassstaff = ab.Staff(listnotes(combinedlist))
    ab.attach(bassclef, bassstaff[0])
    ab.attach(ksignature, bassstaff[0])
    ab.attach(tsignature, bassstaff[0])
    ab.show(bassstaff)

# Plot Chromagram
fig, ax = plt.subplots()
ax.contourf(time,tones[88:],chromagram[88:,:],np.linspace(0,chromagram.max()//2,100),extend='both')
ax.set_facecolor('#440154')
ax.set_title('Chromagram')
ax.set_xlabel('Time (ms)')
ax.set_ylabel('MIDI Pitch Number')
plt.show()

# Generate score
if analysisstaves == 'Treble+Bass':
    treblebass()
elif analysisstaves == 'Bass':
    bass()
else:
    treble()


from socket import *
from select import *
from errno import *
import sys

HEADER_LENGTH = 10
PORT = 11000

user = input("Please enter name to join server: ")
client_socket = socket(AF_INET, SOCK_STREAM)
client_socket.connect(('', PORT))
client_socket.setblocking(False)

username = user.encode()
username_header = f"{len(username):<{HEADER_LENGTH}}".encode()
client_socket.send(username_header + username)

while True:
    message = input(f"{user} : ")

    if message:
        message = message.encode()
        message_header = f"{len(message):<{HEADER_LENGTH}}".encode('utf-8')
        client_socket.send(message_header + message)

    try:
        while True:
            username_header = client_socket.recv(HEADER_LENGTH)
            if not len(username_header):
                print("Connection clossed by server")
                sys.exit()

            username_length = int(username_header.decode())
            username = client_socket.recv(username_length).decode()

            message_header = client_socket.recv(HEADER_LENGTH)
            message_length = int(message_header.decode().strip())
            message = client_socket.recv(message_length).decode()

            print(f"{username} : {message}")

    except IOError as e:
        if e.errno != EAGAIN and e.errno != EWOULDBLOCK:
            print("Reading error", str(e))
            sys.exit()
        continue


from flask import Flask
from flask_sqlalchemy import SQLAlchemy
# import main
from config.config import DefaultConfig
from config.database import DatabaseConfig
from ..extensions import db
from ..carboard.models.platform import Platform
from ..carboard.models.signal import Signal
from ..carboard.models.extrasignal import Extrasignal
from .helpers import slugify
from .constants import ACTIVE
# -------------------- Transformer Class : Version 0.1 ---------------------- #

class Transformer:

    def __init__(self, **kwargs):
        self.registerDB()

    def registerDB(self):
        self.app = Flask(__name__)
        self.app.config.from_object(DefaultConfig)
        self.app.config.from_object(DatabaseConfig)
        db = SQLAlchemy(self.app)

    def setPlatformById(self, platform_id):
        with self.app.app_context():
            self.platform = Platform.query.filter_by(id=platform_id, status=ACTIVE).first()
        self.signals = self.getAvailable()
        return self

    def setPlatformBySlug(self, platform_slug):
        with self.app.app_context():
            self.platform = Platform.query.filter_by(slug=platform_slug, status=ACTIVE).first()
        self.signals = self.getAvailable()
        return self

    def getAvailable(self):
        extra = self.platform.signals
        signals = {}
        for x in extra:
            if x.status == 1 and x.signal.status == 1:
                signals[str(x.name)] = {
                    'main': x.signal.name,
                    'class': x.signal.signalclass.name
                }
        return signals

    def getInfo(self, name, ignore=True):
        x = {}
        if name in self.signals:
            x['name'] = self.signals[name]['main']
            x['class'] = self.signals[name]['class']
        elif ignore == False:
            x['name'] = name
            x['class'] = 'unknown'
        else:
            x = None
        return x

    def getMain(self, extra, ignore=None):
        extra = slugify(extra)
        extrasignal = None
        with self.app.app_context():
            extrasignal = Extrasignal.query.filter_by(name=extra, platform_id=self.platform.id).first()
        ignore = self.ignore if ignore is None else ignore

        if extrasignal is not None:
            main = extrasignal.signal
            if main.status == ACTIVE and extrasignal.status == ACTIVE:
                return {
                    'name': main.name,
                    'main': True,
                    'class': main.signalclass.name,
                    'error': None
                }
            elif not ignore:
                return {
                    'name': main.name,
                    'main': True,
                    'class': main.signalclass.name,
                    'error': 'Signal `{}` is disabled but added to database anyway ! '.format(extra)
                }
            else:
                return {
                    'name': None,
                    'main': None,
                    'class': None,
                    'error': 'Signal `{}` is not currently supported'.format(extra)
                }
        else:
            if not ignore:
                msg = extrasignal
                return {
                    'name': extra,
                    'main': False,
                    'class': 'unknown',
                    'error': 'Signal `{}` is not currently supported but added to database anyway !'.format(extra)
                }
            else:
                return {
                    'name': None,
                    'main': None,
                    'class': None,
                    'error': 'Signal `{}` is not currently supported'.format(extra)
                }

    def getCorrectValue(self, value):
        return value;

    def __str__(self):
        str = ""
        for k, v in self.vars.items():
            str = str + "{} : {} | ".format(k, v)
        return str

    def showVars(self):
        print(self.vars)

    def setVar(self, k, v):
        self.vars[k] = v

    def getVar(self, k):
        return self.vars.get(k, None)


import re
import json
import time

import redis
import gevent
from geventwebsocket import WebSocketServer, WebSocketApplication, Resource
from flask import Flask, send_from_directory

import conf

app = Flask(__name__)
# Catch-All URL
@app.route('/', defaults={'filename': ''})
@app.route('/<path:filename>')
def root(filename):
    # only serve static files for now
    if filename == '' or filename[-1] == '/':
        filename = filename + 'index.html'
    return send_from_directory('', filename)

class WSApp(WebSocketApplication):
    def on_open(self):
        print("Connected!")

    def on_close(self, reason):
        print("Connection closed!")

    def on_message(self, message):
        if message is None:
            pass # closed

server = WebSocketServer(('', conf.port), Resource([
        ('/info', WSApp),
        ('/', app)
    ]))

def keys():
    r = redis.StrictRedis(host=conf.redis_host, port=conf.redis_port)
    info = r.info()
    return sum(v['keys'] for k,v in info.items() if re.match(r'db\d', k))

def beat():
    while True:
        gevent.sleep(conf.interval)
        if hasattr(server, 'clients') and server.clients:
            data = json.dumps({
                'time': time.strftime('%M:%S'),
                'keys': keys()
            })
            for client in server.clients.values():
                client.ws.send(data)

gevent.spawn(beat)

print('listening at %d...' % conf.port)
server.serve_forever()


from django.urls import path
from . import views

app_name = 'user'
urlpatterns = [
    path('', views.login, name='login'),
    path('logout/', views.logout, name='logout'),
    path('login_check/', views.login_check),
    path('register/', views.register),
    path('register_check/', views.register_check),
    path('findID/', views.findID),
    path('findID_check/', views.findID_check),
    path('findPW/', views.findPW),
    path('findPW_check/', views.findPW_check),
    path('id_overlap_check/', views.id_overlap_check),
]


import pandas as pd
import matplotlib.pyplot as plt

# data_file = 'results-sender-1Gb.result'
# data_file = 'receiver-clam-10Mb-55ms.result'
data_file = 'receiver-localhost-10Mb-55ms.result'

# title = '1Gb LAN'
# title = '''Atlanta - Madison
# 10Mb, 55ms'''
title = '''localhost and tc
10Mb, 55ms, burst 100Mb'''

# skip_test_names = ['empms']
skip_test_names = []

# Read the CSV data into a Pandas dataframe
df = pd.read_csv(data_file, comment='#')
print(df)
for skip_test_name in skip_test_names:
    df = df.drop(df[df['test name'] == skip_test_name].index)
print(df)

# Group the data by test name and number of bytes, and calculate the mean wall clock time
grouped_df = df.groupby(['test name', 'number of bytes']).mean(
    numeric_only=True).reset_index()
print(grouped_df)

# Get a list of all unique test names and number of bytess
test_names = grouped_df['test name'].unique()
data_sizes = grouped_df['number of bytes'].unique()

# Set up the plot
fig, ax = plt.subplots()

# Set the width of each bar and the spacing between groups of bars
bar_width = 0.2
spacing = 0.01

for i, test_name in enumerate(test_names):
    # Create a series of wall clock times for the current test name
    test_series = grouped_df[grouped_df['test name'] ==
                             test_name][['wall clock us', 'number of bytes']]
    print(test_series)

    # Calculate the x positions for the bars in the current group
    x_positions = [i * (bar_width + spacing) +
                   j for j in range(len(data_sizes))]

    # Create the bars for the current group
    ax.bar(x_positions, test_series['wall clock us'],
           width=bar_width, label=test_name)

# Set the x ticks and labels for each group of bars
ax.set_xticks([i + (len(data_sizes) - 1) / 2 * (bar_width + spacing)
              for i in range(len(data_sizes))])
ax.set_xticklabels(data_sizes)

# Add labels and legend
ax.set_xlabel('number of bytes')
ax.set_ylabel('Wall Clock Time (us)')
ax.legend()

plt.title(title)

plt.show()


# Adopted from https://github.com/Limingxing00/RDE-VOS-CVPR2022
# under MIT License

import math

import torch

def softmax_w_top(x, top):
    values, indices = torch.topk(x, k=top, dim=1)
    x_exp = values.exp_()

    x_exp /= torch.sum(x_exp, dim=1, keepdim=True)
    x.zero_().scatter_(1, indices, x_exp)  # B * THW * HW

    return x

def make_gaussian(y_idx, x_idx, height, width, sigma=7):
    yv, xv = torch.meshgrid([torch.arange(0, height), torch.arange(0, width)])

    yv = yv.reshape(height * width).unsqueeze(0).float().cuda()
    xv = xv.reshape(height * width).unsqueeze(0).float().cuda()

    y_idx = y_idx.transpose(0, 1)
    x_idx = x_idx.transpose(0, 1)

    g = torch.exp(-((yv - y_idx)**2 + (xv - x_idx)**2) / (2 * sigma**2))

    return g

def kmn(x, top=None, gauss=None):
    if top is not None:
        if gauss is not None:
            maxes = torch.max(x, dim=1, keepdim=True)[0]
            x_exp = torch.exp(x - maxes) * gauss
            x_exp, indices = torch.topk(x_exp, k=top, dim=1)
        else:
            values, indices = torch.topk(x, k=top, dim=1)
            x_exp = torch.exp(values - values[:, 0])

        x_exp_sum = torch.sum(x_exp, dim=1, keepdim=True)
        x_exp /= x_exp_sum
        x.zero_().scatter_(1, indices, x_exp)  # B * THW * HW

        output = x
    else:
        maxes = torch.max(x, dim=1, keepdim=True)[0]
        if gauss is not None:
            x_exp = torch.exp(x - maxes) * gauss

        x_exp_sum = torch.sum(x_exp, dim=1, keepdim=True)
        x_exp /= x_exp_sum
        output = x_exp

    return output

class MemoryBank:

    def __init__(self, compress, k, top_k=20, mode='stm'):
        self.top_k = top_k

        self.CK = None
        self.CV = None

        self.mem_k = None
        self.mem_v = None

        self.num_objects = k
        self.km = 5.6

        self.compress = compress

        self.init_mode(mode)

    def init_mode(self, mode):
        """
        stm, two-frames, gt, last, compress, gt-compress,
        last-compress, two-frames-compress
        """
        self.is_compress = None
        self.use_gt = None
        self.use_last = None
        self.stm = None
        print('mode is {}'.format(mode))
        if mode == 'stm':
            self.stm = True
        elif mode == 'two-frames':
            self.use_gt = True
            self.use_last = True
        elif mode == 'gt':
            self.use_gt = True
        elif mode == 'last':
            self.use_last = True
        elif mode == 'compress':
            self.is_compress = True
        elif mode == 'gt-compress':
            self.use_gt = True
            self.is_compress = True
        elif mode == 'last-compress':
            self.use_last = True
            self.is_compress = True
        elif mode == 'two-frames-compress':
            self.use_gt = True
            self.use_last = True
            self.is_compress = True
        else:
            raise RuntimeError('check mode!')

    def _global_matching(self, mk, qk, H, W):
        # NE means number of elements -- typically T*H*W
        mk = mk.flatten(start_dim=2)
        qk = qk.flatten(start_dim=2)
        B, CK, NE = mk.shape

        a = mk.pow(2).sum(1).unsqueeze(2)
        b = 2 * (mk.transpose(1, 2) @ qk)
        # We don't actually need this, will update paper later
        # c = qk.pow(2).expand(B, -1, -1).sum(1).unsqueeze(1)

        affinity = (-a + b) / math.sqrt(CK)  # B, NE, HW
        # if self.km is not None:
        #     # Make a bunch of Gaussian distributions
        #     argmax_idx = affinity.max(2)[1]
        #     y_idx, x_idx = argmax_idx//W, argmax_idx%W
        #     g = make_gaussian(y_idx, x_idx, H, W, sigma=self.km)
        #     g = g.view(B, NE, H*W)

        #     affinity = kmn(affinity, top=20, gauss=g)  # B, THW, HW
        affinity = softmax_w_top(affinity, top=self.top_k)  # B, THW, HW

        return affinity

    def _readout(self, affinity, mv):
        return torch.bmm(mv, affinity)

    def match_memory(self, qk):
        k = self.num_objects
        _, _, h, w = qk.shape

        qk = qk.flatten(start_dim=2)

        # use gt+last+mem
        if self.temp_k is not None and self.is_compress and self.use_last and \
                self.use_gt:
            # print("mode: gt+last+mem")
            mk = torch.cat([self.mem_k, self.temp_k, self.gt_k, self.gt_k], 2)
            # mv = torch.cat([self.mem_v, self.temp_v], 2)
            try:
                mv = torch.cat([self.mem_v, self.temp_v, self.gt_v, self.gt_v],
                               2)
            except Exception:
                mv = torch.cat([
                    self.mem_v,
                    self.temp_v.unsqueeze(0),
                    self.gt_v.unsqueeze(0),
                    self.gt_v.unsqueeze(0)
                ], 3)
        # use gt+last
        elif self.temp_k is not None and self.use_last and self.use_gt:
            # print("mode: gt+last")
            mk = torch.cat([self.temp_k, self.gt_k, self.gt_k], 2)
            # mv = torch.cat([self.mem_v, self.temp_v], 2)
            try:
                mv = torch.cat([self.temp_v, self.gt_v, self.gt_v], 2)
            except Exception:
                mv = torch.cat([
                    self.temp_v.unsqueeze(0),
                    self.gt_v.unsqueeze(0),
                    self.gt_v.unsqueeze(0)
                ], 3)
        # use last+mem
        elif self.temp_k is not None and self.is_compress and self.use_last:
            # print("mode: last+mem")
            mk = torch.cat([self.mem_k, self.temp_k], 2)
            try:
                mv = torch.cat([self.mem_v, self.temp_v], 2)
            except Exception:
                mv = torch.cat([self.mem_v, self.temp_v.unsqueeze(0)], 3)
        # use gt+mem
        elif self.is_compress and self.use_gt:
            # print("mode: gt+mem")
            # mk = self.mem_k
            # mv = self.mem_v
            mk = torch.cat([self.mem_k, self.gt_k], 2)
            try:
                mv = torch.cat([self.mem_v, self.gt_v], 2)
            except Exception:
                mv = torch.cat([self.mem_v, self.gt_v.unsqueeze(0)], 3)
        # use last
        elif self.temp_k is not None and self.use_last:
            # print("mode: last")
            mk = self.temp_k
            mv = self.temp_v
        # use gt or only use our embedding
        else:
            # print("mode: gt")
            # use nothing
            mk = self.mem_k
            mv = self.mem_v

        affinity = self._global_matching(mk, qk, h, w)
        if len(mv.shape) == 6:
            mv = mv.squeeze(0)
        mv = mv.flatten(start_dim=2)

        # One affinity for all
        readout_mem = self._readout(affinity.expand(k, -1, -1), mv)

        return readout_mem.view(k, self.CV, h, w)

    def add_memory(self, key, value, is_temp=False):
        # Temp is for "last frame"
        # Not always used
        # But can always be flushed
        self.temp_k = None
        self.temp_v = None

        if self.mem_k is None:
            # First frame, just shove it in
            self.mem_k = key  # gt
            self.mem_v = value
            self.CK = key.shape[1]
            self.CV = value.shape[1]
            self.gt_k = key
            self.gt_v = value

        elif self.is_compress:
            # compress the two frames
            if len(self.mem_v.shape) == 5:
                self.mem_v = self.mem_v.unsqueeze(0)
            k = torch.cat([self.mem_k, key], 2)  # [1, 64, 2, 30, 57]
            v = torch.cat([self.mem_v, value.unsqueeze(0)],
                          3)  # [1, 2, 512, 2, 30, 57]
            self.mem_k, self.mem_v = self.compress(k, v)

            # check if use last frame
            if self.use_last:
                self.temp_k = key  # [1, 64, 1, 30, 57]
                self.temp_v = value  # [2, 512, 1, 30, 57]

        elif self.stm:
            # stm style
            # print("stm", self.mem_k.shape)
            self.mem_k = torch.cat([self.mem_k, key], 2)
            self.mem_v = torch.cat([self.mem_v, value], 2)

        else:
            # no compress
            # check if use last frame
            if self.use_last:
                self.temp_k = key
                self.temp_v = value


import torch
import torch.nn as nn
from torchsummary import summary

class ConvLayer(nn.Module):
    def __init__(self, in_channels, out_channels, kernel_size, stride, padding=1) -> None:
        super().__init__()
        self.conv = nn.Conv3d(in_channels, out_channels, kernel_size, stride, padding)
        self.bn = nn.BatchNorm3d(out_channels)
        self.activation = nn.ReLU6()

    def forward(self, x):
        x = self.conv(x)
        x = self.bn(x)
        x = self.activation(x)
        return x

class ConvDwLayer(nn.Module):
    def __init__(self, in_channels, out_channels, kernel_size, stride, padding=1) -> None:
        super().__init__()
        self.depthwise_conv = nn.Conv3d(in_channels, in_channels, kernel_size, stride, padding, groups=in_channels)
        self.bn1 = nn.BatchNorm3d(in_channels)
        self.activation1 = nn.ReLU6()
        self.pointwise_conv = nn.Conv3d(in_channels, out_channels, kernel_size=1, stride=1, padding=0)
        self.bn2 = nn.BatchNorm3d(out_channels)
        self.activation2 = nn.ReLU6()

    def forward(self, x):
        x = self.depthwise_conv(x)
        x = self.bn1(x)
        x = self.activation1(x)
        x = self.pointwise_conv(x)
        x = self.bn2(x)
        x = self.activation2(x)
        return x

class MobileNet(nn.Module):
    def __init__(self, in_channels, n_classes) -> None:
        super().__init__()
        self.conv1 = ConvLayer(in_channels, 32, 3, stride=2, padding=1)
        self.convdw1 = ConvDwLayer(32, 32, 3, stride=1, padding=1)
        self.conv2 = ConvLayer(32, 64, 1, stride=1, padding=0)
        self.convdw2 = ConvDwLayer(64, 64, 3, stride=2, padding=1)
        self.conv3 = ConvLayer(64, 128, 1, stride=1, padding=0)
        self.convdw3 = ConvDwLayer(128, 128, 3, stride=1, padding=1)
        self.conv4 = ConvLayer(128, 128, 1, stride=1, padding=0)
        self.convdw4 = ConvDwLayer(128, 128, 3, stride=2, padding=1)
        self.conv5 = ConvLayer(128, 256, 1, stride=1, padding=0)
        self.convdw5 = ConvDwLayer(256, 256, 3, stride=1, padding=1)
        self.conv6 = ConvLayer(256, 256, 1, stride=1, padding=0)
        self.convdw6 = ConvDwLayer(256, 256, 3, stride=2, padding=1)
        self.conv7 = ConvLayer(256, 512, 1, stride=1, padding=0)

        # 5xconvdw 5xconv
        self.convdw7 = ConvDwLayer(512, 512, 3, stride=1, padding=1)
        self.conv8 = ConvLayer(512, 512, 1, stride=1, padding=0)
        self.convdw8 = ConvDwLayer(512, 512, 3, stride=1, padding=1)
        self.conv9 = ConvLayer(512, 512, 1, stride=1, padding=0)
        self.convdw9 = ConvDwLayer(512, 512, 3, stride=1, padding=1)
        self.conv10 = ConvLayer(512, 512, 1, stride=1, padding=0)
        self.convdw10 = ConvDwLayer(512, 512, 3, stride=1, padding=1)
        self.conv11 = ConvLayer(512, 512, 1, stride=1, padding=0)
        self.convdw11 = ConvDwLayer(512, 512, 3, stride=1, padding=1)
        self.conv12 = ConvLayer(512, 512, 1, stride=1, padding=0)

        self.convdw12 = ConvDwLayer(512, 512, 3, stride=2, padding=1)
        self.conv13 = ConvLayer(512, 1024, 1, stride=1, padding=0)
        self.convdw13 = ConvDwLayer(1024, 1024, 3, stride=1, padding=1)
        self.conv14 = ConvLayer(1024, 1024, 1, stride=1, padding=0)

        # classification layers
        self.pooling = nn.AdaptiveAvgPool3d(1)
        self.flatten = nn.Flatten()
        self.fc = nn.Linear(1024, n_classes)

    def forward(self, x):
        x = self.conv1(x)
        x = self.convdw1(x)
        x = self.conv2(x)
        x = self.convdw2(x)
        x = self.conv3(x)
        x = self.convdw3(x)
        x = self.conv4(x)
        x = self.convdw4(x)
        x = self.conv5(x)
        x = self.convdw5(x)
        x = self.conv6(x)
        x = self.convdw6(x)
        x = self.conv7(x)
        x = self.convdw7(x)
        x = self.conv8(x)
        x = self.convdw8(x)
        x = self.conv9(x)
        x = self.convdw9(x)
        x = self.conv10(x)
        x = self.convdw10(x)
        x = self.conv11(x)
        x = self.convdw11(x)
        x = self.conv12(x)
        x = self.convdw12(x)
        x = self.conv13(x)
        x = self.convdw13(x)
        x = self.conv14(x)
        x = self.pooling(x)
        x = self.flatten(x)
        x = self.fc(x)
        return x

def mobile_net(in_channels=1, out_channels=2):
    net = MobileNet(in_channels, out_channels)
    return net

if __name__ == "__main__":
    net = mobile_net(1, 2)
    print(net)
    summary(net.cuda(), (1, 100, 272, 256), batch_size=1)


from __future__ import unicode_literals

import warnings

__all__ = ["MemberCompat", "SocietyCompat", "AdminsSetCompat"]

class MemberCompat(object):
    @property
    def firstname(self):
        """
        Deprecated

        .. seealso:: :attr:`preferred_name`
        """
        warnings.warn("firstname is deprecated (use preferred_name)",
                      DeprecationWarning)
        return self.preferred_name

    @property
    def initials(self):
        """Deprecated: guesses the member's initials"""
        warnings.warn("initials is deprecated (no longer stored)",
                      DeprecationWarning)
        return self.preferred_name[0].upper() + "."

    @property
    def status(self):
        """
        Deprecated: old memberlist's "status" column

        * "member" if this Member is a member, but not a user
        * "user" if this Member is a member and a user
        * "terminated" if this Member is not a member
          (and therefore cannot be a user)

        .. seealso:: :attr:`member`, :attr:`user`
        """
        warnings.warn("status is deprecated (use member and user attributes)",
                      DeprecationWarning)
        if self.member:
            if self.user:
                return "user"
            else:
                return "member"
        else:
            return "terminated"

    @property
    def joindate(self):
        """
        Deprecated: `joined` as a YYYY/MM string

        .. seealso:: :attr:`joined`
        """
        warnings.warn("joindate is deprecated (use joined)",
                      DeprecationWarning)
        return self.joined.strftime("%Y/%m")

    def socs(self, socs=None):
        warnings.warn("socs is deprecated (use societies)",
                      DeprecationWarning)
        if socs is not None:
            warnings.warn("the socs argument is ignored")
        return self.societies

class SocietyCompat(object):
    @property
    def name(self):
        warnings.warn("name is deprecated (use society)",
                      DeprecationWarning)
        return self.society

    @property
    def joindate(self):
        """
        Deprecated: `joined` as a YYYY/MM string

        .. seealso:: :attr:`joined`
        """
        warnings.warn("joindate is deprecated (use joined)",
                      DeprecationWarning)
        return self.joined.strftime("%Y/%m")

# in the old API, admins was a method
class AdminsSetCompat(set):
    def __call__(self, memberdict=None):
        if memberdict is not None:
            warnings.warn("the memberdict argument is ignored")
        warnings.warn("admins is now an attribute", DeprecationWarning)
        return self


import pickle
import streamlit as st

pickle_in = open('modelmodel.pkl', 'rb')
clf = pickle.load(pickle_in)

#@st.cache()
@st.cache_data()
def make_prediction(bhk, Bathroom, Size, City):
    #city_enc = -1
    if City == 'Mumbai':
        city_enc = 5
    elif City == 'Chennai':
        city_enc = 1
    elif City == 'Bangalore':
        city_enc = 0
    elif City == 'Hyderabad':
        city_enc = 3
    elif City == 'Delhi':
        city_enc = 2
    elif City == 'Kolkata':
        city_enc = 4

    prediction = clf.predict([[bhk, Bathroom, Size, city_enc]])[0]
    return prediction

#print(make_prediction('5', '6', '90000', 'Bangalore'))

def main():
    #front end elements
    html_temp = """

 House Prices Prediction by Ilesanmi

    """
    #front end
    #st.markdown("![](https://thumbs.dreamstime.com/z/house-rent-down-town-35897819.jpg)")
    #using image instead
    from PIL import Image

    img = Image.open("house-rent.jpg")
    resized_img = img.resize((800, 300))
    st.image(resized_img, caption="House for rent in India")

    st.markdown(html_temp, unsafe_allow_html=True)
    #following lines create the visuals
    City = st.selectbox('City', ('Mumbai', 'Chennai', 'Bangalore', 'Hyderabad', 'Delhi', 'Kolkata'))
    Bathroom = st.number_input('Enter number of Bathroom', step = 1)
    Size = st.number_input('Enter Size of the Land', step = 1)
    bhk = st.number_input('Enter number of BHK', step= 1)

    #when 'predict' is clicked, make prediction
    if st.button('Make Prediction'):
        result = make_prediction(bhk, Bathroom, Size, City)
        st.success('The price is most likely: {}'.format(result))

if __name__ == '__main__':
    main()


def main(numbers:list) -> int:
    print(f'Find number of inversions in {numbers}')
    if len(numbers) < 2:
        return 0
    return inversions(numbers)[1]

def inversions(numbers:list) -> tuple:
    if len(numbers) <=1:
        return numbers, 0
    middle_point = len(numbers) // 2
    left, left_inv = inversions(numbers[:middle_point])
    right, right_inv = inversions(numbers[middle_point:])

    result = []
    l = 0
    r = 0
    inverses = 0 + left_inv + right_inv

    while l < len(left) and r < len(right):
        if left[l] <= right[r]:
            result.append(left[l])
            l += 1
        else:
            result.append(right[r])
            inverses += (len(left) - l)
            r += 1

    result += left[l:]
    result += right[r:]

    return result, inverses

print('Answer is: ', main([5,1,2,3,5,12]))
print('Answer is: ', main([9,1,-4,0,0]))


from django.contrib import admin
from django.urls import path, include
from newsapp.views import HomeView, UserRegistrationView, UserLoginView, UserDashboard, UserLogout

urlpatterns = [
    path('', HomeView.as_view(), name='home'),
    path('signup/', UserRegistrationView.as_view(), name='signup'),
    path('login/', UserLoginView.as_view(), name='login'),
    path('user-dashboard/', UserDashboard.as_view(), name='dashboard'),
    path('logout/', UserLogout.as_view(), name='logout')

]


import os
from datetime import datetime
from dateutil import parser as dateparser
import boto.sdb
import boto.s3

from s3pac.model import LongProperty, StringProperty, DateTimeProperty
from s3pac.package import Package, read_package_file

# -----------------------------------------------------------------------------

_FROM_SIMPLEDB = {
    LongProperty: lambda v: int(v) - 2**63,
    DateTimeProperty: dateparser.parse,
    }

_TO_SIMPLEDB = {
    LongProperty: lambda v: "%020d" % (v + 2**63),
    DateTimeProperty: datetime.isoformat,
    }

def _pkg_from_sdb(_dict):
    return Package.load(_FROM_SIMPLEDB, _dict)

def _sdb_from_pkg(pkg):
    return Package.store(_TO_SIMPLEDB, pkg)

# -----------------------------------------------------------------------------

class PackageDatabase:
    """Package repository interface to SimpleDB and S3."""
    def __init__(self, access_key_id, secret_access_key, region_name,
                 sdb_domain_name, s3_bucket_name, s3_prefix):
        # connect to simpledb
        self.sdb = boto.sdb.connect_to_region(region_name,
            aws_access_key_id=access_key_id,
            aws_secret_access_key=secret_access_key)
        self.sdb_domain = self.sdb.get_domain(sdb_domain_name)
        self.sdb_domain_name = sdb_domain_name

        # connect to s3
        self.s3 = boto.s3.connect_to_region(region_name,
            aws_access_key_id=access_key_id,
            aws_secret_access_key=secret_access_key)
        self.s3_bucket = self.s3.get_bucket(s3_bucket_name)
        self.s3_bucket_name = s3_bucket_name
        self.s3_prefix = s3_prefix

    def _pkgitemname(self, pkg):
        return os.path.join(pkg.repo, pkg.arch, pkg.name)

    def _pkgkeyname(self, pkg):
        return os.path.join(self.s3_prefix, pkg.repo, pkg.filename)

    def publish(self, repo, pkgfile, sigfile):
        """Read package archive from `pkgfile` and publish to `repo`."""
        pkg = read_package_file(pkgfile, sigfile)

        pkg.repo = repo
        pkg.publishdate = datetime.utcnow()

        # upload package file to s3
        pkgkey = boto.s3.key.Key(self.s3_bucket, self._pkgkeyname(pkg))
        pkgfile.seek(0)
        pkgkey.set_contents_from_file(pkgfile)

        # insert metadata
        self.sdb_domain.put_attributes(self._pkgitemname(pkg),
                                       _sdb_from_pkg(pkg))

        # remove previous versions if they exist
        for ppkg in self.find(repo=pkg.repo, arch=pkg.arch, name=pkg.name):
            if ppkg.version != pkg.version:
                self.delete(repo=repo, arch=ppkg.arch, name=ppkg.name,
                            version=ppkg.version)

        return pkg

    def find(self, **kwargs):
        """Find matching packages."""
        parts = []
        conds = Package.convertdict(_TO_SIMPLEDB, kwargs)
        for name, values in conds.items():
            if not isinstance(values, list):
                values = [values]
            for value in values:
                if value.startswith('%') or value.endswith('%'):
                    escaped = value[1:-1].replace('%', '\%')
                    parts.append('`%s` LIKE "%s%s%s"' % \
                        (name, value[0], escaped, value[-1]))
                else:
                    parts.append('`%s`="%s"' % (name, value))
        query = "SELECT * FROM `%s` WHERE %s" % \
            (self.sdb_domain_name, " AND ".join(parts))
        results = self.sdb_domain.select(query, consistent_read=True)
        return list(map(_pkg_from_sdb, results))

    def findone(self, **kwargs):
        """Find first matching package."""
        pkgs = self.find(**kwargs)
        return pkgs[0] if pkgs else None

    def url(self, pkg):
        """Return a HTTP download URL for `pkg`."""
        pkgkey = self.s3_bucket.get_key(self._pkgkeyname(pkg))
        return pkgkey.generate_url(3600)

    def delete(self, **kwargs):
        """Delete all matching packages."""
        pkgs = self.find(**kwargs)
        for pkg in pkgs:
            pkgkey = self.s3_bucket.get_key(self._pkgkeyname(pkg))
            if pkgkey is not None:
                pkgkey.delete()
            self.sdb_domain.delete_attributes(self._pkgitemname(pkg))
        return pkgs


"""
def timer(func):
    def inner():
        print('time started')
        # print(func)
        func()
        print('time ended')

    return inner
# timer()()

#Easier way to decorated
@timer
def get_factorial():
    print('Factorial Starting')

get_factorial()

#vajailla way to decorated
# timer(get_factorial)()

 """
import math
import time
def timer(func):
    def inner(*args,**kwargs):
        print('time started')
        start = time.time()
        func(*args,**kwargs)  # chaile key oala args o dite pari  --------(**kwargs)
        print('time ended')
        end = time.time()
        print(f'total time taken {end - start}')
    return inner
#Easier way to decorated
@timer
def get_factorial(n): # jodi inner function a kice dite chai
    print('Factorial Starting')
    result = math.factorial(n)
    print(f'factorial of {n} is : {result}')

get_factorial(n=12)


""" Given a directory w/ hand
    currated CD genes, convert
    the CD files there to
    annotations in the prosite
    form.
"""
import utils_scripting, sys, os, utils_humanVirus

req_args = ['mylist directory',
            'translation file',
            'output file']
examples = ['../../Data/ProfileScan/MyLists/',
            'some translation file',
            'some out file']
utils_scripting.checkStart(sys.argv, req_args, examples, len(req_args), True)

mylist_dir = sys.argv[1]
trans = utils_humanVirus.get_entrez2version(sys.argv[2])
outfile = sys.argv[3]

with open(outfile, 'w') as outf:
    for file in os.listdir(mylist_dir):
        with open(mylist_dir + file) as f:
            for line in f:
                entrez = line.strip()
                if entrez in trans:
                    for version in trans[entrez]:
                        outf.write(version + '\t0\t0\t'
                                   + file + '\tseq\tProfileScan\n')


import os
import json
import time
import string
import math
import random
from numpy import matrix
from keras.layers import Input, Dense
from keras.models import Model
from keras.optimizers import SGD,Adadelta
from keras.models import model_from_json
from keras.layers.normalization import BatchNormalization
from keras.layers.core import Dropout
def nextt(strg):
    sp=strg.split("-")
    if sp[3]=="144":
        sp[2]=str(int(sp[2])+1)
        sp[3]=1
        if len(sp[2])==1:
            sp[2]="0"+sp[2]
    else:
        sp[3]=str(int(sp[3])+1)
    return "%s-%s-%s-%s"%(sp[0],sp[1],sp[2],sp[3])
def steep(val):
    global ovt
    thred=2.5
    f=int(math.floor(val))
    bound=2.0*f*(f+1)/(2*f+1)
    if val<(bound-f)/2+f:
        val=f*1.0
    return val
def predict():
    path=""
    base=""
    name="neupr-0.265"
    path="net-layer5-bias0.001500-ml10-best"
    po=1.0
    mxyp=0.00
    beta=0.8
    baset=0.0015
    sgd = Adadelta()
    for i in xrange(len(X)):
        yp=Y[i]
        if yp>mxyp:
            mxyp=yp
    if os.path.exists("%s.w.xml"%path):
        model=model_from_json(open("%s.xml"%path,"rb").read())
        model.load_weights("%s.w.xml"%path)
    #model.compile(loss='mape', optimizer=sgd)#mean_squared_error
    global tr
    fo=open(base+"%s.csv"%(name),"wb")
    for i in xrange(len(Xt)):
        if Mt[i][0] in Sam_t:
            yp=(model.predict(matrix([Xt[i]])).tolist())[0][0]
            #print yp
            if yp<=baset+1*beta/mxyp:
               yp=1.0/mxyp*beta
            else:
               yp=yp-baset
            #yp=1.0
            yp=yp/beta*mxyp
            yp=steep(yp)
            fo.write("%s,%s,%f\n"%(int(eval(Mt[i][1])),nextt(Mt[i][0]),yp))
    fo.close()
predict()


#!/usr/bin/python
# -*- coding: UTF-8 -*-

''' Installation script for tractconverter package '''
# Inspired by setup.py of dipy (http://nipy.sourceforge.net/dipy/)

import os
import sys
from os.path import join as pjoin, dirname
from glob import glob

import distribute_setup
distribute_setup.use_setuptools()

from setuptools import setup, find_packages

# BEFORE importing distutils, remove MANIFEST. distutils doesn't properly
# update it when the contents of directories change.
if os.path.exists('MANIFEST'):
    os.remove('MANIFEST')

import numpy as np

# Get version and release info, which is all stored in tractconverter/info.py
ver_file = pjoin('tractconverter', 'info.py')
execfile(ver_file)

# force_setuptools can be set from the setup_egg.py script
if not 'force_setuptools' in globals():
    # For some commands, use setuptools
    if len(set(('develop', 'bdist_egg', 'bdist_rpm', 'bdist', 'bdist_dumb',
                'bdist_wininst', 'install_egg_info', 'egg_info',
                'easy_install')).intersection(sys.argv)) > 0:
        force_setuptools = True
    else:
        force_setuptools = False

if force_setuptools:
    # Try to preempt setuptools monkeypatching of Extension handling when Pyrex
    # is missing.  Otherwise the monkeypatched Extension will change .pyx
    # filenames to .c filenames, and we probably don't have the .c files.
    sys.path.insert(0, pjoin(dirname(__file__), 'fake_pyrex'))
    import setuptools

# We may just have imported setuptools, or we may have been exec'd from a
# setuptools environment like pip
if 'setuptools' in sys.modules:
    extra_setuptools_args = dict(
        tests_require=['nose'],
        test_suite='nose.collector',
        zip_safe=False,
        extras_require=dict(
            doc=['Sphinx>=1.0'],
            test=['nose>=0.10.1']),
        install_requires=['nibabel>=' + NIBABEL_MIN_VERSION])

    # We need setuptools install command because we're going to override it
    # further down.  Using distutils install command causes some confusion, due
    # to the Pyrex / setuptools hack above (force_setuptools)
    from setuptools.command import install
else:
    extra_setuptools_args = {}
    from distutils.command import install

# Import distutils _after_ potential setuptools import above, and after removing
# MANIFEST
from distutils.core import setup
from distutils.extension import Extension

# Define extensions
EXTS = []

# Do our own build and install time dependency checking. setup.py gets called in
# many different ways, and may be called just to collect information (egg_info).
# We need to set up tripwires to raise errors when actually doing things, like
# building, rather than unconditionally in the setup.py import or exec
# We may make tripwire versions of build_ext, build_py, install
try:
    from nisext.sexts import package_check, get_comrec_build
except ImportError:  # No nibabel
    msg = ('Need nisext package from nibabel installation'
           ' - please install nibabel first')
    # pybuilder = derror_maker(build_py.build_py, msg)
    # extbuilder = derror_maker(build_ext.build_ext, msg)
# else: # We have nibabel
    # pybuilder = get_comrec_build('tractconverter')
    # Cython is a dependency for building extensions, iff we don't have stamped
    # up pyx and c files.
    # extbuilder = cyproc_exts(EXTS, CYTHON_MIN_VERSION, 'pyx-stamps')

# Installer that checks for install-time dependencies

class installer(install.install):

    def run(self):
        package_check('numpy', NUMPY_MIN_VERSION)
        package_check('nibabel', NIBABEL_MIN_VERSION)
        install.install.run(self)

cmdclass = dict(
    install=installer)

def main(**extra_args):
    setup(name=NAME,
          maintainer=MAINTAINER,
          maintainer_email=MAINTAINER_EMAIL,
          description=DESCRIPTION,
          long_description=LONG_DESCRIPTION,
          url=URL,
          download_url=DOWNLOAD_URL,
          license=LICENSE,
          classifiers=CLASSIFIERS,
          author=AUTHOR,
          author_email=AUTHOR_EMAIL,
          platforms=PLATFORMS,
          version=VERSION,
          requires=REQUIRES,
          provides=PROVIDES,
          packages=['tractconverter',
                    'tractconverter.formats'
                    ],
          ext_modules=EXTS,
          package_data={'tractconverter':
                        [pjoin('data', '*')
                         ]},
          data_files=[('share/doc/tractconverter/examples',
                       glob(pjoin('doc', 'examples', '*.py')))],
          scripts=glob(pjoin('scripts', '*')),
          cmdclass=cmdclass,
          **extra_args
          )

# simple way to test what setup will do
# python setup.py install --prefix=/tmp
if __name__ == "__main__":
    main(**extra_setuptools_args)


# -*- coding: utf-8 -*-
"""
Created on Fri Oct 22 15:33:59 2021

@author: sahru
"""
import numpy as np
import pandas as pd
images = pd.read_csv(r"C:\Users\sahru\Downloads\mnist_train_binary.csv")

images_test = pd.read_csv(r"C:\Users\sahru\Downloads\mnist_test_binary.csv")

X_test_preprocessed = images_test.iloc[:,1:]
X_test_preprocessed = X_test_preprocessed/255

Y_test = images_test['label']
Y = images['label']

#12163,1

X_train_preprocessed = images.iloc[:,1:]
X_train_preprocessed = X_train_preprocessed/255

X_train_preprocessed=np.reshape(np.ravel(X_train_preprocessed),(X_train_preprocessed.shape[0],28,28))

X_test_preprocessed=np.reshape(np.ravel(X_test_preprocessed),(X_test_preprocessed.shape[0],28,28))

Intensity = []
Symmetry = []

O = np.ones(12163)

for i in range(0,len(X_train_preprocessed)):
    Intensity.append(X_train_preprocessed[i].mean())
    Symmetry.append(np.mean(abs(X_train_preprocessed[i] - np.flip(X_train_preprocessed[i]))))

Intensity = np.array(Intensity)
Symmetry = np.array(Symmetry)
X_train_processed = np.column_stack((O,Intensity,Symmetry))

# adding bias to the input
np.random.seed(42)
bias = np.random.uniform(low=0.0,high=1.0)
X_train_processed[1:,:] = X_train_processed[1:,:]

X_transformed = np.column_stack((O,Intensity,Symmetry,Intensity**2,Intensity * Symmetry,Symmetry**2,(Intensity)**3,(Intensity)**2 * Symmetry,Intensity * (Symmetry)**2,(Symmetry)**3,))
Intensity_test = []
Symmetry_test = []

for i in range(0,len(X_test_preprocessed)):
    Intensity_test.append(X_test_preprocessed[i].mean())
    Symmetry_test.append(np.mean(abs(X_test_preprocessed[i] - np.flip(X_test_preprocessed[i]))))
Intensity_test = np.array(Intensity_test)
Symmetry_test = np.array(Symmetry_test)
O = np.ones(2027)
X_test_processed = np.column_stack((O,Intensity_test,Symmetry_test))
X_transformed_test = np.column_stack((O,Intensity_test,Symmetry_test,Intensity_test**2,Intensity_test * Symmetry_test,Symmetry_test**2,(Intensity_test)**3,(Intensity_test)**2 * Symmetry_test,Intensity_test * (Symmetry_test)**2,(Symmetry_test)**3,))

X_dagger = np.dot(np.linalg.inv(np.dot(X_transformed.T, X_transformed)), X_transformed.T)
weights = np.dot(X_dagger, Y)
#print(self.X_dagger.shape)

R,C = X_transformed.shape

for i in range(0,len(Y)):
    if(Y[i] == 5):
        Y[i] = -1
for epoch in range(0,2):
    i = np.random.randint(low=0,high=R)
    s = np.array(np.dot(weights.T,np.array(X_transformed[i])))

    print(s)
    if((Y[i] * s) <= 1) :
        weights = weights + 0.1*(((Y[i])-(s))* X_transformed[i])

X_dagger1 = np.dot(np.linalg.inv(np.dot(X_transformed.T, X_transformed)), X_transformed.T)
Y_pred = np.sign(np.dot(weights.T, X_dagger1)).T

m,n = X_transformed.shape
for i in range(0,len(Y)):
    if(Y[i] == 5):
        Y[i] = 1
    else:
        Y[i] = -1

E_in = sum(Y_pred != Y)  / m
print("third order polynomial training error",E_in)

X_dagger1 = np.dot(np.linalg.inv(np.dot(X_transformed_test.T, X_transformed_test)), X_transformed_test.T)
Y_pred = np.sign(np.dot(weights.T, X_dagger1)).T
m,n = X_transformed.shape
for i in range(0,len(Y)):
    if(Y[i] == 5):
        Y[i] = 1
    else:
        Y[i] = -1

E_in = sum(Y_pred != Y_test)  / m
print("third order polynomial test error",E_in)

print(X_transformed_test.shape)


import cv2
import numpy as np
from matplotlib import pyplot as plt

img = cv2.cvtColor(cv2.imread('sponge.jpg'), cv2.COLOR_BGR2RGB) #cargar imagen original y conversion de espacio

reduccion = 50 #porcentaje reduccion
redu_size = float(reduccion/100) #conversion de unidades
img2= cv2.resize(img, (0, 0), fx=0.5, fy=0.5) #reducir imagen y asignar a nueva
superior = inferior = (img.shape[0] - img2.shape[0]) // 2 #calcular bordes del eje vertical
izquierdo = derecho = (img.shape[1] - img2.shape[1]) // 2 #calcular bordes del eje horizontal
img2bord = cv2.copyMakeBorder(img2, superior, inferior, izquierdo, derecho, cv2.BORDER_CONSTANT, value=[255, 255, 255]) #aplicar bordes para visualizar

img3 = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY) #imagen blanco y negro

img4 = cv2.flip(img, 1) #imagen invertida por eje vertical

edge_kernel = np.array([[-1,-1,-1],[-1,8,-1],[-1,-1,-1]]) #crear edge_kernel
img5 = cv2.filter2D(img2, -1, edge_kernel)#aplicacion de bordes
img5bord = cv2.copyMakeBorder(img5, superior, inferior, izquierdo, derecho, cv2.BORDER_CONSTANT, value=[255, 255, 255])

# Creacion del plot
fig, axs = plt.subplots(nrows = 4, ncols = 2, figsize = (8, 8))

axs[0][0].imshow(img)
axs[0][0].axis("off")
axs[0][0].set_title('Imagen Original')

axs[0][1].imshow(img2bord)
axs[0][1].axis("off")
axs[0][1].set_title('Imagen reducida 50%')

axs[1][0].imshow(img)
axs[1][0].axis("off")

axs[1][1].imshow(img3, cmap="gray")
axs[1][1].axis("off")
axs[1][1].set_title('Imagen Escala Grises')

axs[2][0].imshow(img)
axs[2][0].axis("off")

axs[2][1].imshow(img4)
axs[2][1].axis("off")
axs[2][1].set_title('Imagen invertida eje vertical')

axs[3][0].imshow(img)
axs[3][0].axis("off")

axs[3][1].imshow(img5bord)
axs[3][1].axis("off")
axs[3][1].set_title('Imagen reducida con deteccion de bordes')

plt.show()


# Tic Tac Toe

from tkinter import *
import random

def next_turn(row, col):
    global player

    if grid[row][col]["text"] == "" and winner_check() is False:
        if player == symbol[0]:
            grid[row][col]["text"] = player
            if winner_check() is False:
                player = symbol[1]
                label.config(text= (symbol[1]+" turn~"))
            elif winner_check() is True:
                label.config(text=(symbol[0]+" is the winner!"))
            elif winner_check() == "Tie":
                label.config(text="Tie")
        else:
            grid[row][col]["text"] = player
            if winner_check() is False:
                player = symbol[0]
                label.config(text= (symbol[0]+" turn~"))
            elif winner_check() is True:
                label.config(text=(symbol[1]+" is the winner!"))
            elif winner_check() == "Tie":
                label.config(text="Tie")

def winner_check():

    for row in range(3):
        if grid[row][0]["text"]==grid[row][1]["text"]==grid[row][2]["text"]!="":
            for i in range(3):
                grid[row][i].config(bg="green")
            return True

    for col in range(3):
        if grid[0][col]["text"]==grid[1][col]["text"]==grid[2][col]["text"]!="":
            for i in range(3):
                grid[i][col].config(bg="green")
            return True

    if grid[0][0]["text"]==grid[1][1]["text"]==grid[2][2]["text"]!="":
        for i in range(3):
                grid[i][i].config(bg="green")
        return True

    elif grid[0][2]["text"]==grid[1][1]["text"]==grid[2][0]["text"]!="":
        for i in range(3):
                grid[2-i][i].config(bg="green")
        return True

    elif empty_space() is False:
        for row in range(3):
            for col in range(3):
                grid[row][col].config(bg="orange")
        return "Tie"

    else:
        return False

def empty_space():

    spaces = 9
    for row in range(3):
        for col in range(3):
            if grid[row][col]["text"]!="":
                spaces -= 1
    if spaces==0:
        return False
    else:
        return True

def new_game():
    global player

    player = random.choice(symbol)
    label.config(text = player + " turn~")

    for row in range(3):
        for col in range(3):
            grid[row][col].config(text="", bg="#F0F0F0")

window = Tk()
window.title("Tic Tac Toe")
symbol = ["x", "o"]
player = random.choice(symbol)

grid = [[0, 0, 0], [0, 0, 0], [0, 0, 0]]

label = Label(text=player+" turn~", font=("consolas", 40))
label.pack(side="top")

restart_button = Button(text="Restart", font=("consolas", 20), command=new_game)
restart_button.pack(side="bottom")

frame = Frame(window)
frame.pack()

for row in range(3):
    for col in range(3):
        grid[row][col] = Button(frame, font=("consolas", 40), width=5, height=2,
                                command=lambda row=row, col=col: next_turn(row, col))
        grid[row][col].grid(row=row, column=col)

window.mainloop()


import sys
sys.path.append(r"extractor")

import extractor.SchemaExtractorTestV3 as se
from automaticExtraction import automaticExtraction

from pprint import pprint
from extractor import util
from operator import itemgetter
from SPARQLWrapper import SPARQLWrapper, XML
from extractor.util import mongo, queryGenerator
from xml.dom.minidom import parseString

def downloadPortal(argv):
    print(argv)
    sparql = SPARQLWrapper(argv)
    q = util.queryGenerator.QueryGenerator()
    if argv == "https://www.europeandataportal.eu/sparql":
        sparql.setQuery(q.EuDownload().query)
    elif argv == "https://io.datascience-paris-saclay.fr/sparql":
        sparql.setQuery(q.dataScienceParisDownload().query)
    elif argv == "http://data.europa.eu/euodp/sparqlep":
        sparql.setQuery(q.dataEuDownload().query)
    else:
        sparql.setQuery(q.dataEuDownload().query)

    sparql.setReturnFormat(XML)

    results = sparql.queryAndConvert()
    parsed = se.parseResponseForDatasetExtr(None, results, "test_connection", False)

    #è brutto lasciare un if aperto fino alla fine della funzione
    #aggiungere un controllo per uscire quando parsed è vuoto. c'è da capire cosa ritorna parsed e come capire quando si può considerare vuoto
    if not parsed:
        print(f"Nothing found at {argv}. Is the url correct?")
        return

    endDIct = {}

    cont = 0
    for end in parsed:
        end["url"] = end["url"].split("?")[0]
        if 'title' in end:
            if end['url'] in endDIct:
                tmp = endDIct[end['url']]
                tmp['name'].append(end['title'])
                endDIct[end['url']] = tmp
            else:
                endDIct[end['url']] = {'name':[end['title']]}
        else:
            endDIct[end['url']] = {'name':[end['url']]}
        cont += 1

    datasets = []
    urls = []

    # beware that count contains the last id found in mongodb + 1
    count = mongo.getLastIdEndpointsLodex()
    copy = False

    for key in endDIct:
        endpoint = mongo.getAllEndopoinLodex()
        for e in endpoint:
            if e["url"] == key:
                copy = True
                break

        if not copy:
           ds = {}
           ds = {'url': key, '_id': count, 'name': endDIct[key]['name'][0]}
           urls.append(key)
           count = count+1
           #  ds['datasets'] = [{'name':endDIct[key]['name'][0]}]    --  OLD and I think not correct -- Federico
           ds['datasets'] = endDIct[key]['name']
           datasets.append(ds)

           copy = False

    # Stringa per il parsing
    print(f"La ricerca di nuovi dataset sul portale {argv} ha trovato {cont} risultati")
    print(f"Sono stati trovati {str(len(datasets))} nuovi datasets")
    print(datasets)
    print(urls)

    if len(datasets) > 0:
        mongo.inserLodexDatasets(datasets)
        for i in range(0, len(datasets)):
            url = urls[i]
            automaticExtraction([url])

def downloadDataset(url):
    print(url)
    sparql = SPARQLWrapper(url)
    q = util.queryGenerator.QueryGenerator()
    id = mongo.startTestNew(url)
    print(id)

    # in runInfo, id <C3><A8> il numero dentro a ObjectId
    copy = False
    count = mongo.getLastIdEndpointsLodex()
    datasets = []
    name = ""

    if se.testConnection(url, q, sparql, id):
        endpoint = mongo.getAllEndopoinLodex()
        for e in endpoint:
            if e["url"] == url:
                name = e["name"]
                copy = True
                break

        if copy == False:
            ds = {}
            ds = {'url': url, '_id': count, 'name': url}
            datasets.append(ds)
        else:
            print("-----")
            print(url + " is a valid endpoint but it is already present on our server with the following name: " + name)
            print("The extraction has not been performed.")

    else:
        print("-----")
        print(url + " it is not a valid endpoint or it is not reachable at the moment. Retry later.")
        print("Extraction failed.")

    if len(datasets) > 0:
        mongo.inserLodexDatasets(datasets)
        #penso che questo comando sia inutile, lo commento -- Federico
        #mongo.deleteExtById(count)
        print(datasets)
        automaticExtraction(url)
        print("-----")
        print(url + " is a valid endpoint and it is not present on our server.")
        print("Extraction ended correctly.")

def main(argv):
    for portal in ["https://trafair.eu/sparql", "https://www.europeandataportal.eu/sparql", "https://io.datascience-paris-saclay.fr/sparql", "http://data.europa.eu/euodp/sparqlep"]:
        print(portal)
        downloadPortal(portal)

if __name__ == "__main__":
    main(sys.argv[1:])


__author__ = 'Daoyuan'
from BaseSolution import *
class DecodeWays(BaseSolution):
    '''
    Be careful with the boundary condition
    The key for this question is to write down the
    state transition function.
    '''
    def __init__(self):
        BaseSolution.__init__(self)
        self.push_test(
            params = ("11012209",),
            expects = 2
        )
        self.push_test(
            params = ("1232212322",),
            expects = 48
        )
    def solution(self, s):
        if not s or s == "": return 0
        n = len(s)
        c = [0] * (n+1)
        if int(s[0]) == 0:
            return 0
        else:
            c[0] = 1

        if n == 1: return c[0]

        if int(s[1]) == 0:
            if int(s[0:2]) > 26:
                return 0
            else:
                c[1] = 1
        else:
            if int(s[0:2]) > 26:
                c[1] = 1
            else:
                c[1] = 2

        for j in range(2, n):
            if int(s[j]) != 0 and int(s[j-1]) != 0:
                if int(s[j-1:j+1]) > 26:
                    c[j] = c[j-1]
                else:
                    c[j] = c[j-1] + c[j-2]
            elif int(s[j]) == 0 and int(s[j-1]) != 0:
                 if int(s[j-1:j+1]) <= 26:
                     c[j] = c[j-2]
                 else:
                    return 0
            elif int(s[j]) == 0 and int(s[j-1]) == 0:
                return 0
            else:
                c[j] = c[j-1]

        return c[n-1]