# RCLVatlas Pipeline
# 1. Identify RCLVS using floater package
#      - Checks multiple CDs (0.01,0.02,0.03)
#      - Sets dispersal threshold with CI (>=-0.5)
#      - 85% of particles must have the same sign of the vorticity
# 2. Track the RCLVs through time and ID them
# 3. QC: Check if any RCLVs skipped a date (or 2)
#      - Interpolate over the missing timestep using particle trajectories
# 4. Interpolate eddy bounds for first 3 timesteps
#      - Tracks the RCLV birth
# 5. QC: Address instances of overlapping contours
# 6. Give RCLVs an age

# Lexi Jones
# Last Edited: 12/21/22

import os,itertools,sys
import xarray as xr
import numpy as np
from floater import rclv
from skimage.feature import peak_local_max
from shapely.geometry import Point
from shapely.geometry.polygon import Polygon
from subfunctions_for_RCLV_atlas import *

sys.path.append('../')
from config import *

####################################### 1. Identify RCLVS #######################################

def LAVD_to_RCLV(initial_date):
    """
    Extract the RCLVs from the LAVD using the floater package (https://github.com/ocean-transport/floater).
    To qualify, a feature must have a convex deficiency <= 0.03, area >= min_area, coherency index >= -0.5, and
    85% of the particles need to have the same sign of the vorticity on day 1 and the last day of the sim.

    Input
        initial_date: Lagrangian particle initialization date; determines which LAVD file to open
    Output
        plms: Indices of the peak local max of the LAVD, i.e. center of an RCLV
        cons: Boundaries (in grid index format) for the RCLV contour
        areas: Areas (km^2) of the RCLVs
        cds: Convex deficiencies of the RCLVs
        pols: Polarity (cyclonic or anticyclonic) of the vortex

    """
    # Load data
    traj = xr.open_dataset('%s%s_%s.nc'%(lag_traj_dir,initial_date,filename_str)) #Load Lagrangian trajectories
    LAVD = np.load('%s%s_LAVD_%s.npy'%(LAVD_dir,initial_date,filename_str)) #Load LAVD data
    LAVD = np.ma.masked_where(np.isnan(LAVD),LAVD) #Land mask required for the peak_local_max function to work
    LAVD_reshape = np.transpose(np.reshape(LAVD,(len(traj_lon_array),len(traj_lat_array))))

    # Iterate through all of the local maxima to find structures that meet our RCLV criteria
    plms,cons,areas,cds,pols = [],[],[],[],[] # arrays to store the output
    plm = peak_local_max(LAVD_reshape,min_distance=RCLV_params['min_dist']) # grid indices of the local maxima in the LAVD field
    for ji in plm:
        args = {'lon0':traj_lon_array[ji[1]],
                'lat0':traj_lat_array[ji[0]],
                'dlon':np.abs(traj_lon_array[1]-traj_lon_array[0]),
                'dlat':np.abs(traj_lat_array[1]-traj_lat_array[0])}

        for target_CD in [0.03,0.02,0.01]: #target CDs to test, will iterate through each of these unless if a successful contour is identified
            con,area,cd = rclv.convex_contour_around_maximum(LAVD_reshape,ji,
                                                             init_contour_step_frac=RCLV_params['init_contour_step_frac'],
                                                             convex_def_tol=RCLV_params['convex_def_tol'],
                                                             convex_def=target_CD)

            try: # Code errors if the local maximum is too close to the land where there is missing values
                if (area >= RCLV_params['min_area']) and (cd <= 0.03): # check area and convex deficiency meet thresholds; actual cd identified will be different from target CD
                    lon_inds = [round(i) for i in con[:,1]]
                    lat_inds = [round(i) for i in con[:,0]]

                    # Reformat points of the polygon with parentheses
                    poly_pts = [(traj_lon_array[lon_inds][pt],traj_lat_array[lat_inds][pt]) for pt in np.arange(0,len(lon_inds))]

                    # Get the CI of the particles after simulation run time to make sure it meets threshold set
                    x_mask,y_mask = find_polygon_pts(poly_pts,traj_lon_array,traj_lat_array)
                    eddy_day0_lons,eddy_day0_lats,_ = extract_particles_after_time(traj,x_mask,y_mask,traj_lat_array,sim_params,0) #day 0 data
                    eddy_dayx_lons,eddy_dayx_lats,eddy_dayx_vorts = extract_particles_after_time(traj,x_mask,y_mask,traj_lat_array,sim_params,sim_params['runtime']) # last day
                    contour_CI = CI(eddy_day0_lons,eddy_day0_lats,eddy_dayx_lons,eddy_dayx_lats) #calculate the coherency index (measure of dispersal)

                    if (contour_CI >= -0.5): # Coherency index criteria, minimal dispersal required
                        # Check that the vorticty is consistent within the vortex over the course of the simulation (filters out some weird saddle point features,etc)
                        # Timestep to get initial vorticity needs to be day 1 since at day 0 all particles are initialized with vort=0
                        eddy_day1_lons,eddy_day1_lats,eddy_day1_vorts = extract_particles_after_time(traj,x_mask,y_mask,traj_lat_array,sim_params,1)
                        day1_vort_signs = [np.sign(v) for v in eddy_day1_vorts]
                        day1_anti_count,day1_cyc_count = day1_vort_signs.count(-1.0),day1_vort_signs.count(1.0)
                        day1_vort_freq = np.max((day1_anti_count,day1_cyc_count))/(day1_anti_count+day1_cyc_count) #percentage of particles with the same vorticity

                        if (day1_vort_freq >= 0.85): # 85% or more have the same vort sign on day 1
                            dayx_vort_signs = [np.sign(v) for v in eddy_dayx_vorts]
                            dayx_anti_count,dayx_cyc_count = dayx_vort_signs.count(-1.0),dayx_vort_signs.count(1.0)
                            dayx_vort_freq = np.max((dayx_anti_count,dayx_cyc_count))/(dayx_anti_count+dayx_cyc_count)

                            if day1_anti_count > day1_cyc_count: #get the polarity of the vortex on day 1
                                day1_vort_pol = 'anti'
                            else:
                                day1_vort_pol = 'cyc'
                            if dayx_anti_count > dayx_cyc_count: #get the polarity of the vortex on last day
                                dayx_vort_pol = 'anti'
                            else:
                                dayx_vort_pol = 'cyc'

                            # If all criteria are passed, add values to output arrays & move on to next local maxima
                            if (dayx_vort_freq >= 0.85) and (day1_vort_pol == dayx_vort_pol): # Final criteria: vort is consistent for 85% of particles on the last day
                                region_area = rclv.polygon_area(rclv.project_vertices(con, **args)) #Convert area units from pixels to m^2
                                plms.append(ji)
                                cons.append(con)
                                areas.append(region_area/(10**6)) #convert from m^2 -> km^2
                                cds.append(cd)
                                pols.append(day1_vort_pol)
                                break # do not need to look at other CDs if the criteria to this point has been met
            except TypeError: # too close to land to define an RCLV
                break
    return plms,cons,areas,cds,pols

def set_up_RCLV_atlas(date_list):
    """
    Generate the structure for an atlas based on RCLVs identified each date in the date_list

    Input
        date_list: dates to search for RCLVs; need to have already produced an LAVD field for this date
    """
    print('Identifying RCLVs...')

    # Set up RCLV atlas
    RCLV_data = [['Date', 'RCLV ID', 'Orientation', 'Age (days)', 'Area (km^2)', 'Center Lon', 'Center Lat', 'CD', 'flag', 'Boundary Coords']]
    for date in date_list:
        print(date)

        plms,cons,areas,cds,pols = LAVD_to_RCLV(date)
        for i in np.arange(0,len(cons)):
            temp = [date,np.nan,pols[i],np.nan,areas[i],traj_lon_array[plms[i][1]],traj_lat_array[plms[i][0]],cds[i],0]

            # Convert from grid indices to lat/lon coordinates for the output
            lon_bounds = traj_lon_array[[round(j) for j in cons[i][:, 1]]]
            lat_bounds = traj_lat_array[[round(j) for j in cons[i][:, 0]]]
            for j in np.arange(0,len(lon_bounds)):
                temp.extend([lon_bounds[j],lat_bounds[j]])
            RCLV_data.append(temp) # Add RCLV data to main array
    return RCLV_data

####################################### 2. Track the RCLVs through time and ID them #######################################

def track_and_ID_RCLVs(RCLV_data,date_list):
    """
    Track RCLVs through time based on if the center particle in located inside an identified RCLV in the following timestep.
    """
    print('Tracking eddies through time & IDing...')

    # QC step: Bug in the peak_local_max function sometimes outputs the same plm twice..
    # This loop ensures we don't get the same contour twice in the dataset
    remove_contour_inds = []
    for date_to_check in date_list:
        # Look for RCLVs on the same date with identical bounds
        RCLV_inds = np.where(np.array([row[0] for row in RCLV_data]) == date_to_check)[0]
        for a, b in itertools.combinations(RCLV_inds, 2): #tries all combinations from the list
            if RCLV_data[a][9:] == RCLV_data[b][9:]:
                remove_contour_inds.append(a)
    print('Number of identical contours to remove: %s'%(len(remove_contour_inds)))
    RCLV_data = np.delete(RCLV_data,remove_contour_inds)

    RCLV_id = 1
    for d in np.arange(0,len(date_list)): # Iterate through the dates in order to ID eddies that have not been yet
        date = date_list[d]
        print(date)

        # Open the Lagrangian trajectory data for the current date
        traj = xr.open_dataset('%s%s_%s.nc'%(lag_traj_dir,date,filename_str))
        particle_lon,particle_lat = traj.variables["lon"],traj.variables["lat"]

        day_inds = np.where((np.array([row[0] for row in RCLV_data]) == date))[0] # Get indeces of the RCLV data from the current date
        for r in np.arange(0,len(day_inds)): # iterate through RCLVs on the current date
            row = RCLV_data[day_inds[r]]
            if np.isnan(row[1]): #only check rows that don't already have an ID
                row[1] = RCLV_id # Assign the RCLV a new ID
                x_ind = int(np.where(traj_lon_array==row[5])[0]) # center point lon ind
                y_ind = int(np.where(traj_lat_array==row[6])[0]) # center point lat ind
                particle_num = x_ind*len(traj_lat_array) + y_ind #identifier for the center particle in the trajectory file

                # Check if the particle is in an RCLV on the days before the search date, but next in the 'date_list' (because date_list is in backwards order)
                if (d < (len(date_list) - 1)): # Can potentially go up until the end of the date list
                    eddy_stop,counter = 0,1
                    while ((eddy_stop == 0) & (counter < (len(date_list)-d))): #stops when eddy is not found on the next date OR after going through all available dates
                        new_date = date_list[d+counter] # this is the date of the data we will look through in RCLV_data (back ~8 days from date)

                        # Track center particle back in time to the previous initialization date
                        if counter == 1: # get trajectory from the open dataset for 1 timestep back
                            days_btwn = num_days_between(new_date,date) # num of days between date & new_date (typically 8 except for when in between years)
                            # Get lat,lon coords of center particle back 1 timestep
                            backxdays_center_lon = float(particle_lon[particle_num,days_btwn*int(24/sim_params['output_freq'])])
                            backxdays_center_lat = float(particle_lat[particle_num,days_btwn*int(24/sim_params['output_freq'])])

                        else: # open traj dataset for previous date (sequentially forward one timestep) to track particle to current date (counter will be > 1)
                            prev_date = date_list[d+counter-1] # forward one time step from new date
                            prev_traj = xr.open_dataset('%s%s_%s.nc'%(lag_traj_dir,prev_date,filename_str))
                            prev_particle_lon,prev_particle_lat = prev_traj.variables["lon"],prev_traj.variables["lat"]

                            # The new 'center position' will not line up perfectly with the grid of the next initialization, so need to use find_nearest()
                            days_btwn = num_days_between(new_date,prev_date)

                            x_ind = int(np.where(traj_lon_array==backxdays_center_lon)[0]) # center point lon ind
                            y_ind = int(np.where(traj_lat_array==backxdays_center_lat)[0])
                            particle_num = x_ind*len(traj_lat_array) + y_ind #identifier for the center particle in prev_traj_file

                            # Replace old variables with the locations tracked back 8 more days
                            backxdays_center_lon = float(prev_particle_lon[particle_num,days_btwn*int(24/sim_params['output_freq'])])
                            backxdays_center_lat = float(prev_particle_lat[particle_num,days_btwn*int(24/sim_params['output_freq'])])

                        # Check if the center particle back 1 timestep is inside another contour; if so, assign the contour the RCLV ID
                        new_center_point = Point(backxdays_center_lon,backxdays_center_lat)
                        new_day_inds = np.where((np.array([irow[0] for irow in RCLV_data]) == new_date))[0]
                        for nr in np.arange(0,len(new_day_inds)): #iterate through all of the RCLVs at the search date
                            nrow = RCLV_data[new_day_inds[nr]]

                            if (np.isnan(nrow[1])) and (nrow[2] == row[2]): #only want to check RCLVs that don't have an ID & have the same orientation
                                x_bnds, y_bnds = nrow[9:][0::2], nrow[9:][1::2]
                                polygon = Polygon([(x_bnds[pt],y_bnds[pt]) for pt in np.arange(0,len(x_bnds))])
                                if (polygon.contains(new_center_point)): # check if point is inside RCLV boundaries on the previous date
                                    nrow[1] = RCLV_id #change nan to ID
                                    backxdays_center_lon,backxdays_center_lat = nrow[5],nrow[6] # Assign LAVD local max of this contour as the new particle to track
                                    counter += 1
                                    break #stop look through RCLVs for this date and go to the next (back up to the top of the while loop)

                            if nr == (len(new_day_inds) - 1): #on the last RCLV and still haven't found one that contains the center point
                                eddy_stop = 1
                RCLV_id += 1

    return RCLV_data

####################################### 3. QC: Check if any RCLVs skipped a date (or 2) #######################################

def draw_eddy_boundaries_back_timesteps(log_file,date_list,eddy_id,current_date,traj,poly_lons,poly_lats,center_particle_num,n):
    """
    Input
        log_file: Notes about any issues are outputted here
        eddy_id: assigned from RCLV tracking
        current_date: intialization date to draw boundaries back from
        traj: trajectory dataset
        poly_lons,poly_lats: coordinates of the polygon
        center_particle_num: particle ID number in the trajectory dataset
        n: number of times to go back a timestep (i.e., to draw boundaries back ~8 days 4 times, n = 4)
            NOTE: there is a limit on doing this; can't do more than length of the simulation
    Output
        successful_dates: dates that a boundary was successfully drawn
        traj_back_timesteps_boundaries: boundary coordinates
        eddy_center_lons,eddy_center_lats: eddy center coordinates
        areas_km2: area in kilometers of the feature
    """
    # Project the contour on the grid to find all particles inside the eddy
    poly_pts = [(poly_lons[pt],poly_lats[pt]) for pt in np.arange(0,len(poly_lons))] # Reformat lat/lon boundary points
    x_mask,y_mask = find_polygon_pts(poly_pts,traj_lon_array,traj_lat_array)

    # Get the dates of the timesteps
    traj_back_timesteps_dates = []
    current_date_ind = date_list.index(current_date)
    for step in np.arange(1,n+1):
        if (current_date_ind+step) <= (len(date_list)- 1): # do not want to project back from the earliest time
            traj_back_timesteps_dates.append(date_list[current_date_ind+step])

    # Get the coordinates of the particles
    particle_lon,particle_lat = traj.variables["lon"],traj.variables["lat"] # particle lat/lons from the trajectory file

    eddy_center_lons,eddy_center_lats,traj_back_timesteps_boundaries,areas_km2 = [],[],[],[]
    successful_dates = []
    for t in traj_back_timesteps_dates: #iterate through each timestep
        days_btwn = num_days_between(t,current_date)
        time_ind = int(24/sim_params['output_freq'])*days_btwn

        # All particles in the contour locations at timestep t
        eddy_timestepx_lons,eddy_timestepx_lats,_ = extract_particles_after_time(traj,x_mask,y_mask,traj_lat_array,sim_params,days_btwn)
        pts = np.array([[eddy_timestepx_lons[i],eddy_timestepx_lats[i]] for i in np.arange(0,len(eddy_timestepx_lons))])
        new_edges = stitch_boundaries(alpha_shape(pts,0.1)) # alpha=0.1 to get a contour resolution similar to floater output
        lon_bounds,lat_bounds = [],[]
        for i,j in new_edges[0]:
            lon_bounds.append(round(pts[[i,j],0][0],5))
            lon_bounds.append(round(pts[[i,j],0][1],5))
            lat_bounds.append(round(pts[[i,j],1][0],5))
            lat_bounds.append(round(pts[[i,j],1][1],5))

        # Stop drawing contours if the trajectories go outside of the grid domain
        if (np.max(lat_bounds)>traj_lat_array[-1]) or (np.min(lat_bounds)<traj_lat_array[0]) or (np.max(lon_bounds)>traj_lon_array[-1]) or (np.min(lon_bounds)<traj_lon_array[0]):
            log_file.write("RCLV %s went out of bounds on %s\n"%(eddy_id,t))
            break

        successful_dates.append(t)

        # Center particle location at timestep t
        eddy_center_lons.append(float(particle_lon[center_particle_num,time_ind]))
        eddy_center_lats.append(float(particle_lat[center_particle_num,time_ind]))

        areas_km2.append(calc_area_of_stitched_bounds(lon_bounds,lat_bounds,traj_lon_array,traj_lat_array)) #calculate area of new contour

        # Reformat boundaries to be appended to the RCLV dataset
        bounds = []
        for j in np.arange(0,len(lon_bounds)):
            bounds.extend([lon_bounds[j],lat_bounds[j]])
        traj_back_timesteps_boundaries.append(bounds)

    return successful_dates,traj_back_timesteps_boundaries,eddy_center_lons,eddy_center_lats,areas_km2

def interpolate_skipped_contours(RCLV_data,log_file,date_list):
    """
    Look for a skipped contour over one or two timesteps. This will appear in the dataset as two RCLVs with different IDs, but they are actually the same
    water mass. By allowing skips, we correct for accidentally counting the second instance water mass as a new eddy.
    """

    print('Checking if any RCLVs skipped a date...')
    RCLV_data_no_header = np.array(RCLV_data)[1:]
    all_ids = np.unique([r[1] for r in RCLV_data[1:]])

    # Sort eddy ids into dictionaries that contain their first and last date they appear in the dataset
    first_date_dict,last_date_dict = {},{}
    for d in date_list:
        first_date_dict[d],last_date_dict[d] = [],[]

    for eddy_id in all_ids:
        eddy_data = RCLV_data_no_header[np.where(np.array([int(row[1]) for row in RCLV_data_no_header]) == int(eddy_id))]
        eddy_dates = [int(row[0]) for row in eddy_data]
        first_date,last_date = min(eddy_dates),max(eddy_dates)
        first_date_dict[str(first_date)].append(eddy_id)
        last_date_dict[str(last_date)].append(eddy_id)

    # Checking for 1 or 2 timesteps skipped between RCLV contours
    for d in np.arange(0,len(date_list[:-3])):
        current_date = date_list[d]
        print(current_date)

        traj = xr.open_dataset('%s%s_%s.nc'%(lag_traj_dir,current_date,filename_str))
        particle_lon,particle_lat = traj.variables["lon"],traj.variables["lat"]

        for n in [2,3]: #check 2 timesteps back, then 3 timesteps back; contours 1 step back have been given ID already in the tracking function
            back_ntimesteps_date,back_nminus1timesteps_date = date_list[d+n],date_list[d+n-1]
            days_btwn_ntimesteps = num_days_between(back_ntimesteps_date,current_date)
            days_btwn_nminus1timesteps = num_days_between(back_nminus1timesteps_date,current_date) # This is used later in code only if n = 3

            # Subset the data by the dates of interest
            current_date_eddy_data = RCLV_data_no_header[np.where(np.array([int(row[0]) for row in RCLV_data_no_header]) == int(current_date))]
            back_ntimesteps_eddy_data = RCLV_data_no_header[np.where(np.array([int(row[0]) for row in RCLV_data_no_header]) == int(back_ntimesteps_date))]

            # Iterate through RCLVs that start on the current_date in the dictionary
            for eddy_id in first_date_dict[current_date]:
                eddy_data = current_date_eddy_data[np.where(np.array([int(row[1]) for row in current_date_eddy_data]) == int(eddy_id))][0]
                orientation,center_lon,center_lat = eddy_data[2],eddy_data[5],eddy_data[6]
                x_ind = int(np.where(traj_lon_array==float(center_lon))[0])
                y_ind = int(np.where(traj_lat_array==float(center_lat))[0])
                particle_num = x_ind*len(traj_lat_array) + y_ind #identifier for the center particle

                # Get location of center particle back 2 time steps (~16 days) or 3 time steps (~24 days)
                back_ntimesteps_center_lon = float(particle_lon[particle_num,int(24/sim_params['output_freq'])*days_btwn_ntimesteps])
                back_ntimesteps_center_lat = float(particle_lat[particle_num,int(24/sim_params['output_freq'])*days_btwn_ntimesteps])
                new_center_point = Point(back_ntimesteps_center_lon,back_ntimesteps_center_lat)

                # Iterate through the RCLVs that end n timesteps back to check if it is the same water mass
                for back_ntimesteps_eddy_id in last_date_dict[back_ntimesteps_date]:
                    if back_ntimesteps_eddy_id != eddy_id: # ignore contours already belonging to this eddy
                        back_ntimesteps_eddy_data_to_check = back_ntimesteps_eddy_data[np.where(np.array([int(row[1]) for row in back_ntimesteps_eddy_data]) == int(back_ntimesteps_eddy_id))][0]
                        if back_ntimesteps_eddy_data_to_check[2] == orientation: #speeds up computation by only checking eddies with the same orientation
                            bnds = back_ntimesteps_eddy_data_to_check[9:]
                            x_bnds, y_bnds = bnds[0::2],bnds[1::2]
                            polygon = Polygon([(x_bnds[pt],y_bnds[pt]) for pt in np.arange(0,len(x_bnds))])

                            # Check if the center point is inside eddy boundaries
                            if (polygon.contains(new_center_point)): # RCLVs are the same water mass; change the IDs to be the same
                                inds_to_change_id = np.where((np.array([int(irow[1]) for irow in RCLV_data[1:]]) == int(back_ntimesteps_eddy_id)))[0] #look for id in main dataset, ignoring header
                                inds_to_change_id = [i+1 for i in inds_to_change_id] #fix index for main dataset WITH header
                                for ind in inds_to_change_id:
                                    RCLV_data[ind][1] = eddy_id #change ID

                                # Remove the "false" ID from dictionaries
                                last_date_dict[back_ntimesteps_date].remove(back_ntimesteps_eddy_id)
                                for key,values in first_date_dict.items():
                                    if back_ntimesteps_eddy_id in values:
                                        values.remove(back_ntimesteps_eddy_id)
                                        first_date_dict[key].append(eddy_id)

                                # Draw a boundary around the particles for the date that was skipped in between these two eddy contours & add it to the dataset
                                traj_nminus1_timesteps_back_dates,traj_nminus1_timesteps_back_boundaries,traj_nminus1_timesteps_back_center_lons,traj_nminus1_timesteps_back_center_lats,areas_km2 = draw_eddy_boundaries_back_timesteps(log_file,date_list,eddy_id,current_date,traj,eddy_data[9::][0::2],eddy_data[9::][1::2],particle_num,n-1)

                                # Add new data to RCLV_data
                                for k in np.arange(0,len(traj_nminus1_timesteps_back_dates)):
                                        #Format: ['Date', 'RCLV ID', 'Orientation', 'Age (days)', 'Area (km^2)', 'Center Lon', 'Center Lat', 'CD', 'flag', 'Boundary Coords']
                                        #           0        1          2               3             4             5             6            7     8              9
                                    temp = [traj_nminus1_timesteps_back_dates[k],eddy_id,orientation,np.nan,areas_km2[k],
                                            traj_nminus1_timesteps_back_center_lons[k],traj_nminus1_timesteps_back_center_lats[k],np.nan,1] #skip interp flag=1
                                    temp.extend(traj_nminus1_timesteps_back_boundaries[k])
                                    RCLV_data.append(temp)

                                break #stop looking for an eddy match, go back up looking for a match for the next eddy (2 for loops back)

    return RCLV_data

####################################### 4. Interpolate eddy bounds for first 3 timesteps #######################################

def interpolate_first_3timesteps(RCLV_data,log_file,date_list):
    """
    Since the RCLVs are identified every 8 days from 32 day simulations, the first appearance of an RCLV will not have a history before it is
    32 days old. To extract the contours from these early days of the eddy birth, I track the particles back in time and draw a contour around
    the particles.
    """

    print('Interpolating eddy bounds for first 3 timesteps...')
    RCLV_data_no_header = np.array(RCLV_data)[1:]
    all_ids = np.unique([r[1] for r in RCLV_data[1:]])
    for eddy_id in all_ids:

        # Get date & data from the RCLV's earliest appearance
        eddy_data = RCLV_data_no_header[np.where(np.array([int(row[1]) for row in RCLV_data_no_header]) == int(eddy_id))]
        first_date = min([int(row[0]) for row in eddy_data])
        print(first_date)

        first_date_data = eddy_data[np.where(np.array([int(row[0]) for row in eddy_data]) == first_date)][0]
        first_date_lons,first_date_lats = first_date_data[9::2],first_date_data[10::2] #boundary coords
        x_ind = int(np.where(traj_lon_array==first_date_data[5])[0])
        y_ind = int(np.where(traj_lat_array==first_date_data[6])[0])
        particle_num = x_ind*len(traj_lat_array) + y_ind #identifier for the center particle

        # Track the contour back (up to) 3 timesteps
        first_date = str(first_date)
        if first_date != date_list[-1]: #only track backwards if not the first day of the dataset
            if first_date == date_list[-2]: # track backwards 1 timestep
                n = 1
            elif first_date == date_list[-3]: # track backwards 2 timesteps
                n = 2
            else: # track backwards 3 timesteps
                n = 3
            traj = xr.open_dataset('%s%s_%s.nc'%(lag_traj_dir,first_date,filename_str))
            traj_back_timesteps_dates,traj_back_timesteps_boundaries,traj_back_timesteps_center_lon,traj_back_timesteps_center_lat,areas_km2 = draw_eddy_boundaries_back_timesteps(log_file,date_list,eddy_id,first_date,traj,first_date_lons,first_date_lats,particle_num,n)
            for j in np.arange(0,len(traj_back_timesteps_boundaries)):
                #['Date', 'RCLV ID', 'Orientation', 'Age (days)', 'Area (km^2)', 'Center Lon', 'Center Lat', 'CD', 'flag', 'Boundary Coords']
                #   0        1          2               3             4             5             6            7         8              9
                temp = [traj_back_timesteps_dates[j],eddy_id,first_date_data[2],np.nan,areas_km2[j],
                            traj_back_timesteps_center_lon[j],traj_back_timesteps_center_lat[j],np.nan,2] # flag=2 for first 3 timestep interps
                temp.extend(traj_back_timesteps_boundaries[j])
                RCLV_data.append(temp) # Add data to main RCLV array

    return RCLV_data

####################################### 5. QC: Address instances of overlapping contours #######################################

def overlapping_RCLV_QC(RCLV_data,log_file,date_list):
    """
    First step is to clean up RCLV contours by removing small external polygons and lines from the shape.

    In some rare cases there will be overlapping contours of RCLVs from one of the interpolated boundaries steps. Some example
    phenomena of what could cause this include eddy splitting, eddy merging, an "escaping" center particle, etc.
    """
    print('Find instances of overlapping RCLV contours...')

    count3,count4,count5,count6 = 0,0,0,0
    contain_IDs = {}
    for d in date_list:
        print(d)
        RCLV_inds = np.where(np.array([row[0] for row in RCLV_data]) == d)[0]# get all of the data for the RCLVs on the current date (d)

        ############ Keep the largest polygon for 'Multipolygon' or 'GeometryCollection' contours ############
        for ind in RCLV_inds:
            ID = RCLV_data[ind][1]
            bnds = RCLV_data[ind][9:]
            contour_pts = Polygon([(float(bnds[0::2][i]),float(bnds[1::2][i])) for i in np.arange(0,len(bnds[0::2]))])

            # Check intersection with itself to break the shape down to its polygons/lines
            intersection = contour_pts.intersection(contour_pts)
            if intersection.type != 'Polygon':
                count3 += 1
                log_file.write('RCLV %s on %s is a MultiPolygon or GeometryCollection that was reduced to the largest polygon\n'%(ID,d))

                polygons,polygon_areas = [],[] # collect just the polygons, exclude lines
                for geom in intersection:
                    if geom.type == 'Polygon':
                        xx, yy = geom.exterior.coords.xy
                        lon_bounds,lat_bounds = xx.tolist(),yy.tolist()
                        polygons.append(geom)
                        polygon_areas.append(calc_area_of_stitched_bounds(lon_bounds,lat_bounds,traj_lon_array,traj_lat_array))

                # Keep only the polygon with the maximum area
                max_area = max(polygon_areas)
                max_index = polygon_areas.index(max_area)
                new_contour = polygons[max_index]
                new_poly_x, new_poly_y = new_contour.exterior.coords.xy

                # Redefine RCLV boundaries in the main data
                reformat_bounds = []
                for l in np.arange(0,len(new_poly_x)):
                    reformat_bounds.append(new_poly_x[l])
                    reformat_bounds.append(new_poly_y[l])

                RCLV_data[ind][9:] = reformat_bounds
                RCLV_data[ind][8] = str(int(RCLV_data[ind][8])) + str(3) #<- 3 flag will mean the RCLV was a MultiPolygon & part was removed

        ############ Check for overlaps ############
        for a, b in itertools.combinations(RCLV_inds, 2): #tries all combinations from the list
            ID1,ID2 = RCLV_data[a][1],RCLV_data[b][1]
            bnds1,bnds2 = RCLV_data[a][9:],RCLV_data[b][9:]
            contour_pts1 = Polygon([(float(bnds1[0::2][i]),float(bnds1[1::2][i])) for i in np.arange(0,len(bnds1[0::2]))])
            contour_pts2 = Polygon([(float(bnds2[0::2][i]),float(bnds2[1::2][i])) for i in np.arange(0,len(bnds2[0::2]))])

            ############ First check if one contour fully contains the other ##############
            if contour_pts1.contains(contour_pts2):
                count6 += 1
                log_file.write('RCLV %s contains %s on %s\n'%(ID1,ID2,d))
                key_string = '%s,%s'%(ID1,ID2)
                if key_string not in contain_IDs: #Dictionary key order matters here: first ID contains the second
                    contain_IDs[key_string] = [d]
                else:
                    contain_IDs[key_string].append(d)
            elif contour_pts2.contains(contour_pts1):
                count6 += 1
                log_file.write('RCLV %s contains %s on %s\n'%(ID2,ID1,d))
                key_string = '%s,%s'%(ID2,ID1)
                if key_string not in contain_IDs:
                    contain_IDs[key_string] = [d]
                else:
                    contain_IDs[key_string].append(d)

            ############ Next check if there is an intersection ##############
            elif contour_pts1.intersects(contour_pts2): # order does not matter here
                # Intersection is one polygon
                intersection = contour_pts1.intersection(contour_pts2) # get the shape of the intersection
                if intersection.type == 'Polygon':
                    xx, yy = intersection.exterior.coords.xy
                    lon_bounds,lat_bounds = xx.tolist(),yy.tolist()

                    try:
                        area_intersect = calc_area_of_stitched_bounds(lon_bounds,lat_bounds,traj_lon_array,traj_lat_array)
                    except: #area too small to compute
                        area_intersect = 0
                # Intersection is several polygons
                elif intersection.type == 'MultiPolygon':
                    for poly in list(intersection.geoms): # iterate through all of the polygons from the intersection (can also be 1)
                        xx, yy = poly.exterior.coords.xy
                        lon_bounds,lat_bounds = xx.tolist(),yy.tolist()
                        area_intersect = 0
                        try:
                            area_intersect += calc_area_of_stitched_bounds(lon_bounds,lat_bounds,traj_lon_array,traj_lat_array)
                        except: #area too small to compute
                            pass

                # If the intersection is small (<5% of the area of both polygons), then remove the intersecting polygon from both RCLVs
                if (area_intersect/RCLV_data[a][4] <= 0.05) and (area_intersect/RCLV_data[b][4] <= 0.05):
                    count4 += 1
                    log_file.write('Removing intersection (<=5 percent of area) between RCLV %s and %s on %s \n'%(ID1,ID2,d))
                    temp_data_dict = {a:[contour_pts1],b:[contour_pts2]}
                    for ind in (a,b):

                        # Adjust the RCLV data boundaries
                        if ind==a:
                            primary_contour,secondary_contour = contour_pts1,contour_pts2
                        elif ind==b:
                            primary_contour,secondary_contour = contour_pts2,contour_pts1

                        intersection = primary_contour.difference(secondary_contour)
                        new_poly_x,new_poly_y = intersection.exterior.coords.xy

                        reformat_bounds = []
                        for l in np.arange(0,len(new_poly_x)):
                            reformat_bounds.append(new_poly_x[l])
                            reformat_bounds.append(new_poly_y[l])
                        RCLV_data[ind][9:] = reformat_bounds
                        RCLV_data[ind][8] = str(int(RCLV_data[ind][8])) + str(4) #<- 4 flag will mean adjusted boundaries from 5% overlap

                # If the intersection is large, flag it and do nothing
                else:
                    count5 += 1
                    log_file.write('Intersection (>5 percent of area) between RCLV %s and %s on %s \n'%(ID1,ID2,d))
                    for ind in (a,b):
                        RCLV_data[ind][8] = str(int(RCLV_data[ind][8])) + str(5) #<- 5 flag will mean there is an RCLV overlap / split

    #### Combine RCLVs that are fully contained in another into one RCLV ID by removing the contained contour ####
    remove_contour_inds = []
    for key,values in contain_IDs.items():
        ID_to_keep,ID_to_remove = key.split(',')
        ID_to_remove_inds = np.where([row[1] == int(ID_to_remove) for row in RCLV_data])[0]

        # Iterate through overlapping instances
        for ind in ID_to_remove_inds:
            current_date = RCLV_data[ind][0]
            if current_date in values: #check if this is an overlap date
                remove_contour_inds.append(ind) # remove this data point (do it later to not mess up the indexing)
            else: # change ID & flag
                RCLV_data[ind][1] = ID_to_keep # change the ID
                RCLV_data[ind][8] = str(int(RCLV_data[ind][8])) + str(6) #<- 6 flag will mean there was an overlap/ID change

    RCLV_data = np.delete(RCLV_data,remove_contour_inds)

    ### Write summary data to log file ###
    log_file.write('%s small intersections removed\n'%(count4))
    log_file.write('%s large intersections flagged\n'%(count5))
    log_file.write('%s overlapping RCLV instances collapsed\n'%(count6))
    return RCLV_data

####################################### 6. Give RCLVs an age #######################################

def age_RCLVs(RCLV_data):
    print('Giving RCLVs an age & saving the final atlas...')

    # Get the unique IDs
    all_ids = np.unique([r[1] for r in RCLV_data[1:]])

    # Age each instance of each ID
    for i in all_ids:
        eddy_inds = np.where(np.array([str(row[1]) for row in RCLV_data]) == str(i))[0]
        eddy_dates = [int(RCLV_data[e][0]) for e in eddy_inds]
        eddy_dates_sorted, eddy_inds_sorted = (list(t) for t in zip(*sorted(zip(eddy_dates, eddy_inds))))

        eddy_flags = np.array([int(RCLV_data[e][8]) for e in eddy_inds_sorted])

        #Find the first identified RCLV (32 days old); flag will not include '2' (in most cases)

        ref_ind = np.where((['2' not in str(f) for f in eddy_flags]))[0][0]
        if ref_ind > 3: #Rare case: Need to override where there was an overlap at the eddy genesis
            ref_ind = np.where((['6' not in str(f) for f in eddy_flags]))[0][0]
        ref_date = str(eddy_dates_sorted[ref_ind])

        # Iterate through each instance of this eddy ID
        for j in np.arange(0,len(eddy_inds_sorted)):
            if j == ref_ind:
                RCLV_data[eddy_inds_sorted[j]][3] = sim_params['runtime'] #find the first non-interpolated contour, this is a 32-day old instance
            elif j < ref_ind: #subtract dates that come before reference
                RCLV_data[eddy_inds_sorted[j]][3] = sim_params['runtime'] - num_days_between(ref_date,str(eddy_dates_sorted[j]))
            else: #add dates that come after reference
                RCLV_data[eddy_inds_sorted[j]][3] = sim_params['runtime'] + num_days_between(ref_date,str(eddy_dates_sorted[j]))

    # Clean up flags so all are integer strings
    for r in RCLV_data:
        if ('.' in str(r[8])):
            r[8] = int(float(r[8]))

    return RCLV_data


import os
import xlrd
import xlsxwriter

def get_rows_data(filename):
     #既可以打开xls类型的文件，也可以打开xlsx类型的文件
    #w = xlrd.open_workbook('text.xls')
    #w = xlrd.open_workbook('acs.xlsx')
    datas = []
    w = xlrd.open_workbook(filename)
    ws = w.sheets()[0]
    nrows = ws.nrows
    for i in range(nrows):
        data = ws.row_values(i)
        datas.append(data)
    #    print(datas)
    return datas

# 获取所有被合并单元格列表
# 列表中每一项为一个元组，其结构信息如下：
# (合并的首行索引，合并的尾行索引+1，合并的首列索引，合并的尾列索引+1 )
def get_merged_cells(filename):
    wb = xlrd.open_workbook(filename, formatting_info=True)
    ws = wb.sheet_by_index(0)
    return ws.merged_cells[:]

def get_file_datas(filename,row_deal_function=None,grid_end=0,start_row=1):
    """start_row＝1 有一行标题行；gred_end=1 末尾行不导入"""
    """row_del_function 为每行的数据类型处理函数，不传则对数据类型不作处理 """
    wb = xlrd.open_workbook(filename)
    ws = wb.sheets()[0]
    nrows = ws.nrows
    datas = []
    for i in range(start_row,nrows-grid_end):
        row = ws.row_values(i)
        # print(row)
        if row_deal_function:
            row = row_deal_function(row)
        datas.append(row)
    return datas

def get_all_sheets_data(filename,row_deal_function=None,tail_rows=0,start_rows=0, retain_headline=False):
    """ start_rows＝1 有一行标题行；gred_end=1 末尾行不导入
        row_del_function 为每行的数据类型处理函数，不传则对数据类型不作处理
        retain_headline 保留首个sheet的标题行标志"""

    # names = data.sheet_names()
    # table = data.sheet_by_name(sheet_name)

    wb = xlrd.open_workbook(filename)
    # ws = wb.sheets()[0]
    names = wb.sheet_names()
    nsheets = wb.nsheets
    datas = []
    for index, sheet_index in enumerate(range(nsheets)):
        ws = wb.sheet_by_index(sheet_index)
        nrows = ws.nrows
        if retain_headline and index == 0:
            start = 0
        else:
            start = start_rows
        for i in range(start,nrows-tail_rows):
            row = ws.row_values(i)
            if row_deal_function:
                row = row_deal_function(row)
            datas.append(row)
    return datas

def get_cols_data(filename,headline_row_num=0):
    # 按列获取数据
    datas = []
    w = xlrd.open_workbook(filename)
    ws = w.sheets()[0]
    ncols = ws.ncols
    for i in range(ncols):
        data = ws.col_values(i)[headline_row_num:]
        datas.append(data)
    #    print(datas)
    return datas

def get_files(directory):
    files = []
    files = os.listdir(directory)
    files = [f for f in files if f.endswith('.xls') or f.endswith('.xlsx')]
    files = [os.path.join(directory,f) for f in files]
    return files

def countit(datas):
    # 对(datas)列表中的数据项进行分类统计并输出
    from collections import Counter
    c = Counter(datas)
    for k,v in c.items():
        print(k,v,sep='\t')
    return c

def save_datas_xlsx(filename,datas):
    #将一张表的信息写入电子表格中XLSX文件格式
    w = xlsxwriter.Workbook(filename)
    w_sheet = w.add_worksheet('sheet1')
    for rowi,row in enumerate(datas):
        for coli,celld in enumerate(row):
            w_sheet.write(rowi,coli,celld)
    w.close()

def save_data_sheets_xlsx(filename,datass,sheetnames):
    #将对应表中的信息列表写入对应的work_sheet中
    #例：datass=[sheet1表的信息列表，sheet2表的信息列表],sheetnames=['sheet1','sheet2']
    w = xlsxwriter.Workbook(filename)
    sheets = [w.add_worksheet(sheetname) for sheetname in sheetnames]
    for ws,datas in zip(sheets,datass):
        for rowi,row in enumerate(datas):
            for coli,celld in enumerate(row):
                ws.write(rowi,coli,celld)
    w.close()

def summary_col(filename,col_seq_num,res_filename='sum.xlsx'):
    # 统计指定电子表格文件中的指定序号列到指定的文件中
    # filename 指定电子表格文件
    # col_seq_num 列序号从0开始
    # res_filename 统计结果存放文件名
    cols_data = get_data_cols(filename)
    cols_data = cols_data[col_seq_num]
    res = countit(cols_data)
    res = [[k,v] for k,v in res.items()]
    save_datas_xlsx(res_filename,res)

def merge_files_data(mydir,res_filename,headline_rows=0, tail_rows=0):
    # 合并指定目录(mydir)下的分表数据到一个电子表格文件(res_filename)中的一张表中
    if not os.path.exists(mydir):
        print('Directory is not exist.')
        return
    filenames = get_files(mydir)
    datass = []
    for index, filename in enumerate(filenames):
        if index == 0:
            datas = get_all_sheets_data(filename, tail_rows=tail_rows, start_rows=headline_rows, retain_headline=True)
        else:
            datas = get_all_sheets_data(filename, tail_rows=tail_rows, start_rows=headline_rows)
        datass.extend(datas)

    save_datas_xlsx(res_filename,datass)

if __name__ == '__main__':
    merge_files_data('.\\data','res.xlsx',headline_rows=1)


# Name : Abhishek Dethe
# Roll No: 2201037
# Div  A

#4. Write Python program to Count the Number of occurrences of a key-value pair in a text file

def count_key_value(filename, key, value):
    count = 0
    with open(filename, 'r') as file:
        for line in file:
            data = line.split()
            for item in data:
                if item == key:
                    next_item = data[data.index(item) + 1]
                    if next_item == value:
                        count += 1
    return count

filename = 'filename.txt'
key = 'key'
value = 'value'
occurrences = count_key_value(filename, key, value)
print("Number of occurrences:", occurrences)


class Solution:
    def isValidSudoku(self, board: List[List[str]]) -> bool:

        dic_row = defaultdict(list)
        dic_column = defaultdict(list)
        dic_box = defaultdict(list)

        for row in range(len(board)):
            for column in range(len(board[0])):
                val = board[row][column]

                if val == '.':
                    continue
                if (val in dic_row[row] or val in dic_column[column] or val in dic_box[(row//3,column//3)]):
                    print(val)
                    return False
                dic_row[row].append(val)
                dic_column[column].append(val)
                dic_box[(row//3,column//3)].append(val)
        return True


class Solution:
    def myAtoi(self, s: str) -> int:
        result, i, negative, positive = 0, 0, False, False

        while i < len(s) and s[i] == ' ':
            i += 1

        if i < len(s) and s[i] == '+':
            positive = True
            i += 1

        if i < len(s) and s[i] == '-':
            negative = True
            i += 1

        if positive and negative:
            return 0

        while i < len(s) and ord('9') >= ord(s[i]) >= ord('0'):
            result = result * 10 + ord(s[i]) - ord('0')
            i += 1

        if negative:
            result *= -1

        if result < pow(-2, 31):
            return pow(-2, 31)
        elif result > (pow(2, 31) - 1):
            return pow(2, 31) - 1
        else:
            return result


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import logging
import sys

logging.basicConfig(
    stream=sys.stdout,
    level=logging.DEBUG,
    format='%(asctime)s %(name)s-%(levelname)s: %(message)s',
    datefmt='%Y-%m-%d %H:%M:%S')
import argparse
import matplotlib as mpl

mpl.use('Agg')  # TO DISABLE GUI. USEFUL WHEN RUNNING ON CLUSTER WITHOUT X SERVER
from iterationrunner import *

logger = logging.getLogger("StringSimuServer")

def create_argparser():
    parser = argparse.ArgumentParser(
        epilog='String with swarms of tra1jectories code. Intended to dispatch bash jobs and analyze the resulting trajectories.\nBy Oliver Fleetwood 2017-2018.')
    """IMPORTANT CONFIG FOR METHOD"""
    parser.add_argument('-wd', '--working_dir', type=str, help='working directory', default="../gpcr/")
    parser.add_argument('-i', '--iteration', type=int, help='String Iteration', required=True)
    parser.add_argument('-mi', '--max_iteration', type=int, help='Maximum iteration number or the job will finish',
                        required=False,
                        default=15)  # Fairly low so that you make sure to check that everything is working as expected
    parser.add_argument('--min_swarm_batches', type=int,
                        help='Min. number of swarm batches per string point',
                        default=20)
    parser.add_argument('--max_swarm_batches', type=int,
                        help='Max. number of swarm batches per string point.',
                        default=20)
    parser.add_argument('--swarm_batch_size', type=int,
                        help='Number of swarms per submitted batch job.',
                        default=1)
    parser.add_argument('--swarm_convergence_fraction', type=float,
                        help='Maximum fraction of the distance between previous and new average drift vector from the swarms relative to the new drift vector.',
                        default=0.2)
    parser.add_argument("--fixed_endpoints", help="Hold string endpoints fixed",
                        type=lambda x: (str(x).lower() == 'true'),
                        default=True)
    parser.add_argument("-npf", "--new_point_frequency", type=int,
                        help="Frequency at which to add a new point to the string",
                        default=sys.maxint)
    parser.add_argument("--max_number_points", type=int,
                        help="Max number of points on the string",
                        default=60)
    parser.add_argument("--swarm_drift_scale", help="Float to multiply the swarms total drift by",
                        type=float,
                        default=1.)
    parser.add_argument("--equidistant_points", help="Keep points equidistant along string (or dynamic distances will be computed from the swarms)",
                        type=lambda x: (str(x).lower() == 'true'),
                        default=False)
    """ENVIRONMENT CONFIG"""
    parser.add_argument("-env", "--environment", type=str, help="Environment type (local/cluster...)", required=True)
    parser.add_argument("-sm", "--start_mode", help="Start mode (server etc.)",
                        choices=['server', 'setup', 'postprocess', 'analysis', 'append'],
                        required=True)
    parser.add_argument("--simulator", help="Simulation package used",
                        type=str,
                        default="plumed")
    parser.add_argument("--command_gmx", help="Command to run gromacs",
                        # nargs='+',
                        type=str,
                        default="gmx")
    parser.add_argument("--command_submit", help="Command to submit jobs",
                        type=str,
                        default="bash")
    parser.add_argument("--version", help="Code version. ",
                        type=float,
                        default=2.2)
    """DIRECTORIES AND FILES"""
    parser.add_argument('-sp', '--string_filepath', type=str,
                        help='string textfile path with which should be formatted with the current iteration number. Relative to cvs_dir if not absolute.',
                        default="string-paths/string%s.txt")
    parser.add_argument('-cd', '--cvs_dir', type=str, help='CV directory. Relative to working_dir if not absolute.',
                        default="cvs/cvs-len5_good/")
    parser.add_argument('--cvs_filetype', type=str, help='CVs filetype (json or pkl).',
                        default=None)
    parser.add_argument("-sdir", "--structure_dir", type=str,
                        help="Top directory for structures (topology, equilibrated structured and index) for simu system. Relative to working_dir if not absolute.",
                        required=True)
    parser.add_argument('-sd', '--simulation_dir', type=str,
                        help='Simulation directory. Relative to working_dir if not absolute.',
                        default=".string_simu/")
    parser.add_argument('--template_dir', type=str,
                        help='Path to mdp templates etc. Relative to working_dir if not absolute.',
                        default="simulation_config/")
    # parser.add_argument("-top", "--topology" type=str, help="Topology file for simu system", required=True)# default="confout.gro")
    parser.add_argument('-ro', '--restraints_out_file', type=str,
                        help='Restrains output file (relative to CV directory). Restraints will be appended to this file',
                        default="restraints.dat")  # default="topology/restraints/PROA_rest.itp")
    """OTHER"""
    parser.add_argument("-restol", "--restraint_tolerance", type=float, help="Tolerance for restraint force",
                        default=1.2e-3)
    parser.add_argument("--convergence_limit", type=float,
                        help="Limit for when the string has converged between two iterations (we recommend you to confirm convergence in other ways)",
                        default=1e-5)
    parser.add_argument("--simu_id", help="Id to distinguish string simulations from eachother",
                        type=str,
                        default="")
    parser.add_argument("--job_pool_size", help="Maximum number of simultaneous jobs",
                        type=int,
                        default=200)

    return parser

def run_server(runner):
    logger.info("Running Server")
    while runner.iteration <= args.max_iteration:
        runner.run()
        processor = postprocess(runner)
        runner.after_iteration()
        if processor.convergence < args.convergence_limit:  # Will in practice never happen
            logger.info("String converged with %s after iteration %s. Stopping", processor.convergence,
                        runner.iteration)
            break
        else:
            logger.debug("Convergence after iteration %s:%s", runner.iteration, processor.convergence)
        runner.init_iteration(runner.iteration + 1)

def postprocess(runner):
    logger.info("Postprocessing")
    processor = SingleIterationPostProcessor(runner, save=True, plot=False)
    processor.run()
    logger.info("String converge after iteration %s: %s", runner.iteration, processor.convergence)
    return processor

def setup(runner):
    logger.info("Setup")
    runner.create_files()
    return runner

if __name__ == "__main__":
    logger.info("----------------Starting string of swarms simulator by Oliver Fleetwood 2017-2018------------")
    parser = create_argparser()
    args = parser.parse_args()
    # args.command_gmx = '"' + ' '.join(args.command_gmx).replace('"', '') + '"'
    logger.info("Starting simulation with arguments: %s", args)
    runner = StringIterationRunner(args)
    if args.start_mode == 'server' or args.start_mode == 'append':
        run_server(runner)
    elif args.start_mode == 'postprocess':
        postprocess(runner)
    elif args.start_mode == 'setup':
        setup(runner)
    else:
        logger.error("Startmode %s not supported", args.start_mode)
    logger.info("Finished.")


#!/usr/bin/env python3

import cffi_pure

def test_range():
    point = cffi_pure.lib.get_origin()
    point.x = 10
    point.y = 10

    assert not cffi_pure.lib.is_in_range(point, 14)
    assert cffi_pure.lib.is_in_range(point, 15)

def test_ffi():
    point = cffi_pure.ffi.new("Point *", (10, 20))
    assert point.x == 10
    assert point.y == 20


import pandas as pd
import numpy as np
import xgboost as xgb
from xgboost import XGBClassifier
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import accuracy_score,roc_auc_score
from data_process import preprocess_dataset

train_data=pd.read_csv("./data/train.csv")
test_data=pd.read_csv("./data/test.csv")
datasets=[train_data,test_data]
preprocess_dataset(datasets)
drop_elements = ['PassengerId', 'Name', 'Ticket', 'Cabin', 'SibSp','Parch']
train_data = train_data.drop(drop_elements, axis = 1)
test_data  = test_data.drop(drop_elements, axis = 1)

x_train,x_valid,y_train,y_valid=train_test_split(
        train_data.drop(['Survived'],axis=1),
        train_data['Survived'],test_size=0.2,random_state=24)

x_train=x_train.as_matrix()
y_train=y_train.as_matrix()
x_valid=x_valid.as_matrix()
y_valid=y_valid.as_matrix()

clf=XGBClassifier(
    booster='gbtree',nthread=4,
    learning_rate=0.1,min_child_weight=1,
    max_depth=5,
    subsample=0.8,
    colsample_bytree=0.8,
    scale_pos_weight=1,
    objective='binary:logistic',
    seed=24
    )
eval_set=[(x_valid,y_valid)]
clf.fit(x_train,y_train,early_stopping_rounds=100,
    eval_metric='logloss',eval_set=eval_set,verbose=True)
clf.save_model("./model/xgboost.model")
pre=clf.predict(x_valid)
pro=clf.predict_proba(x_valid)[:,1]

print("Accuracy: %f"%accuracy_score(y_valid,pre))
print("AUC Score: %f"%roc_auc_score(y_valid,pro))


import boto3
from botocore.exceptions import ClientError
from google.cloud import storage
import os

IMAGES_MEDIA = (
    "jpg",
    "png",
    "svg",
    "webp",
    "mp3",
    "mp4",
    "mpeg4",
    "wmv",
    "3gp",
    "webm",
)
DOCS = ("doc", "docx", "csv", "pdf")

class Upload:
    """Upload files to AWS S3/Google Cloud Storage

    Methods:
        __init__(self, args, log) -> None: Initializes a new instance of the Upload class with the given args and log
        create_aws_session(self) -> Bucket Object: Returns a AWS S3 bucket session
        create_gcs_session(self) -> Bucket Object: Returns a Google Cloud Storage bucket session
        load_to_s3(self, file, bucket) -> Boolean: Returns True if file upload succesfully else False
        load_to_gcs(self, file, bucket) -> Boolean: Returns True if file upload succesfully else False

    Usage:
        >>> upload = Upload(args, log)
        >>> bucket = upload.create_aws_session()
        >>> file = "test_file.jpg"
        >>> upload.load_to_s3(file, bucket)
        True
        >>> bucket = upload.create_gcs_session()
        >>> file = "test_doc.pdf"
        >>> upload.load_to_gcs(file, bucket)
        True
    """

    def __init__(self, args=None, log=None):
        """Initialize the input args and log object

        gcp_creds: Initialize the google cloud service json file
        """

        self.args = args
        self.log = log
        self.rel_path = os.path.dirname(os.path.realpath(__file__))
        self.gcp_creds = os.path.join(self.rel_path, "gcp_creds.json")

    def load_to_s3(self, _file, bucket):
        """Upload a file to an S3 bucket

        :param _file: File to upload
        :return: True if file was uploaded, else False
        """

        try:
            object_name = os.path.basename(_file)
            bucket.upload_file(_file, object_name)
            return True
        except Exception as error:
            self.log.error(msg=str(error))
            return False

    def load_to_gcs(self, _file, bucket):
        """Upload a file to an Google Cloud Storage Bucket

        :param _file: File to upload
        :return: True if file was uploaded, else False
        """

        try:
            object_name = os.path.basename(_file)
            # Name of the file on the GCS once uploaded
            blob = bucket.blob(object_name)
            blob.upload_from_filename(_file)
            return True
        except Exception as error:
            self.log.error(msg=str(error))
            return False

    def create_aws_session(self):
        """Create AWS session and select bucket

        return: bucket instance if bucket exists in the AWS S3 resource
        """

        try:
            session = boto3.Session(
                aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
                aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
            )

            # Select AWS S3 resource
            s3 = session.resource("s3")
            # Select S3 bucket
            s3.meta.client.head_bucket(Bucket=os.environ["AWS_S3_BUCKET"])
            bucket = s3.Bucket(os.environ["AWS_S3_BUCKET"])
            return bucket
        except ClientError as error:
            self.log.error(msg=str(error))
            return False

    def create_gcs_session(self):
        """Create GSC session and select the bucket

        return: bucket instance if bucket exists in the GCS
        """

        try:
            # Authenticate Google Cloud Storage using the service account private key
            client = storage.Client.from_service_account_json(
                json_credentials_path=self.gcp_creds
            )
            return storage.Bucket(client, os.environ["GCP_BUCKET"])
        except Exception as error:
            self.log.error(msg=str(error))
            return False

    def run(self):
        aws_files, gcp_files = [], []
        # Get files from current and subdirectories
        for root, dirs, files in os.walk(self.args.input_dir):
            if not files:
                self.log.info("No valid files found")
                continue
            for file in files:
                file_type = file.split(".")[-1]
                if file_type in IMAGES_MEDIA:
                    aws_files.append(os.path.join(root, file))
                elif file_type in DOCS:
                    gcp_files.append(os.path.join(root, file))

        # Perform upload operation to AWS S3 bucket if files exists related IMAGES_MEDIA
        if aws_files:
            bucket = self.create_aws_session()
            for file in aws_files:
                self.load_to_s3(file, bucket)

        # Perform upload operation to GSC bucket if files exists related DOCS
        if gcp_files:
            bucket = self.create_gcs_session()
            for file in gcp_files:
                self.load_to_gcs(file, bucket)


from django.urls import path, include
from rest_framework.routers import DefaultRouter
from drf_spectacular.views import (
    SpectacularAPIView, SpectacularSwaggerView, SpectacularRedocView)

from .views import (
    TgUserViewSet, UserMessageViewSet, GPTMessageViewSet,
    TariffViewSet, PaymentViewSet
)

router = DefaultRouter()

router.register(r'tg_users', TgUserViewSet)
router.register(r'tariffs', TariffViewSet)
router.register(r'gpt_messages', GPTMessageViewSet)
router.register(
    r'tg_users/(?P<username>[\w.@+-]+)/payments',
    PaymentViewSet,
    basename='payments'
)
router.register(
    r'tg_users/(?P<username>[\w.@+-]+)/user_messages',
    UserMessageViewSet,
    basename='user_messages'
)

urlpatterns = [
    path('', include(router.urls)),
    path('schema/', SpectacularAPIView.as_view(), name='schema'),
    path(
        'schema/swagger-ui/',
        SpectacularSwaggerView.as_view(url_name='schema'),
        name='swagger-ui'
    ),
    path(
        'schema/redoc/',
        SpectacularRedocView.as_view(url_name='schema'),
        name='redoc'
    ),
]


from odoo import models, api, fields, _
from odoo.exceptions import ValidationError, UserError

from odoo.tools import float_is_zero, float_compare
from itertools import groupby

######### Creación de Pagos ###########

class sale_order_payment_wizard(models.TransientModel):
    _name = "sale.order.payment.wizard"
    _description = "Creación de Pagos desde Ventas"

    @api.model
    def default_get(self, fields):
        res = super(sale_order_payment_wizard, self).default_get(fields)
        record_ids = self._context.get('active_ids', [])
        sale_order_obj = self.env['sale.order']
        if not record_ids:
            return {}
        tickets = []

        partner_ids = []
        currency_ids = []
        mail_channel = self.env['mail.channel']
        channel_ids = []
        channel_id = mail_channel.search([('payment_notification_default','=',True)], limit=1)
        if channel_id:
            res.update(channel_id=channel_id.id)
            res.update(channel_ids=[(6,0,[channel_id.id])])
        amount_total = 0.0
        for ticket in sale_order_obj.browse(record_ids):
            amount_total += ticket.amount_total
            if ticket.state not in ('sale','done'):
                raise UserError("No se puede generar un pago en un Pedido que no este confirmado.")
            if ticket.partner_id.id not in partner_ids:
                partner_ids.append(ticket.partner_id.id)
            if ticket.pricelist_id.currency_id.id not in currency_ids:
                currency_ids.append(ticket.pricelist_id.currency_id.id)
        if len(partner_ids) > 1:
            raise UserError("No se puede generar un pago para multiples clientes.")

        if len(currency_ids) > 1:
            raise UserError("No se puede generar un pago para multiples monedas.")

        res.update(amount_total=amount_total)
        return res

    date       = fields.Datetime(string='Fecha', default=fields.Datetime.now(), required=True,
                              help='This date will be used as the invoice date and period will be chosen accordingly!')
    journal_id = fields.Many2one('account.journal', string='Diario Pago', required=True)

    payment_notification_msg = fields.Char('Mensaje Notificación', size=128)

    channel_id  = fields.Many2one('mail.channel', 'Grupo a Notificar')

    channel_ids  = fields.Many2many('mail.channel', 'mail_channel_payments_wizard_rel', 'wizard_id', 'payment_id',
                                    string='Grupo a Notificar')

    amount_total = fields.Float('Monto Pago')

    def create_payments(self):
        record_ids = self._context.get('active_ids', [])
        sale_order_obj = self.env['sale.order']

        payment_notification_msg = ""
        sales_ref = ""
        partners_mails = ""
        currency_id = self.env.user.company_id.currency_id.id
        amount_total = self.amount_total
        partner_id = False
        for ticket in sale_order_obj.browse(record_ids):
            if ticket.pricelist_id.currency_id:
                currency_id = ticket.pricelist_id.currency_id.id
            sales_ref = sales_ref+", "+ticket.name if sales_ref else ticket.name
            partner_id = ticket.partner_id.id
            ticket.payment_notification_msg = self.payment_notification_msg

        partners_mails = ""
        recipient_ids = []
        # if self.channel_id:
            # for member in self.channel_id.channel_last_seen_partner_ids:
        if self.channel_ids:
            for channel in self.channel_ids:
                for member in channel.channel_last_seen_partner_ids:
                    if member.partner_email:
                        partners_mails = partners_mails+","+member.partner_email if partners_mails else member.partner_email
                    if member.partner_id.id not in recipient_ids:
                        recipient_ids.append(member.partner_id.id)

        payment_vals = {
                            'payment_type': 'inbound',
                            'partner_type': 'customer',
                            'partner_id': partner_id,
                            'ref': sales_ref,
                            'sales_ref': sales_ref,
                            'currency_id': currency_id,
                            'date': self.date,
                            'amount': amount_total,
                            'payment_notification_msg': self.payment_notification_msg,
                            'partners_mails': partners_mails,
                            'recipient_ids': [(6,0,recipient_ids)] if recipient_ids else False,
                        }
        payment_id = self.env['account.payment'].create(payment_vals)
        payment_id.action_post()

        for ticket in sale_order_obj.browse(record_ids):
            ticket.last_payment_id = payment_id.id
            ticket.partners_mails = partners_mails

            msg_body = """

        Estimado(a) usuario,

Se realizo el Pago %s para el pedido de venta.

Información Adicional:  %s

""" % (payment_id.name, self.payment_notification_msg if self.payment_notification_msg else 'NA')
            ticket.message_post(body=msg_body)

        return {
            'name': "Pago Creado",
            'view_mode': 'form',
            'view_id': self.env.ref('account.view_account_payment_form').id,
            'res_model': 'account.payment',
            'context': "{}", # self.env.context
            'type': 'ir.actions.act_window',
            'res_id': payment_id.id,
        }

class AccountPayment(models.Model):
    _inherit ='account.payment'

    def action_post(self):
        res = super(AccountPayment, self).action_post()
        invoice_obj = self.env['account.move']

        for payment in self:
            #### Envio de Correo #####
            mail_compose_message_pool = self.env['mail.compose.message']
            template_id = self.env['mail.template'].search([('name', '=', 'Correo para el Envío Notificaciones de Pagos')], limit=1)

            if template_id and payment.partners_mails:
                recipient_ids = payment.recipient_ids.ids if payment.recipient_ids else False
                ctx = dict(
                    default_model='account.payment',
                    default_res_id=payment.id,
                    default_use_template=bool(template_id),
                    default_template_id=template_id.id,
                    default_composition_mode='comment',
                )
                ## CHERMAN
                context2 = dict(self._context)

                xres = mail_compose_message_pool.with_context(context2).onchange_template_id(template_id=template_id.id,
                                                                                composition_mode=None,
                                                                                model='account.payment',
                                                                                res_id=payment.id)

                message = mail_compose_message_pool.with_context(ctx).create(xres['value'])
                message.partner_ids = [(6,0,recipient_ids)] if recipient_ids else False
                xx = message.action_send_mail()
        return res


import os
import cv2

IMG_PREFIX = "sat"
MASK_PREFIX = "mask"
DEFAULT_IMAGES_FILEPATH = "../../Forest Segmented/all_images"
DEFAULT_MASKS_FILEPATH = "../../Forest Segmented/all_masks"
MASK = True
IMAGE = False

def make_tif_from_images(image_and_paths, is_masks, dest_folder):
    """
    This function takes a set of loaded all_images and their file paths, and saves them in a new folder as a .tif
    if the all_images are all_masks, make them binary and change their name to match the prev image
    :param image_and_paths: 2 columns that save the all_images, and the paths
    :param is_masks: boolean
    :param dest_folder: folder where the all_images came from
    :return: void
    """
    os.mkdir(dest_folder)
    for i in range(len(image_and_paths)):
        img = image_and_paths[i][0]
        name = image_and_paths[i][1]
        if is_masks:
            img = black_and_whitefy(img)
            # NOTICE: this is correct only for the database that we have where each mask is tagged the same as
            # the image only "mask" instead of "sat"
            name = name[:name.find(MASK_PREFIX)] + IMG_PREFIX + name[name.find(MASK_PREFIX) + len(MASK_PREFIX):]
        cv2.imwrite(dest_folder + name[:name.find(".")] + ".tif", img)

def load_images_from_folder(folder):
    """
    gets all the all_images from a folder into a list
    :param folder: folder path
    :return: all_images paired with their filename
    """
    images = []
    for filename in os.listdir(folder):
        img = cv2.imread(os.path.join(folder, filename))
        if img is not None:
            images.append((img, filename))
    return images

def black_and_whitefy(img):
    """
    Takes an image and makes it black and white (only 0 or 255)
    :param img: RGB image
    :return: 0 or 255 image
    """
    gray_image = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    (thresh, black_and_white_image) = cv2.threshold(gray_image, 127, 255, cv2.THRESH_BINARY)
    return black_and_white_image

def load_folder_to_folder(is_mask_folder, masks_filepath=DEFAULT_MASKS_FILEPATH,
                          images_filepath=DEFAULT_IMAGES_FILEPATH):
    if is_mask_folder:
        source = masks_filepath
    else:
        source = images_filepath
    dest = source + '_processed/'
    make_tif_from_images(load_images_from_folder(source), is_mask_folder, dest)

if __name__ == '__main__':
    # format of file is directory all_masks and all_images
    # load_folder_to_folder(IMAGE)
    # load_folder_to_folder(MASK)
    load_folder_to_folder(MASK, '../../Forest Segmented/some_manual_masks')


import pandas as pd
import pickle
import numpy as np
from feature_eng import *
from our_model import *
from predictions import *

def fun_addi(cluster_id):

   product_mappings = pd.read_csv("Dataset\\products.csv")
   product_mappings = product_mappings[['product_id','product_name']]
   product_mappings.to_pickle("Pickle\\product_mappings.pkl")

   departments = pd.read_csv('Dataset\\departments.csv')
   prior_order = pd.read_csv('Dataset\\previous_orders_data.csv')
   train_order = pd.read_csv('Dataset\\training_orders_data.csv')
   orders = pd.read_csv('Dataset\\orders.csv')
   products = pd.read_csv('Dataset\\products.csv')

   if cluster_id != -1:
      clusters = pd.read_csv('Dataset\\clusters.csv')

      users = list(clusters[clusters['cluster_id']==cluster_id]['user_id'])
      orders = orders[orders['user_id'].isin(users)]

   orders.fillna(value = 0, inplace = True)

   prior_train_orders = prior_order.sort_values(by=['order_id'])
   prior_train_orders = pd.merge(left = prior_train_orders, right = products,
                              left_on='product_id', right_on='product_id').sort_values(by=['order_id']).reset_index(drop=True)
   prior_train_orders = pd.merge(left = prior_train_orders, right = departments,
                              left_on='department_id', right_on='department_id').sort_values(by=['order_id']).reset_index(drop=True)
   prior_train_orders = pd.merge(left = prior_train_orders, right = orders,
                              left_on='order_id', right_on='order_id').sort_values(by=['order_id']).reset_index(drop=True)

   col_order = ['user_id',
   'order_id',
   'product_id',
   'department_id',
   'add_to_cart_order',
   'reordered',
   'product_name',
   'department',
   'eval_set',
   'order_number',
   'order_dow',
   'order_hour_of_day',
   'days_since_prior_order']

   prior_train_orders = prior_train_orders[col_order]

   x = prior_train_orders.groupby(['product_id','order_dow','order_hour_of_day','product_name'])['product_name'].size().reset_index(name = 'count').sort_values(
      by=['order_dow','order_hour_of_day','count'], ascending = False)
   top10 = x.groupby(['order_dow','order_hour_of_day']).head(10).reset_index(drop = True)
   # print(top10)
   top10.to_pickle("Pickle\\top10_products.pkl")

   user_last_purchase = pd.DataFrame(columns = ['user_id','date'])
   user_last_purchase['user_id'] = orders['user_id'].unique()
   user_last_purchase['date'] = '2021-03-21'

   user_last_purchase.to_pickle("Pickle\\user_last_purchase.pkl")

def fun_main2(X, cluster_id):
   fun_addi(cluster_id)
   fun_FE(cluster_id)
   fun_train()
   return(get_recommendations({"user_id":X}))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

def Wordclasses(object):
    object.tokens.Token.set_extension("is_EMO1V", getter = lambda token: token.lemma_ in ('like', 'adore', 'want', 'prefer', 'love', 'enjoy') and token.pos_ == 'VERB', force = True)

    object.tokens.Token.set_extension("is_EMO1N", getter = lambda token: token.lemma_ in ('like', 'adoration', 'want', 'preference', 'love', 'enjoyment') and token.pos_ == 'NOUN', force = True)

    object.tokens.Token.set_extension("is_EMO2V", getter = lambda token: token.lemma_ in ('hate', 'dislike', 'disprefer') and token.pos_ == 'VERB', force = True)

    object.tokens.Token.set_extension("is_EMO2N", getter = lambda token: token.lemma_ in ('hate', 'dislike', 'dispreference') and token.pos_ == 'VERB', force = True)


import os
import sys
import time
import json
import datetime
import pymongo as pm
import pandas as pd
import numpy as np
from random import randint

def ts_to_dt(timestamp):
    return datetime.datetime.fromtimestamp(timestamp/1000).strftime('%y.%m.%d.%H:%M:%S')

def query_data(db_info, pair_path, flag=False, video_name=False, dt=False, matched=False, empty=False, only_meta=False):
    ''' Connect DB '''
    client = pm.MongoClient(db_info['address'])
    client.data.authenticate(db_info['autheticate']['name'], db_info['autheticate']['pw'], mechanism=db_info['autheticate']['mechanism'])
    db = client.data
    annotation_data = db.N1SeminarRoom825_Annotation
    sensor_data = db.N1SeminarRoom825_data

    options = {'date': { '$regex' : "2017", '$options' : 'i' }}
    selects = ['start_timestamp', 'end_timestamp', 'label', 'avg_n_human']

    pair_df = pd.read_excel(pair_path, sheet_name=None)

    env_contexts = list(pair_df['env']['name'])
    env_sensors = list(pair_df['env']['sensor_name'])
    new_env_name = dict(zip(env_contexts, env_sensors))

    act_contexts = list(pair_df['act']['name'])
    act_sensors = list(pair_df['act']['sensor_name'])
    new_act_name = dict(zip(act_contexts, act_sensors))

    sound_contexts = list(pair_df['sound']['name'])
    sound_sensors = list(pair_df['sound']['sensor_name'])
    new_sound_name = dict(zip(sound_contexts, sound_sensors))

    if flag:
        options['flag'] = True
    if matched:
        options['matched'] = True
    if video_name:
        selects.append('video_names')

    result_df_columns = ['label', 'start_ts', 'end_ts', 'avg_n_human', 'duration']
    if video_name:
        result_df_columns.append('video_names')
    if dt:
        result_df_columns.append('start_dt')
        result_df_columns.append('end_dt')
    result_df = pd.DataFrame(columns=result_df_columns)

    no_door, yes_door = 0, 0

    ''' Query Activity list '''
    activities = [tuple(map(lambda x: d[x], selects)) for d in \
        annotation_data.find(options).sort('date').sort('start_timestamp')]

    for activity in activities:
        start_ts, end_ts, label, avg_n_human = activity[0], activity[1], activity[2], activity[3]
        if video_name:
            video_names = activity[4]

        ''' Wrong label check '''
        if label in ['?', '---', '-']:
            continue

        ''' Query context and action data'''
        # ambient, monnit, light, aircon, projector
        contexts = [[d['timestamp'], new_env_name[d['name']], d['value']] for d in \
            sensor_data.find({'name': {'$in': env_contexts}, 'timestamp': {'$gt': start_ts, '$lt': end_ts}, 'type': 'context'}).sort('timestamp')]

        # only for dooragent
        actions = [[d['timestamp'], new_act_name[d['name']], 'activate'] for d in \
            sensor_data.find({'name': {'$in': act_contexts}, 'timestamp': {'$gt': start_ts, '$lt': end_ts}, 'type': 'action'}).sort('timestamp')]

        # sound
        sounds = [[d['timestamp'], new_sound_name[d['name']], d['value']] for d in \
            sensor_data.find({'name': {'$in': sound_contexts}, 'timestamp': {'$gt': start_ts, '$lt': end_ts}, 'type': 'context'}).sort('timestamp')]

        ''' Wrong brightness value handling '''
        contexts = list(filter(lambda x: not (x[1] in ['Brightness_1', 'Brightness_2'] and x[2] < 2), contexts))

        sound_table = {'Sound_L': [], 'Sound_R': [], 'Sound_C': []}
        stack_table = {'Sound_L': [], 'Sound_R': [], 'Sound_C': []}
        last_ts_table = {'Sound_L': 0, 'Sound_R': 0, 'Sound_C': 0}

        for i, sdata in enumerate(sounds):
            sound_name = sdata[1]
            if sdata[0] - last_ts_table[sound_name] > 5000:
                if len(stack_table[sound_name]) > 0:
                    sound_table[sound_name].append([stack_table[sound_name][0][0], sound_name, sum(list(map(lambda x: x[2], stack_table[sound_name])))/len(stack_table[sound_name])])
                    stack_table[sound_name].clear()
            stack_table[sound_name].append(sdata)
            last_ts_table[sound_name] = sdata[0]

        ''' Data Processing '''
        sensor_values = sorted(contexts + actions + sound_table['Sound_L'] + sound_table['Sound_R'] + sound_table['Sound_C'], key=lambda x: x[0])
        columns= ['timestamp', 'sensor_name', 'value']

        ''' List empty check '''
        if not sensor_values:
            if empty:
                continue
            print(f'{label}_{start_ts}')
            sensor_values = [['', '', '']]

        if dt:
            columns.append('datetime')
            if sensor_values[0][0] != '':
                sensor_values = list(map(lambda x: x + [ts_to_dt(x[0])], sensor_values))
            else:
                sensor_values[0].append('')

        sensor_table = pd.DataFrame(np.array(sensor_values), columns=columns)

        if not 'Door' in list(sensor_table['sensor_name']):
            first_door = pd.DataFrame([[start_ts+randint(0, 50), 'Door', 'activate']], columns=columns)
            last_door = pd.DataFrame([[end_ts-randint(0, 50), 'Door', 'activate']], columns=columns)
            sensor_table = first_door.append(sensor_table)
            sensor_table = sensor_table.append(last_door)
            no_door += 1

        else:
            yes_door += 1

        metadata_header = result_df_columns
        metadata_content = [f'{label}', f'{start_ts}', f'{end_ts}', f'{avg_n_human}', f'{(end_ts-start_ts)//1000}']
        if video_name:
            metadata_content.append(f'{video_names}')
        if dt:
            metadata_content.append(f'{ts_to_dt(start_ts)}')
            metadata_content.append(f'{ts_to_dt(end_ts)}')

        result_df = result_df.append(pd.DataFrame([metadata_content], columns=metadata_header), ignore_index=True)

        ''' Setup to save result files '''
        sensor_dir = 'sensor'
        metadata_dir = 'metadata'
        if not os.path.isdir(sensor_dir):
            os.mkdir(sensor_dir)
        if not os.path.isdir(metadata_dir):
            os.mkdir(metadata_dir)

        sensor_fname = f'{sensor_dir}/{label}_{start_ts}.csv'
        metadata_fname = f'{metadata_dir}/{label}_{start_ts}.txt'

        ''' Save Sensor data'''
        if not only_meta:
            sensor_table.to_csv(sensor_fname, sep=',', na_rep='', index=False)

        ''' Save Meta data '''
        with open(metadata_fname, 'w') as f:
            for i in range(len(metadata_header)):
                f.write(f'{metadata_header[i]}: {metadata_content[i]}\n')
            f.close()

    print(f'YES door: {yes_door}, NO door: {no_door}')
    ''' Save summary file '''
    result_df.to_excel('SensorData_summary.xlsx', index=False)
    client.close()

if __name__ == '__main__':
    json_file = 'info/db_info.json'
    with open(json_file, 'r') as f:
        db_info = json.load(f)
        f.close()

    pair_path = 'info/context_name_map.xlsx'

    flag = False
    video_name = False
    dt = False
    matched = False
    monnit = False
    empty = False
    only_meta = False

    query_data(db_info, pair_path, flag, video_name, dt, matched, empty, only_meta)


from django import forms

from django.forms.models import inlineformset_factory
from .models import Vehicle, Battery

class VehicleCreateForm(forms.ModelForm):
    batteries_number = forms.IntegerField()

    class Meta:
        model = Vehicle
        exclude = ['']

    def save(self, commit=True):
        if commit:
            self.instance.save()
            for i in range(self.cleaned_data.get('batteries_number')):
                Battery.objects.create(vehicle=self.instance)
        else:
            return super().save(commit)
        return self.instance

class VehicleUpdateForm(forms.ModelForm):

    class Meta:
        model = Vehicle
        exclude = ['']

    def save(self, commit=True):
        if commit:
            self.instance.save()
        else:
            return super().save(commit)
        return self.instance

BatteryFormSet = inlineformset_factory(Vehicle, Battery, form=VehicleUpdateForm, extra=1)


# multiAgents.py
# --------------
# Licensing Information:  You are free to use or extend these projects for
# educational purposes provided that (1) you do not distribute or publish
# solutions, (2) you retain this notice, and (3) you provide clear
# attribution to UC Berkeley, including a link to http://ai.berkeley.edu.
#
# Attribution Information: The Pacman AI projects were developed at UC Berkeley.
# The core projects and autograders were primarily created by John DeNero
# (denero@cs.berkeley.edu) and Dan Klein (klein@cs.berkeley.edu).
# Student side autograding was added by Brad Miller, Nick Hay, and
# Pieter Abbeel (pabbeel@cs.berkeley.edu).

from util import manhattanDistance
from game import Directions
import random, util

from game import Agent

class ReflexAgent(Agent):
    """
    A reflex agent chooses an action at each choice point by examining
    its alternatives via a state evaluation function.

    The code below is provided as a guide.  You are welcome to change
    it in any way you see fit, so long as you don't touch our method
    headers.
    """

    def getAction(self, gameState):
        """
        You do not need to change this method, but you're welcome to.

        getAction chooses among the best options according to the evaluation function.

        Just like in the previous project, getAction takes a GameState and returns
        some Directions.X for some X in the set {NORTH, SOUTH, WEST, EAST, STOP}
        """
        # Collect legal moves and successor states
        legalMoves = gameState.getLegalActions()

        # Choose one of the best actions
        scores = [self.evaluationFunction(gameState, action) for action in legalMoves]
        bestScore = max(scores)
        bestIndices = [index for index in range(len(scores)) if scores[index] == bestScore]
        chosenIndex = random.choice(bestIndices)  # Pick randomly among the best

        "Add more of your code here if you want to"

        return legalMoves[chosenIndex]

    def evaluationFunction(self, currentGameState, action):
        """
        Design a better evaluation function here.

        The evaluation function takes in the current and proposed successor
        GameStates (pacman.py) and returns a number, where higher numbers are better.

        The code below extracts some useful information from the state, like the
        remaining food (newFood) and Pacman position after moving (newPos).
        newScaredTimes holds the number of moves that each ghost will remain
        scared because of Pacman having eaten a power pellet.

        Print out these variables to see what you're getting, then combine them
        to create a masterful evaluation function.
        """
        # Useful information you can extract from a GameState (pacman.py)
        successorGameState = currentGameState.generatePacmanSuccessor(action)
        newPos = successorGameState.getPacmanPosition()
        newFood = successorGameState.getFood()
        newGhostStates = successorGameState.getGhostStates()
        newScaredTimes = [ghostState.scaredTimer for ghostState in newGhostStates]

        # initialize the closest food as very, very far
        closest_food = 9999
        for food in successorGameState.getFood().asList():
            # and find the actually closest food
            closest_food = min(closest_food, manhattanDistance(newPos, food))

        # if a ghost is
        for ghost in successorGameState.getGhostPositions():
            # uncomfortably close
            if manhattanDistance(newPos, ghost) < 2:
                # run
                return -9999

        return successorGameState.getScore() + 1.0 / closest_food

def scoreEvaluationFunction(currentGameState):
    """
    This default evaluation function just returns the score of the state.
    The score is the same one displayed in the Pacman GUI.

    This evaluation function is meant for use with adversarial search agents
    (not reflex agents).
    """
    return currentGameState.getScore()

class MultiAgentSearchAgent(Agent):
    """
    This class provides some common elements to all of your
    multi-agent searchers.  Any methods defined here will be available
    to the MinimaxPacmanAgent, AlphaBetaPacmanAgent & ExpectimaxPacmanAgent.

    You *do not* need to make any changes here, but you can if you want to
    add functionality to all your adversarial search agents.  Please do not
    remove anything, however.

    Note: this is an abstract class: one that should not be instantiated.  It's
    only partially specified, and designed to be extended.  Agent (game.py)
    is another abstract class.
    """

    def __init__(self, evalFn='scoreEvaluationFunction', depth='2'):
        self.index = 0  # Pacman is always agent index 0
        self.evaluationFunction = util.lookup(evalFn, globals())
        self.depth = int(depth)

class MinimaxAgent(MultiAgentSearchAgent):
    """
    Your minimax agent (question 2)
    """

    def getAction(self, gameState):
        """
        Returns the minimax action from the current gameState using self.depth
        and self.evaluationFunction.

        Here are some method calls that might be useful when implementing minimax.

        gameState.getLegalActions(agentIndex):
        Returns a list of legal actions for an agent
        agentIndex=0 means Pacman, ghosts are >= 1

        gameState.generateSuccessor(agentIndex, action):
        Returns the successor game state after an agent takes an action

        gameState.getNumAgents():
        Returns the total number of agents in the game

        gameState.isWin():
        Returns whether or not the game state is a winning state

        gameState.isLose():
        Returns whether or not the game state is a losing state
        """

        # used by pacman
        def max_finder(gameState, depth):
            # if we need to return
            # depth is always +1 to find depth for max
            if gameState.isWin() or gameState.isLose() or depth + 1 == self.depth:
                return self.evaluationFunction(gameState)
            # initialize maxvalue
            largest = -9999
            # replace if there's any better value
            # actions for 0 because it's asking for pacman's actions
            for action in gameState.getLegalActions(0):
                # 0 because it's asking for pacman's successors
                successor = gameState.generateSuccessor(0, action)
                largest = max(largest, min_finder(successor, depth + 1, 1))
            return largest

        # used by ghosts
        def min_finder(gameState, depth, agentIndex):
            # if we need to return
            if gameState.isWin() or gameState.isLose():
                return self.evaluationFunction(gameState)
            smallest = 9999
            # traversing the tree
            for action in gameState.getLegalActions(agentIndex):
                successor = gameState.generateSuccessor(agentIndex, action)
                if agentIndex == (gameState.getNumAgents() - 1):
                    smallest = min(smallest, max_finder(successor, depth))
                else:
                    smallest = min(smallest, min_finder(successor, depth, agentIndex + 1))
            return smallest

        # initialize score and action
        currentScore = -999999
        returnAction = ''
        for action in gameState.getLegalActions(0):
            # traverse the tree - agent 1 is the ghost
            score = min_finder(gameState.generateSuccessor(0, action), 0, 1)
            # if there's a better action (max successors - based on score) choose that
            if score > currentScore:
                returnAction = action
                currentScore = score
        return returnAction

class AlphaBetaAgent(MultiAgentSearchAgent):
    """
    Your minimax agent with alpha-beta pruning (question 3)
    """

    def getAction(self, gameState):
        """
        Returns the minimax action using self.depth and self.evaluationFunction
        """

        # Used by pacman
        def max_finder(gameState, depth, alpha, beta):
            # if we need to return (and depth is +1 to find depth for max)
            if gameState.isWin() or gameState.isLose() or depth + 1 == self.depth:
                return self.evaluationFunction(gameState)
            # initialize maxvalue
            largest = -9999
            temp_alpha = alpha
            # actions for - because it's asking for pacman's actions
            for action in gameState.getLegalActions(0):
                # 0 because it's asking for pacman's successors
                successor = gameState.generateSuccessor(0, action)
                largest = max(largest, min_finder(successor, depth + 1, 1, temp_alpha, beta))
                # prune: not largest - equal, since stated in the project guide
                if largest > beta:
                    return largest
                temp_alpha = max(temp_alpha, largest)
            return largest

        # used by ghosts
        def min_finder(gameState, depth, agentIndex, alpha, beta):
            # if we need to return
            if gameState.isWin() or gameState.isLose():  # Terminal Test
                return self.evaluationFunction(gameState)
            minvalue = 9999
            temp_beta = beta
            # traversing the tree
            for action in gameState.getLegalActions(agentIndex):
                successor = gameState.generateSuccessor(agentIndex, action)
                if agentIndex == (gameState.getNumAgents() - 1):
                    minvalue = min(minvalue, max_finder(successor, depth, alpha, temp_beta))
                    # prune
                    if minvalue < alpha:
                        return minvalue
                    temp_beta = min(temp_beta, minvalue)
                else:
                    minvalue = min(minvalue, min_finder(successor, depth, agentIndex + 1, alpha, temp_beta))
                    # prune
                    if minvalue < alpha:
                        return minvalue
                    temp_beta = min(temp_beta, minvalue)
            return minvalue

        # initialize score and action
        currentScore = -9999
        returnAction = ''
        alpha = -9999
        beta = 9999
        for action in gameState.getLegalActions(0):
            # Next level is a min level. Hence calling min for successors of the root.
            score = min_finder(gameState.generateSuccessor(0, action), 0, 1, alpha, beta)
            # if there's a better action (max successors - based on score) choose that
            if score > currentScore:
                returnAction = action
                currentScore = score
            # Updating alpha
            if score > beta:
                return returnAction
            alpha = max(alpha, score)
        return returnAction

class ExpectimaxAgent(MultiAgentSearchAgent):
    """
      Your expectimax agent (question 4)
    """

    def getAction(self, gameState):
        """
        Returns the expectimax action using self.depth and self.evaluationFunction

        All ghosts should be modeled as choosing uniformly at random from their
        legal moves.
        """

        # used by pacman
        def max_finder(gameState, depth):
            # if we need to return (and depth is +1 to find depth for max)
            if gameState.isWin() or gameState.isLose() or depth + 1 == self.depth:
                return self.evaluationFunction(gameState)
            # initialize maxvalue
            maxvalue = -9999
            # replace if there's any better value
            # action for 0 because it's asking for pacman's actions
            for action in gameState.getLegalActions(0):
                # 0 because it's asking for pacman's successors
                successor = gameState.generateSuccessor(0, action)
                maxvalue = max(maxvalue, random_min(successor, depth + 1, 1))
            return maxvalue

        # used by ghosts
        def random_min(gameState, depth, agentIndex):
            # if we need to return
            if gameState.isWin() or gameState.isLose():  # Terminal Test
                return self.evaluationFunction(gameState)
            # get actions here
            all_actions = gameState.getLegalActions(agentIndex)
            expectation = 0
            actions_len = len(all_actions)
            # traversing the tree
            for action in all_actions:
                successor = gameState.generateSuccessor(agentIndex, action)
                # one agent works in pacman's favor and one doesn't
                if agentIndex == (gameState.getNumAgents() - 1):
                    expectedvalue = max_finder(successor, depth)
                else:
                    expectedvalue = random_min(successor, depth, agentIndex + 1)
                expectation = expectation + expectedvalue
            if actions_len == 0:
                return 0
            return float(expectation) / float(actions_len)

        # initialize score and action
        currentScore = -9999
        returnAction = ''
        for action in gameState.getLegalActions(0):
            # traverse tree
            score = random_min(gameState.generateSuccessor(0, action), 0, 1)
            # if there's a better action (max successors - based on score) choose that
            if score > currentScore:
                returnAction = action
                currentScore = score
        return returnAction

def betterEvaluationFunction(currentGameState):
    """
    Your extreme ghost-hunting, pellet-nabbing, food-gobbling, unstoppable
    evaluation function (question 5).

    DESCRIPTION: <write something here so we know what you did>
    """
    current_position = currentGameState.getPacmanPosition()
    current_food = currentGameState.getFood().asList()
    remaining_food = currentGameState.getNumFood()
    remaining_caps = len(currentGameState.getCapsules())

    closest_food = 9999
    for food in current_food:
        closest_food = min(closest_food, manhattanDistance(current_position, food))

    closest_ghost = 0
    for ghost in currentGameState.getGhostPositions():
        closest_ghost = manhattanDistance(current_position, ghost)
        if closest_ghost < 2:
            return -9999

    additionalFactors = 0
    if currentGameState.isLose():
        additionalFactors -= 50000
    elif currentGameState.isWin():
        additionalFactors += 50000

    return 1.0 / (remaining_food + 1) * 900000 + closest_ghost + \
           1.0 / (closest_food + 1) * 900 + \
           1.0 / (remaining_caps + 1) * 9000 + additionalFactors

# Abbreviation
better = betterEvaluationFunction