## Import required modules
import matplotlib.pyplot as plt # for plotting
import matplotlib # for plotting
import numpy as np # for manipulating arrays
import os # for making/deleting directories
import bioformats # for reading image series
import javabridge # for interfacing with java (required for bioformats)
from tifffile import xml2dict # for parsing the metadata from bioformats
import pickle # for saving python objects and other data
from scipy.optimize import curve_fit # for making fits to the PSF
from scipy.ndimage import gaussian_laplace, gaussian_filter # for dot localization (image filtering)
from skimage import measure # for segmenting images
from skimage.morphology import remove_small_objects, closing, disk # for morphological filtering of images
from skimage.segmentation import clear_border # for filtering images
from skimage.filters import threshold_otsu
import pandas as pd # for creating and manipulating tabulated data
from collections import Iterable
from itertools import product
import copy
import scipy

# settings for making nice pdfs
plt.rcParams['pdf.fonttype'] = 42
plt.rcParams['ps.fonttype'] = 42
plt.rcParams['font.sans-serif'] = "DejaVu Sans"
plt.rcParams['font.family'] = "sans-serif"

javabridge.start_vm(class_path=bioformats.JARS) # start java virtual machine

def get_CZI_metadata(filename,filepath=None,verbose=False):
    """
    Obtains the metadata from a CZI image series.

    Parameters
    ----------
    filename : str
               Name of the file from which to retrieve the z-stack.

    filepath : str, optional
               Path to the file.

    verbose : {T,F}, optional
              If true, prints (sizeX,sizeY,sizeZ,sizeT,num_channels) to standard output

    Returns
    -------
    (sizeX,sizeY,sizeZ,sizeT,num_channels) : tuple of ints
                Information on the length of the sizes of the `X`, `Y`, `Z` (spatial) and `T`
                (temporal) dimensions of the image series and the number of channels, `num_channels`.
                In case of failutre to load, returns a 5-tuple of values 0.

    metadata : dict, or None
               Dictionary containing the full metadata formatted in the Bioformats OME style.
               If loading is unsuccessful, `None` is returned.
    """

    if not filepath is None:
        czi_image = os.path.join(filepath,filename)
    else:
        czi_image = filename

    if not os.path.exists(czi_image):
        return (0,0,0,0,0), None

    metadata = xml2dict(bioformats.get_omexml_metadata(czi_image))
    sizeT = metadata['OME']['Image']['Pixels']['SizeT']
    sizeX = metadata['OME']['Image']['Pixels']['SizeX']
    sizeY = metadata['OME']['Image']['Pixels']['SizeY']
    sizeZ = metadata['OME']['Image']['Pixels']['SizeZ']
    num_channels = len(metadata['OME']['Image']['Pixels']['Channel'])

    if verbose:
        print(sizeX,sizeY,sizeZ,sizeT,num_channels)

    return (sizeX,sizeY,sizeZ,sizeT,num_channels), metadata

def get_CZI_zstack(filename,frame,channel,filepath=None,img_info=None):
    """
    Obtains a single z-stack from a 3D imaging time-series for a specified time and channel.

    Parameters
    ----------
    filename : str
               Name of the file from which to retrieve the z-stack.

    frame : int
               The temporal slice of the image series from which to retrieve the z-stack.

    channel : int
               The channel from which to retrieve the z-stack.

    filepath : str, optional
               Path to the file.

    img_info : tuple of ints, optional
               5-tuple containing lengths of the `X`, `Y`, `Z` (spatial), `T` (temporal) dimensions
               of the image series, and the number of channels, `num_channels`.
               E.g. (sizeX,sizeY,sizeZ,sizeT,num_channels). See output of get_CZI_metadata().
               Pass these pre-computed values for increased speed in batch processing.

    Returns
    -------
    zstack : numpy.ndarray, or None
             Z-stack of the image series specified by the desired `frame`; contains 3 spatial
             dimensions. If loading is unsuccessful, `None` is returned.
    """

    # prepare file name, check that file exists
    if not (filepath is None):
        czi_image = os.path.join(filepath,filename)
    else:
        czi_image = filename
    if not os.path.exists(czi_image):
        return None

    # retrieve image dimensions, and number of channels
    if img_info is None:
        (sizeX,sizeY,sizeZ,sizeT,num_channels), _ = get_CZI_metadata(filename,filepath=filepath)
    else:
        assert len(img_info) == 5
        (sizeX,sizeY,sizeZ,sizeT,num_channels) = img_info

    # make sure frame and channel are in bounds
    assert frame < sizeT
    assert channel < num_channels

    #initialize array and load z-stack
    zstack = np.zeros((sizeZ, sizeY,sizeX))
    with bioformats.ImageReader(czi_image) as reader:
        for z in range(sizeZ):
            zstack[z,:,:] = reader.read(t=frame,z=z,c=channel)

    return zstack

def filter_zstack_DoG(zstack,dog_sigma1 = 1.5,dog_sigma2 = 15,absolute_value=True):
    """
    Applies Difference of Gaussian (DoG) filtering on a single z-stack.

    Parameters
    ----------
    zstack : numpy.ndarray [sizeY by sizeX by sizeZ]
             Z-stack of the image series for a single channel (containing 3 spatial dimentions)

    dog_sigma1 : float, optional
                 Standard deviation of the first Gaussian distribution of the DoG filter.
                 `dog_sigma1` should be close in size to the "dots" being tracked.

    dog_sigma2 : float, optional
                 Standard deviation of the second Gaussian distribution of the DoG filter.
                 `dog_sigma2` should be ~10-times larger than `dog_sigma1`; it helps to smooth
                 local sources noise and background of the image.

    absolute_value : {T,F}, optional
                     Toggles on/off taking the absolute value of the DoG filter result.

    Returns
    -------
    filtered_zstack : numpy.ndarray
                      Absolute value of Difference of Gaussian filtered z-stack.
    """
    filtered_zstack = gaussian_filter(zstack,dog_sigma1)- gaussian_filter(zstack,dog_sigma2)

    if absolute_value==True:
        filtered_zstack = np.abs(filtered_zstack)

    return filtered_zstack

def get_image_threshold(image,method,**kwargs):
    """
    Returns a threshold value for binarizing an image for morphological filtering and dot localization.

    Parameters
    ----------
    image : numpy.ndarray [sizeY by sizeX]

    method : str {'otsu','percentile'}

    kwargs : For method 'otsu'
             `nbins` : int (optinal)
             number of bins used for otsu method

             For method 'percentile'
             `percentile_threshold` : float
             value ranging from 0 to 100

    Returns
    -------
    threshold : float
                Value of threshold determined by the specified method. By default, it is the
                99th percentile of pixel intensities of the image.

    """
    method = method.lower()
    assert method in ['otsu','percentile']

    if 'otsu' == method:
        if 'nbins' in kwargs.keys():
            threshold = threshold_otsu(image,kwargs['nbins'])
        else:
            threshold = threshold_otsu(image)
    else: #'percentile' == method:
        if 'percentile_threshold' in kwargs.keys():
            threshold = np.percentile(image,kwargs['percentile_threshold'])
        else:
            threshold = np.percentile(image,99)

    return threshold

def localize_dots_XY_projection(filtered_zstack, min_object_area=50,\
                                intensity_threshold=None, projectionAxis=2):
    """
    Roughly localizes dots in maximum projection image using morphological filtering.

    Parameters
    ----------
    filtered_zstack : numpy.ndarray [sizeY by sizeX by sizeZ]
                      Z-stack containing 3 spatial dimentions.

    min_object_area : float, optional
                      Minimum area (in pixels) of the object being localized.

    intensity_threshold : float, optional
                          Threshold value by which to binarize the image. By default, this value will
                          be the 99th percentile of pixel intensity values of the maximum projection
                          image. For other ways to choose the `intensity_threshold` value, we refer to:
                          skimage.filters (e.g. threshold_otsu).

    projectionAxis : {2,1,0}, optional
                     Value of the dimension along which to compute the maximum intensity projection.
                     The default is 2 (i.e. removes the Z-dimension).

    Returns
    -------
    centroids : list of ints
                List of integer pixel values close to the centroid of each located "dot" in the
                maximum intensity projection image

    (blobs, blobs_labels,blob_regions) : numpy.ndarray, numpy.ndarray, list of RegionProperties
            `blobs` is the thresholded, morphologically filtered maximum intensity projection image.
            `blobs_labels` is the segmentation of the image after connecting proximal pixels.
            `blob_metrics` is an object containing a list of measured attribues for each unique
             region of `blobs_labels`; `blob_metrics` is the output of skimage.measure.regionprops().
    """

    max_proj = np.max(filtered_zstack,axis=projectionAxis) # get maximum intensity projection

    if intensity_threshold is None:
        intensity_threshold = np.percentile(max_proj,99)

    blobs = max_proj  >  intensity_threshold # binarize image based on global threshold

    # filter objects based on size
    blobs = remove_small_objects(blobs, min_size=min_object_area)

    # remove objects touching the edges of the image
    blobs = clear_border(blobs)

    # "closing" operation to connect proximal pixels
    # blobs =  closing(blobs > intensity_threshold, disk(2))

    # get segmentation of the image from connected pixels
    blobs_labels = measure.label(blobs, background=0)

    # measure things for each unique feature identified in blobs_labels
    blob_metrics = measure.regionprops(blobs_labels, max_proj )

    # get centroids of objects. i.e. (x,y) coordinates
    # note that the values are actually returned as (y,x) coordinates
    centroids = [tuple(np.array(x.weighted_centroid,dtype=int)) for x in blob_metrics]

    return centroids, (blobs, blobs_labels,blob_metrics)

def fit_Gaussian_3D_PSF(zstack, dot_positions_xy, window_size=10,\
                        do_classification=False,do_gaussian_fitting=False,verbose=False):
    """
    Fits specified dots in zstack to 3D Gaussian function.

    Parameters
    ----------
    zstack : numpy.ndarray [sizeY by sizeX by sizeZ]
            Original Z-stack from the image series.

    dot_positions_xy : list of 2-tuples of ints
            List of approximate (X,Y) positions of dots in the Z-stack.

    window_size : int, optional
            Length of area used to crop features out of the z-stack. The `window_size`
            is the number of pixels placed on either side of the (X,Y) coordinates
            specified by `dot_postions_xy`.

    do_classification : {T,F}
            Classifies the number of modes (i.e. number of unique features) in each cropped image.

    do_gaussian_fitting : {T,F}
            If True, a true 3D PSF is fit to the data, otherwise, maximum intensity & x,y,z positions are
            returned, and guesses for the variances.

    Returns
    -------
    dot_fits_dict : dict
                    Contains 3D PSF parameter fit values, and other metrics used
                    for quality control of the fit and feature localization.

                    Attributes of `dot_fits_dict`.
                    'max_projection_xy_data' : maximum intensity projection of the data (XY plane)
                    'max_projection_xz_data' : maximum intensity projection of the data (XZ plane)
                    'max_projection_yz_data' : maximum intensity projection of the data (YZ plane)
                    'max_projection_xy_fit' : maximum intensity projection of the fit (XY plane)
                    'max_projection_xz_fit' : maximum intensity projection of the fit (XZ plane)
                    'max_projection_yz_fit' : maximum intensity projection of the fit (YZ plane)
                    'I0_fit' : maximum intensity of the dot (from fit)
                    'wxy_fit' : standard deviation of the dot along the x and y dimensions (from fit)
                    'wz_fit' : standard deviation of the dot along the z dimension (from fit)
                    'x0_fit' : x dimension best fit value for dot center
                    'y0_fit' : y dimension best fit value for dot center
                    'z0_fit' : z dimension best fit value for dot center
                    'pcov' : covariance matrix for the parameters
                            (I0_fit,wxy_fit,wz_fit,x0_fit,y0_fit,z0_fit)
                    'num_modes' : number of modes identified in `max_projection_{}_data` image
    """

    dot_fits_dict = {}
    win = window_size

    for di, (xc, yc) in enumerate(dot_positions_xy):

        # skip points too close to the frame edge
        sizeX = zstack.shape[0]
        sizeY = zstack.shape[1]
        sizeZ = zstack.shape[2]
        if (xc < win) or (xc >= sizeX-win) or (yc < win) or (yc >= sizeY-win):
            continue

        # crop out the "dot" from the zstack
        dot_volume =  zstack[xc-win:xc+win,yc-win:yc+win,:]

        # flatten the voxels around the dot for fitting purposes
        flat_vol =  np.ndarray.flatten(dot_volume)

        # define the 3D PSF kernel (for plotting)
        def _gauss3D(I0,wxy,wz,x0,y0,z0,background):
            xx = np.arange(xc-win,xc+win)
            yy = np.arange(yc-win,yc+win)
            zz = np.arange(0,sizeZ)
            xmesh,ymesh,zmesh = np.meshgrid(xx, yy,zz, sparse=True)
            divxy = 2*wxy**2
            divz = 2*wz**2
            prefactor = (2*np.pi)**1.5*wxy**2*wz
            return I0*np.exp(-((xmesh-x0)**2+(ymesh-y0)**2)/divxy-(zmesh-z0)**2/divz)/prefactor

        # define the 3D PSF kernel (for fitting)
        def _gauss3D_fit(self,I0,wxy,wz,x0,y0,z0,background):
            xx = np.arange(xc-win,xc+win)
            yy = np.arange(yc-win,yc+win)
            zz = np.arange(0,sizeZ)
            xmesh,ymesh,zmesh = np.meshgrid(xx, yy,zz, sparse=True)
            divxy = 2*wxy**2
            divz = 2*wz**2
            prefactor = (2*np.pi)**1.5*wxy**2*wz
            gauss_ker = I0*np.exp(-((xmesh-x0)**2+(ymesh-y0)**2)/divxy-(zmesh-z0)**2/divz)/prefactor+background
            return np.ndarray.flatten(gauss_ker)

        # generate initial guess of fit values for the curve fitting algorithm
        I0_guess = np.max(dot_volume)
        wxy_guess = 2
        wz_guess = 0.5

        # refine original "centroid" coordinates with a better guess
        yc_rel, xc_rel, zc_rel = np.unravel_index(np.argmax(dot_volume, axis=None), dot_volume.shape)
        yc_guess = yc + yc_rel - window_size
        xc_guess = xc + xc_rel - window_size
        zc_guess = zc_rel

        if do_gaussian_fitting == True:
            # add background parameter to the fit
            background_guess = np.median(dot_volume)
            initial_guess = [I0_guess,wxy_guess,wz_guess,xc_guess,yc_guess,zc_guess,background_guess]

            # place bounds on the fitting parameters
            unc = 2 # pixel uncertainty on the centroid position
            maxI = np.max(dot_volume)
            minI = np.min(dot_volume)
            lower_bounds = [minI,0,0,xc_guess-unc,yc_guess-unc,zc_guess-unc,minI]
            upper_bounds = [maxI,window_size,window_size,\
                            xc_guess+unc,yc_guess+unc,zc_guess+unc,maxI]

            # get the fit parameters
            try:
                (I0_fit,wxy_fit,wz_fit,x0_fit,y0_fit,z0_fit,background_fit), pcov = \
                            curve_fit(_gauss3D_fit,flat_vol, flat_vol,p0=initial_guess,\
                                     bounds=(lower_bounds,upper_bounds))
            except:
                if verbose == True:
                    print('failed at dot {}'.format(di))
                continue
        else:
            I0_fit = I0_guess
            wxy_fit = wxy_guess
            wz_fit = wz_guess
            x0_fit = xc_guess
            y0_fit = yc_guess
            z0_fit = zc_guess
            background_fit = 0
            pcov = []

        # generate the fit volume
        fit_psf = _gauss3D(I0_fit,wxy_fit,wz_fit,x0_fit,y0_fit,z0_fit,background_fit)

        max_projection_xy_data = np.max(dot_volume,axis=2)
        max_projection_xz_data = np.max(dot_volume,axis=0)
        max_projection_yz_data = np.max(dot_volume,axis=1)
        max_projection_xy_fit = np.max(fit_psf,axis=2)
        max_projection_xz_fit = np.max(fit_psf,axis=0)
        max_projection_yz_fit = np.max(fit_psf,axis=1)

        # write maximum projection data and fits to dictionary
        dot_fits_dict[di] = {'max_projection_xy_data':max_projection_xy_data,\
                          'max_projection_xz_data':max_projection_xz_data, \
                          'max_projection_yz_data':max_projection_yz_data, \
                          'max_projection_xy_fit':max_projection_xy_fit, \
                          'max_projection_xz_fit':max_projection_xz_fit, \
                          'max_projection_yz_fit':max_projection_yz_fit, \
                          'I0_fit':I0_fit,\
                          'wxy_fit':wxy_fit,\
                          'wz_fit':wz_fit,\
                          'x0_fit':x0_fit,\
                          'y0_fit':y0_fit,\
                          'z0_fit':z0_fit,\
                          'pcov':pcov,\
                          'num_modes': {}}

        # classify the number of modes in each maximum projection data image
        if do_classification == True:
            num_modes = {}
            for img_key in ['max_projection_xy_data','max_projection_xz_data','max_projection_yz_data']:
                img = dot_fits_dict[di][img_key]
                dot_fits_dict[di]['num_modes'].update({img_key : count_dots_from_threshold(img)})

    return dot_fits_dict

def do_one_frame(filename,frame, channel=0, img_info=None, dog_sigma1=1.5, dog_sigma2=3, \
                 min_object_area=50, intensity_threshold_method='percentile',
                 window_size=10, classify_dots=True,do_gaussian_fitting=False, load_file_path=None, \
                 save_intermediates_file_path=None, return_intermediates=False,verbose=False,**kwargs ):
    """
    Localizes dots and performs 3D PSF fitting on a single frame (z-stack)

    Parameters
    ----------
    filename : str
               Name of the file from which to retrieve the z-stack.

    frame : int
               The temporal slice of the image series from which to retrieve the z-stack.

    channel : int, optional
               The channel from which to retrieve the z-stack.

    img_info : tuple of ints, optional
                Pre-retrieved metadata for increased speed in batch processing.
                5-tuple containing lengths of the `X`, `Y`, `Z` (spatial), `T` (temporal)
                dimensions of the image series, and the number of channels, `num_channels`.
                See output of get_CZI_metadata().

    dog_sigma1 : float, optional
                Standard deviation of the first Gaussian distribution of the DoG filter.
                `dog_sigma1` should be close in size to the "dots" being tracked.
                See filter_zstack_DoG().

    dog_sigma2 : float, optional
                Standard deviation of the second Gaussian distribution of the DoG filter.
                `dog_sigma2` should be larger than `dog_sigma1`. See filter_zstack_DoG().

    min_object_area : float, optional
                Minimum area (in pixels) of the object being localized.
                See localize_dots_XY_projection().

    intensity_threshold_method : str, optional
                Method of selecting the threshold value by which to binarize the filtered z-stack
                image. By default, the method is 'percentile', and will use the 99th percentile
                of pixel intensity values. For other methods, see get_image_threshold().

    window_size : int, optional
                Length of area used to crop features out of the z-stack. The `window_size`
                is the number of pixels placed on either side of localized dot centroids.
                See fit_Gaussian_3D_PSF()

    classify_dots : {T,F}
                Counts the number of dots found in each cropped feature (of window size
                defined by `window_size`).

    load_file_path : str, optional
                Path to the file from which to retrieve the z-stack.

    save_intermediates_file_path : str, optional
                Path to a folder in which to save intermediate results from the analysis.
                Intermediates saved will include `dot_fits_dict`, `blobs`, `filtered_zstack`.
                If the specified folder does not exist, it is created.

    return_intermediates : {T,F}, optional
                Option to return not only `fits_df` but also the intermediates including
                `dot_fits_dict`, `blobs`, `blobs_labels`, `blob_metrics` and `filtered_zstack`.

    verbose : {T,F}, optional
                Prints to standard output the steps being performed.

    **kwargs : optional
                Pass key word arguments. For example to get_image_threshold() to specify
                parameters for the thresholding method (e.g. if `intensity_threshold_method`
                is 'percentile', one can optionally pass `percentile_threshold=90` to threshold
                at the 90th percentile instead of the default of 99th percentile).

    Returns
    -------
    fits_df : pandas DataFrame
                DataFrame containing information on the X,Y,Z PSF localization, frame number,
                channel and intensity of each localized dot in the z-stack.

    Additionally Returns (if `return_intermediates`== True):
    --------------------

    dot_fits_dict : dict
                Contains 3D PSF parameter fit values, and other metrics used
                for quality control of the fit and feature localization.

                Attributes of `dot_fits_dict`.
                'max_projection_xy_data' : maximum intensity projection of the data (XY plane)
                'max_projection_xz_data' : maximum intensity projection of the data (XZ plane)
                'max_projection_yz_data' : maximum intensity projection of the data (YZ plane)
                'max_projection_xy_fit' : maximum intensity projection of the fit (XY plane)
                'max_projection_xz_fit' : maximum intensity projection of the fit (XZ plane)
                'max_projection_yz_fit' : maximum intensity projection of the fit (YZ plane)
                'I0_fit' : maximum intensity of the dot (from fit)
                'wxy_fit' : standard deviation of the dot along the x and y dimensions (from fit)
                'wz_fit' : standard deviation of the dot along the z dimension (from fit)
                'x0_fit' : x dimension best fit value for dot center
                'y0_fit' : y dimension best fit value for dot center
                'z0_fit' : z dimension best fit value for dot center
                'pcov' : covariance matrix for the parameters
                        (I0_fit,wxy_fit,wz_fit,x0_fit,y0_fit,z0_fit)
                'num_modes' : dict; key is `max_projection_{}_data`, value is # modes found in image

    zstack : numpy.ndarray [sizeY by sizeX by sizeZ]
             Z-stack of the image series for a single channel (containing 3 spatial dimentions)

    filtered_zstack : numpy.ndarray
                      Absolute value of Difference of Gaussian filtered z-stack.

    centroids : list of ints
                List of integer pixel values close to the centroid of each located "dot" in the
                maximum intensity projection image

    blobs_labels : numpy.ndarray
                `blobs_labels` is the segmentation of the image after thresholding, morphologically
                filtering and connecting proximal pixels of `filtered_zstack_max_projection` image.

    blob_metrics : object
                Metrics for each `blob_labels` region can be obtained from skimage.measure.regionprops().

    Saved to disk (if `save_intermediates_file_path` is provided)
    -------------

    fits_df : pandas DataFrame
                (see `fits_df` above)

    dot_fits_dict : dict
                (see `dot_fits_dict` above)

    filtered_zstack_max_projection : numpy.ndarray [sizeY by sizeX]
                Maximum projection of the filtered Z-stack onto the X,Y plane.

    centroids : list of ints
                (see `centroids` above)

    blobs_labels : numpy.ndarray
                (see `blob_labels` above)

    """

    # loads a z-stack from a single channel from the specified frame
    if verbose: print("\nLoading: {}\nFrame {} Channel {}".format(filename,frame,channel))
    zstack = get_CZI_zstack(filename,frame,channel,filepath=load_file_path,img_info=img_info)

    # apply DoG filter
    if verbose: print("1) Appling Difference of Gaussian filter to z-stack.")
    filtered_zstack = filter_zstack_DoG(zstack,dog_sigma1=dog_sigma1,dog_sigma2=dog_sigma2)

    # obtain image threshold
    if verbose: print("2) Obtaining image threshold using method: {}".format(intensity_threshold_method))
    thresh = get_image_threshold(np.max(filtered_zstack,2), intensity_threshold_method, **kwargs)

    # localize dots [(X,Y) coordinates] from the maximum projection of `filtered_zstack`
    if verbose: print("3) Morphological filtering and localizing dots in 2D.")
    loc_output = localize_dots_XY_projection(filtered_zstack, min_object_area=min_object_area,\
                      intensity_threshold=thresh, projectionAxis=2)
    centroids, (blobs, blobs_labels,blob_metrics) = loc_output

    # do 3D PSF fitting
    if verbose: print("4) 3D PSF fitting and localizing dots in 3D.")
    dot_fits_dict = fit_Gaussian_3D_PSF(zstack, centroids, window_size=window_size,\
                                        do_classification=classify_dots, \
                                        do_gaussian_fitting=do_gaussian_fitting, verbose=False)

    # unpack `dot_fits_dict` fit values into a pandas DataFrame
    if verbose: print("5) Generating pandas DataFrame from the PSF fits")
    I0_fits = []
    wxy_fits = []
    wz_fits = []
    x0_fits = []
    y0_fits = []
    z0_fits = []
    num_modes = []
    for key in dot_fits_dict.keys():
        I0_fits.append(dot_fits_dict[key]['I0_fit'])
        wxy_fits.append(dot_fits_dict[key]['wxy_fit'])
        wz_fits.append(dot_fits_dict[key]['wz_fit'])
        x0_fits.append(dot_fits_dict[key]['x0_fit'])
        y0_fits.append(dot_fits_dict[key]['y0_fit'])
        z0_fits.append(dot_fits_dict[key]['z0_fit'])
        modes_list = [dot_fits_dict[key]['num_modes'][k] for k in dot_fits_dict[key]['num_modes'].keys()]
        num_modes.append(np.mean(modes_list)) # allow for 1 false-positive

    colnames = ['channel','frame','x','y','z','intensity','avg_num_modes']
    channels = [int(channel)]*len(I0_fits)
    frames = [int(frame)]*len(I0_fits)

    fits_df = pd.DataFrame([channels,frames,x0_fits,y0_fits,z0_fits,I0_fits,num_modes],index=colnames).transpose()

    # save intermediates
    if not save_intermediates_file_path is None:
        if verbose: print("6) Saving intermediates.")
        # check if folder exists, if not make it
        if not os.path.exists(save_intermediates_file_path):
            os.makedirs(save_intermediates_file_path)

        # generate file name and save Data Frame to csv filel
        data_frame_filename = os.path.join(save_intermediates_file_path,\
                                           'frame{}_channel{}_dotFitsDict.csv'.format(frame,channel))
        fits_df.to_csv(data_frame_filename)

        # generate file name and save other information to pickled object
        analysis_intermediates = {'filtered_zstack_max_projection': np.max(filtered_zstack,2),\
                                 #'blobs': blobs,\
                                 'blobs_labels': blobs_labels,\
                                 #'blob_metrics': blob_metrics,\
                                 'centroids': centroids,\
                                 'dot_fits_dict': dot_fits_dict}
        intermediates_filename = os.path.join(save_intermediates_file_path,\
                               'frame{}_channel{}_AnalysisIntermediates.pkl'.format(frame,channel))
        pickle.dump(analysis_intermediates, open(intermediates_filename,'wb'))

    if return_intermediates == True:
        return fits_df, dot_fits_dict, zstack, filtered_zstack, centroids, blobs_labels, blob_metrics
    else:
        return fits_df

def do_all_frames_all_channels(save_output_path=None,**params_dict):
    """
    Parameters
    ----------
    params_dict : dict
                Dictionary containing 3 mandatory components:
                1) filename : str
                              Name of the image series to be analyzed
                2) filepath : str
                              Full file path to the image series
                3) channel_params_dict : dict (or list of dicts)
                              A dictionary of keyword arguments that will be passed to do_one_frame().
                              If `channel` is anonymous (i.e. if it is not specified in the arguments),
                              the parameters passed  to do_one_frame() will be the same for all channels.
                              If `channel_params_dict` contains a list of dictionaries (e.g. specifying
                              parameters for each channel), the channel-specific parameters will be passed
                              to do_one_frame(); if channel-specific parameters are unspecified, the
                              function attempts to use parameters from the last anonymous channel. Otherwise,
                              function defaults are used.

    save_output_path : str, optional
                Path specifying where to save `df_all` to disk

    Example
    -------
    E.g. params_dict = {'filename': 'my_file.czi',
                        'filepath': './file_location/',
                        'channel_params_dict': [{'channel': 0,
                                              'dog_sigma1': 1.5,
                                              'dog_sigma2': 15,
                                              'min_object_area': 50,
                                              'intensity_threshold_method': 'percentile',
                                              'percentile_threshold': 99,
                                              'window_size': 10,
                                              'save_intermediates_file_path': './tmp'},

                                               {'channel': 1,
                                               'dog_sigma1': 1.5,
                                               'dog_sigma2': 3,
                                               'min_object_area': 35,
                                               'intensity_threshold_method': 'percentile',
                                               'percentile_threshold': 99,
                                               'window_size': 10,
                                               'save_intermediates_file_path': './tmp'},
                                                ]
                         }
    Returns
    -------
    df_all : pandas DataFrame
            DataFrame containing 3D PSF fit information for all localized dots in the image series.
            This DataFrame is structured such that it can be passed to .

    Saving intermediate steps
    --------------------------
    Note. Intermediate outputs are saved to disk if `save_intermedites_file_path` is specified.
          Saved outputs are structured as specified in do_one_frame().
    """

    """
    To do: fix depracated: channel_params_dict = params_dict['channel_params_dict'][channel]
    """

    df_list = []

    filename = params_dict['filename']
    filepath = params_dict['filepath']

    # get metadata
    img_info, _ = get_CZI_metadata(filename,filepath)
    (sizeX,sizeY,sizeZ,sizeT,num_channels) = img_info

    # iterate over channels and frames
    for (channel,frame) in product(range(num_channels), range(sizeT)):

        # retrieve fitting parameters for specified channel
        channel_params_dict = params_dict['channel_params_dict']
        this_channel_params_dict = None

        # search list for channel-specific parameters dicts
        if type(channel_params_dict)==list:
            for d in channel_params_dict:
                if 'channel' in d:
                    if d['channel'] == channel:
                        # use channel-specific parameters
                        this_channel_params_dict = d
                        break
                else:
                    # use anonymous channel parameters
                    this_channel_params_dict = d

        # search for channel-specific parameters
        elif type(channel_params_dict) == dict:
            if 'channel' in channel_params_dict:
                if channel_params_dict['channel'] == channel:
                    this_channel_params_dict = channel_params_dict
            else:
                # use anonymous channel parameters
                this_channel_params_dict = channel_params_dict

        # if there is no dict for an anonymous channel, use default parameters
        if this_channel_params_dict is None:
            this_channel_params_dict = {}

        # do analysis for one frame
        df = do_one_frame(filename,frame, channel, \
                          img_info, load_file_path=filepath, \
                          verbose = True, **this_channel_params_dict)

        # append output to list
        df_list.append(df)
    df_all = pd.concat(df_list)

    # generate file name and save `df_all` to csv file
    if not save_output_path is None:
        if not os.path.exists(save_output_path):
                os.makedir(save_output_path)
        fits_df.to_csv(os.path.join(save_output_path,'Combined_dotFitsDict.csv'))

    return df_all

def batch_parameter_sweep(function_to_call=None,**batch_arguments):
    """
    Makes `params_dict` dictionaries for all combinations of the parameters passed
    to `batch_arguments`. Subsequently, calls `function_to_call` usings `params_dict`.

    Usage
    ------
    E.g. The following will run "do_first_and_last()" for 'file1' and 'file2' for all
         combinations of 'dog_sigma2' and 'dog_sigma1'.

        batch_parameter_sweep(do_first_and_last, filename='['file1','file2'], \
                              dog_sigma2=[3,15], dog_sigma1=[1.5,2])

    Parameters
    ----------
    function_to_call : function
                    Any function can use `params_dict` as an input argument
                    such as: do_all_frames_all_channels()

    **batch_arguments : dict
                    Dictionary of keyword, value pairs. Keywords should be arguments for
                    `channel_params_dict` and the corresponding values may be iterables.

    Returns
    -------
    params_dict_list : list of dicts
                    List of `params_dict` dictionaries - one dictionary for each permutation
                    of values passed to `batch_arguments`

    function_to_call_output : obj
                    The output object will depend on whatever is the output of `function_to_call`

    """

    # if filepath is specified, return error
    if 'filepath' in batch_arguments.keys():
        raise Exception('Do not specify ''filepath'' as an argument. Include the file path in filename.')

    special_keys = ['filename','channel']

    # if any "value" in batch_arguments is not iterable, make it iterable
    # do not treat strings as iterable objects -> put non-iterable items into a list
    for key, value in batch_arguments.items():
        if not isinstance(value, Iterable):
            batch_arguments[key] = [value]
        elif isinstance(value,str):
            batch_arguments[key] = [value]

    # get all keyword : (iterable) value pairs that are not 'filename' and 'filepath'
    batch_dict = {x: batch_arguments[x] for x in batch_arguments if x not in special_keys}

    # get all keyword : (iterable) value pairs that are not 'filename' and 'channel'
    names_dict = {x: batch_arguments[x] for x in batch_arguments if x in special_keys}

    # generate all permutations of keyword : value pairs from the iterables in batch_dict
    channel_params_dict_list = [dict(zip(batch_dict.keys(), a)) for a in product(*batch_dict.values())]

    tmp_list = []
    if 'channel' in names_dict:
        for di, dict_item in enumerate(channel_params_dict_list):
            tmp_list.append([])
            # copy each channel_params_dict in channel_params_dict_list
            # now add 'channel' information
            for ci, ch in enumerate(names_dict['channel']):
                tmp_list[di].append(copy.deepcopy(dict_item))
                tmp_list[di][ci].update({'channel':ch})
        channel_params_dict_list = tmp_list

    # create a list of params_dict
    params_dict_list = []
    for fullname in names_dict['filename']:
         for channel_params in channel_params_dict_list:

            filepath, filename  = os.path.split(fullname)
            params_dict_list.append({'filename':filename, \
                                 'filepath':filepath, \
                                 'channel_params_dict': channel_params})

    function_to_call_output = None
    if not function_to_call is None:
        for params_dict in params_dict_list:
            function_to_call_output = function_to_call(params_dict)

    return params_dict_list

def do_first_and_last(plot_order=('frame','channel'),these_channels_only=None,\
                      figure_save_path=None, verbose=False, **params_dict):
    """
    Performs "do_one_frame()" on the first and last frame of the data series for the channels
    specified by `channel_params_dict` and outputs intermediates for easy visualization. If no
    channels are specified, all channels are used

    Parameters
    ----------
    plot_order : ('frame','channel') or ('channel', 'frame')
               Default value ('frame','channel') plots the 'first' and 'last' frame, grouped by 'channel'.
               The value ('channel','frame') plots the channels in order, grouped by frame.

    these_channels_only : list of ints, optional
                The list of integers in `these_channels_only` specify the specific channels for which
                ouput plots are desired. Plots are generated for the first and last frame.
                If `these_channels_only` is unspecified, plots are generated for all channels.

    figure_save_path : str, optional
                Path to a directory where the ouput plots will be saved to disk. If the directory
                does not exist, it is created. If `figure_save_path` is not set, plots are not saved.

    verbose : {T,F}, optional
                Toggles 'on' or 'off' the verbose option of do_one_frame(). See do_one_frame()

    **params_dict : dict or keyword/value pairs, containing 3 mandatory components
                1) filename : str
                              Name of the image series to be analyzed
                2) filepath : str
                              Full file path to the image series
                3) channel_params_dict : dict (or list of dicts)
                              A dictionary of keyword arguments that will be passed to do_one_frame().
                              If `channel` is anonymous (i.e. if it is not specified in the arguments),
                              the parameters passed  to do_one_frame() will be the same for all channels.
                              If `channel_params_dict` contains a list of dictionaries (e.g. specifying
                              parameters for each channel), the channel-specific parameters will be passed
                              to do_one_frame(); if channel-specific parameters are unspecified, the
                              function attempts to use parameters from the last anonymous channel. Otherwise,
                              function defaults are used.

                If a dict is passed (instead of keyword/value pairs), `params_dict` values must be unpacked
                (i.e. pass `**params_dict` to the function instad of simply `params_dict`).

    Example formatting
    ------------------
    E.g. params_dict = {'filename': 'my_file.czi',
                        'filepath': './file_location/',
                        'channel_params_dict': [{'channel': 0,
                                              'dog_sigma1': 1.5,
                                              'dog_sigma2': 15,
                                              'min_object_area': 50,
                                              'intensity_threshold_method': 'percentile',
                                              'percentile_threshold': 99,
                                              'window_size': 10,
                                              'save_intermediates_file_path': './tmp'},

                                               {'channel': 1,
                                               'dog_sigma1': 1.5,
                                               'dog_sigma2': 3,
                                               'min_object_area': 35,
                                               'intensity_threshold_method': 'percentile',
                                               'percentile_threshold': 99,
                                               'window_size': 10,
                                               'save_intermediates_file_path': './tmp'},
                                                ]
                         }
    See also: batch_parameter_sweep() as a method to easily generate this dict.

    Output
    -------

    Figure 1: {filename}_LocaDotPlots.pdf
                Shows the maximum intensity projection of onto the xy plane of 1) the raw z-stack,
                2) the filtered z-stack, 3) The dot segmentation (with 'dot' IDs).

    Figure 2 and up : {filename}_DotsFitPlots_Channel{channel}_Frame_{frame}.pdf
                Each figure will show the maximum projection of a small window around each dot.
                For each dot, xy, xz and yz projections are shown for 1) the raw data and
                2) the PSF fits.

    """

    filename = params_dict['filename']
    filepath = params_dict['filepath']

    # get metadata
    img_info, _ = get_CZI_metadata(filename,filepath)
    (sizeX,sizeY,sizeZ,sizeT,num_channels) = img_info

    # prepare figure for plotting
    numPlots = len(list(product(range(num_channels),[0,sizeT-1])))
    xwidth = 3
    ywidth = numPlots
    width_inches = 3

    fig, gs = _gridspec_inches(wcols = np.array([width_inches]*xwidth),\
                              hrows =np.array([width_inches]*ywidth),\
                             hspace=0.35,wspace=0.25)

    # iterate over channels and frames
    count = 0

    if these_channels_only is 'None':
        constrain_to_channels = range(num_channels)
    else:
        constrain_to_channels = these_channels_only

    if plot_order == ('channel','frame'):
        channel_frame_pairs = product(constrain_to_channels,[0,sizeT-1])
    else:
        channel_frame_pairs = [(y,x) for (x,y) in list(product([0,sizeT-1],constrain_to_channels))]

    for (channel,frame) in channel_frame_pairs:

        # retrieve fitting parameters for specified channel
        channel_params_dict = params_dict['channel_params_dict']
        this_channel_params_dict = None

        # search list for channel-specific parameters dicts
        if type(channel_params_dict)==list:
            for d in channel_params_dict:
                if 'channel' in d:
                    if d['channel'] == channel:
                        # use channel-specific parameters
                        this_channel_params_dict = d
                        break
                else:
                    # use anonymous channel parameters
                    this_channel_params_dict = d

        # search for channel-specific parameters
        elif type(channel_params_dict) == dict:
            if 'channel' in channel_params_dict:
                if channel_params_dict['channel'] == channel:
                    this_channel_params_dict = channel_params_dict
            else:
                # use anonymous channel parameters
                this_channel_params_dict = channel_params_dict

        # if there is no dict for an anonymous channel, use default parameters
        if this_channel_params_dict is None:
            this_channel_params_dict = {}

        # do analysis for one frame
        fits_df, dot_fits_dict, zstack, filtered_zstack, centroids, blobs_labels, _ = \
                        do_one_frame(filename,frame, img_info=img_info, load_file_path=filepath, \
                          return_intermediates=True,verbose = verbose, **this_channel_params_dict)

        # plot maximum projection of the z-stack
        max_proj = np.max(zstack,2)
        filt_max_proj = np.max(filtered_zstack,2)

        # save segmentation of images
        plt.figure(fig.number)
        plt.subplot(gs[count]); count += 1;
        plt.imshow(max_proj, vmin=np.percentile(max_proj,2),\
                   vmax=np.percentile(max_proj,99.5),cmap='coolwarm')
        plt.title('Channel {} Frame {} (Raw)'.format(channel,frame))
        plt.subplot(gs[count]); count += 1;
        plt.imshow(filt_max_proj, vmin=np.percentile(filt_max_proj,2),\
                   vmax=np.percentile(filt_max_proj,99.5),cmap='coolwarm')
        plt.title('(Filtered)'.format(channel,frame))
        plt.subplot(gs[count]); count += 1;
        plt.imshow(blobs_labels>0,vmax=1,cmap='gray')
        plt.title('(Localization)'.format(channel,frame))
        # superimpose the dots on the segmented image
        for dot_key, dot_dict in dot_fits_dict.items():
            plt.text(dot_dict['y0_fit'], dot_dict['x0_fit'],"{}".format(dot_key),color='y')
            #plt.plot(dot_dict['y0_fit'], dot_dict['x0_fit'],'o',markersize=3)

        ## show dots and fits
        numDots = len(dot_fits_dict)
        dot_fits_keys = [key for key in dot_fits_dict[0].keys() if 'max' in key]
        numKeys = len(dot_fits_keys)
        xwidth = 3
        ywidth = int(numKeys*numDots/3)

        fig_dots, gs_dots = _gridspec_inches(wcols = np.array([width_inches]*xwidth),\
                          hrows =np.array([width_inches]*ywidth),\
                         hspace=0.25,wspace=0.25)

        plt.figure(fig_dots.number)
        dot_subplot_count = 0
        for dot_id in dot_fits_dict.keys():
            dot_dict = dot_fits_dict[dot_id]
            for key in dot_fits_keys:
                plt.subplot(gs_dots[dot_subplot_count])
                if 'z' in key:
                    plt.imshow(dot_dict[key].T)
                else:
                    plt.imshow(dot_dict[key])
                if key in dot_dict['num_modes']:
                    plt.title("{}\n# dots found: {}".format(key,dot_dict['num_modes'][key]))
                else:
                    plt.title(key)
                if np.mod(dot_subplot_count,3)==0:
                    plt.ylabel("Dot ID: {}".format(dot_id))
                dot_subplot_count += 1

        if not figure_save_path is None:
            # make directory if it does not yet exist
            if not os.path.exists(figure_save_path):
                os.mkdir(figure_save_path)
            # save figure
            plt.figure(fig_dots.number)
            dot_figure_name = "{}_DotsFitPlots_Channel{}_Frame_{}.pdf" \
                                .format(filename[:-4],channel,frame)
            plt.savefig(os.path.join(figure_save_path,dot_figure_name), bbox_inches = "tight" )

    if not figure_save_path is None:
        # make directory if it does not yet exist
        if not os.path.exists(figure_save_path):
            os.mkdir(figure_save_path)
        # save figure
        plt.figure(fig.number) # call the correct figure
        dot_figure_name = "{}_LocaDotPlots.pdf" \
                            .format(filename[:-4],channel,frame)
        plt.savefig(os.path.join(figure_save_path,dot_figure_name), bbox_inches = "tight" )

    return fits_df

# internal helper function to help with plotting
def _gridspec_inches(
    wcols,
    hrows,
    wspace=0.75,
    hspace=0.5,
    fig_kwargs={}):

    fig = plt.figure()
    fig_height_inches = (
        sum(hrows)
        )

    fig_width_inches = (
        sum(wcols)
        )

    fig=plt.figure(
        figsize=(fig_width_inches,fig_height_inches),
        subplotpars=matplotlib.figure.SubplotParams(
        left=0,
        right=1,
        bottom=0,
        top=1,
        wspace =0,
        hspace = 0.0),
        **fig_kwargs)
    fig.set_size_inches(fig_width_inches,fig_height_inches,forward=True)

    gs = matplotlib.gridspec.GridSpec(
        len(hrows),
        len(wcols),
        left=0,
        right=1,
        top=1,
        bottom=0,
        wspace=wspace,
        hspace=hspace,
        width_ratios=wcols,
        height_ratios=hrows
        )
    return fig, gs

def count_dots_from_threshold(img,threshold_percentage=98,\
                                 min_object_area = 2,return_segmentation=False):
    """
    Segments images based on a simple threshold and returns the number of observed spots

    Parameters
    ----------

    img : numpy.ndarray
            Image on which to perform segmentation

    threshold_percentage : float
            Percentage threshold on the `img` intensity values used to binarize the image

    min_object_area : int
            Minimum number of pixels used to call/identify object

    return_segmentation : {T,F}
            If True, the function returns 1) the number of dots, 2) metrics about each dot.
            If False, the function only returns the number of dots

    Returns
    -------

    1) len(blobs_metrics) : int
            Number of dots identified from the image segmentation

    2) `blobs_metrics` : object
            Object from skimage.measure.regionprops() applied on the thresholded `img`.
    """

    blobs=  img>np.percentile(img,threshold_percentage)

    # filter objects based on size
    blobs = remove_small_objects(blobs, min_size=min_object_area)

    # remove objects touching the edges of the image
    blobs = clear_border(blobs)

    # get segmentation of the image from connected pixels
    blobs_labels = measure.label(blobs, background=0)

    # measure things for each unique feature identified in blobs_labels
    blob_metrics = measure.regionprops(blobs_labels, img)

    if return_segmentation == True:
        return len(blob_metrics), blobs
    else:
        return len(blob_metrics)

def log_zerocross(img,log_sigma=2,num_std_threshold=0):
    """
    Returns the edges detected by the Laplacian of Gaussian method applied on the image.

    Parameters
    ----------
    img : numpy.ndarray
            Image on which to perform the edge detection.

    log_sigma : float
            The standard deviation of the gaussian used in 'Laplacian of Gaussian'. It is
            akin to a smoothing parameter.

    num_std_threshold : float
            The number of standard deviations used above "zero" to classify a zero-crossing
            event of the 'Laplacian of Gaussian' as being an edge. Only set `num_std_threshold`
            greater than zero; also, only

    Returns
    -------
    log_img

    edges_img

    """

    log_img = gaussian_laplace(img, log_sigma)
    threshold = np.absolute(log_img).std() * num_std_threshold
    edges_img = np.zeros(img.shape)
    w = edges_img.shape[1]
    h = edges_img.shape[0]
    for y in range(1, h - 1):
        for x in range(1, w - 1):
            region = log_img[y-1:y+2, x-1:x+2]
            val = log_img[y, x]
            max_val = region.max()
            min_val = region.min()
            if (val > 0):
                zerocross = True if min_val < 0 else False
            else:
                zerocross = True if max_val > 0 else False
            if ((max_val - min_val) > threshold) and zerocross:
                edges_img[y, x] = 1
    return log_img, edges_img

def _get_weighted_centroid(zstack,xc,yc, win,dot_intensity_percentile,\
                           img_info, min_dot_size = 50, max_dot_size=500):
    """
    Parameters
    ----------
    (see )

    Output
    ------
    centerX, centerY, centerZ : float

    mean_intensity : float

    mean_surrounding : float

    sum_intensity :float

    dot_size_pixels : int
    """
    sizeX,sizeY,sizeZ,sizeT,num_channels = img_info
    # crop out the "dot" from the zstack
    dot_volume =  zstack[:,yc-win:yc+win,xc-win:xc+win]
    binary_volume = np.array((dot_volume >
                              np.percentile(dot_volume,dot_intensity_percentile))
                             ,dtype=int)

    # create a mask for the dot using connected pixels components on binarized volume
    blobs, num_features = scipy.ndimage.measurements.label(binary_volume)

    # filter out for size
    freq = np.bincount(np.ndarray.flatten(blobs))
    freq[freq>max_dot_size] = 0
    freq[freq<min_dot_size] = 0
    dot_label = np.argmax(freq)
    mask = blobs
    mask[mask != dot_label] = 0

    # calculate the centroid position
    weights = np.ndarray.flatten(mask*dot_volume)
    if np.sum(weights) == 0:
        return None # (if there are no dots, skip)
    else:
        # prepare meshgrid for finding centroid positions of the dots
        zz, yy, xx = np.meshgrid(range(zstack.shape[0]),
                                 np.arange(yc-win,yc+win),
                                 np.arange(xc-win,xc+win),indexing='ij')
        # calculate the centroid position
        centerX = np.average(np.ndarray.flatten(xx),weights=weights)
        centerY = np.average(np.ndarray.flatten(yy),weights=weights)
        centerZ = np.average(np.ndarray.flatten(zz),weights=weights)

    # calculate the total intensity/ mean intensity of the dot, and surrounding background
    mean_intensity = np.nanmean(dot_volume[mask!=0])
    mean_surrounding = np.nanmean(dot_volume[mask==0])
    sum_intensity = np.nansum(dot_volume[mask!=0])
    dot_size_pixels = np.count_nonzero(np.ndarray.flatten(mask))

    return centerX, centerY, centerZ, mean_intensity, mean_surrounding, sum_intensity, dot_size_pixels

def get_weighted_centroid_from_dot_volume(dot_volume,x_low,y_low,z_low, win,dot_intensity_percentile,\
                           img_info, min_dot_size = 50, max_dot_size=500):
    """
    Parameters
    ----------

    xc, yc, zc : float
        Real dot position in the Z-stack (not relative position)

    Output
    ------
    """
    sizeX,sizeY,sizeZ,sizeT,num_channels = img_info # metadata

    winZ, winY, winX = win # window size around localized dot for weighted average

    # crop out the "dot" from the zstack & render binary
    binary_volume = np.array((dot_volume >
                              np.percentile(dot_volume,dot_intensity_percentile))
                             ,dtype=int)

    # create a mask for the dot using connected pixels components on binarized volume
    blobs, num_features = scipy.ndimage.measurements.label(binary_volume)

    # filter out dots based on size
    freq = np.bincount(np.ndarray.flatten(blobs))
    freq[freq>max_dot_size] = 0
    freq[freq<min_dot_size] = 0
    dot_label = np.argmax(freq)
    mask = blobs
    mask[mask != dot_label] = 0

    # calculate the centroid position
    weights = np.ndarray.flatten(mask*dot_volume)
    if np.sum(weights) == 0:
        return None # (if there are no dots, skip)
    else:
        # prepare meshgrid for finding centroid positions of the dots
        zz, yy, xx = np.meshgrid(np.arange(z_low,z_low+winZ),
                                 np.arange(y_low,y_low+winY),
                                 np.arange(x_low,x_low+winX),indexing='ij')
        # calculate the centroid position
        centerX = np.average(np.ndarray.flatten(xx),weights=weights)
        centerY = np.average(np.ndarray.flatten(yy),weights=weights)
        centerZ = np.average(np.ndarray.flatten(zz),weights=weights)

    # calculate the total intensity/ mean intensity of the dot, and surrounding background
    mean_intensity = np.nanmean(dot_volume[mask!=0])
    mean_surrounding = np.nanmean(dot_volume[mask==0])
    sum_intensity = np.nansum(dot_volume[mask!=0])
    dot_size_pixels = np.count_nonzero(np.ndarray.flatten(mask))

    return (centerX, centerY, centerZ, mean_intensity, mean_surrounding, sum_intensity, dot_size_pixels)

import re
import itertools
def natural_sort(l):
    # found here: https://stackoverflow.com/questions/4836710/is-there-a-built-in-function-for-string-natural-sort
    convert = lambda text: int(text) if text.isdigit() else text.lower()
    alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', key) ]
    return sorted(l, key = alphanum_key)


from django.conf.urls import url
from django.contrib import admin
from . import views
from story import views
app_name = "story"

urlpatterns = [

    url(r'^$', views.AppView.as_view(), name = 'index'),
    url(r'^today/$', views.IndexView.as_view(), name = 'index-today'),
    url(r'^register/$',views.UserFormView.as_view(),name='register'),
    url(r'^(?P<pk>[0-9]+)/$', views.DetailView.as_view(), name = 'detail'),

    url(r'^login/', views.Login),
    url(r'^logout/', views.Logout,name="logout"),

    #story/add
    url(r'^add/$',views.StoryCreate.as_view(), name='add-story'),
    url(r'^edit/(?P<pk>[0-9]+)/$',views.StoryUpdate.as_view(), name='edit-story'),
    url(r'^delete/(?P<pk>[0-9]+)/$',views.StoryDelete.as_view(), name='delete-story'),

    #talks/
    url(r'^talks$', views.TalkIndexView.as_view(), name = 'talk-index'),
    url(r'^talks/(?P<pk>[0-9]+)/$', views.TalkDetailView.as_view(), name = 'talk-detail'),

    url(r'^talks/add/$',views.TalkCreate.as_view(), name='add-talk'),
    url(r'^talks/edit/(?P<pk>[0-9]+)/$',views.TalkUpdate.as_view(), name='edit-talk'),
    url(r'^talks/delete/(?P<pk>[0-9]+)/$',views.TalkDelete.as_view(), name='delete-delete'),

    #ngos/
    url(r'^ngos$', views.NgoIndexView.as_view(), name = 'ngo-index'),
    url(r'^ngos/(?P<pk>[0-9]+)/$', views.NgoDetailView.as_view(), name = 'ngo-detail'),

    url(r'^ngos/edit/(?P<pk>[0-9]+)/$',views.NgoUpdate.as_view(), name='edit-ngo'),
    url(r'^ngos/delete/(?P<pk>[0-9]+)/$',views.NgoDelete.as_view(), name='delete-ngo'),

    #sponsors/
   # url(r'^sponsors$', views.SponsorIndexView.as_view(), name = 'sponsor-index'),
   # url(r'^sponsors/(?P<pk>[0-9]+)/$', views.SponsorDetailView.as_view(), name = 'sponsor-detail'),

   # url(r'^sponsors/add/$',views.SponsoCreate.as_view(), name='add-sponsor'),
   # url(r'^sponsors/edit/(?P<pk>[0-9]+)/$',views.SponsorUpdate.as_view(), name='edit-sponsor'),
   # url(r'^sponsors/delete/(?P<pk>[0-9]+)/$',views.SponsorDelete.as_view(), name='delete-sponsor'),

    #market/
   # url(r'^products$', views.ProductIndexView.as_view(), name = 'product-index'),
   # url(r'^products/(?P<pk>[0-9]+)/$', views.ProductDetailView.as_view(), name = 'product-detail'),

   # url(r'^products/add/$',views.ProductCreate.as_view(), name='add-product'),
   # url(r'^products/edit/(?P<pk>[0-9]+)/$',views.ProductUpdate.as_view(), name='edit-product'),
   # url(r'^products/delete/(?P<pk>[0-9]+)/$',views.ProductDelete.as_view(), name='delete-product'),

]


import lxml.html
import requests

def refactor_extracted_comments(comments: list) -> str:
    """
    Refactors and cleans the extracted comments

    :param comments: comments being refactored
    :return: refactored comments
    """
    comments = "".join(comments)
    lines = comments.split("\n")
    cleaned_lines = [line.strip() for line in lines]

    refactored_lines = [line for line in cleaned_lines if line]
    refactored_text = "\n".join(refactored_lines).replace("\n\n", "\n").replace("reply", "\nreply:")

    return refactored_text

def extract_hackernews_page_content(url: str) -> dict:
    """
    Extracts the page content from the specified url

    :param url: url for which content is extracted
    :return: extracted page content
    """
    response = requests.get(url=url)

    page = lxml.html.fromstring(response.text)
    title_text = page.xpath("//span[@class='titleline']//a/text()")
    description_text = page.xpath("//div[@class='toptext']/text()")
    comments = page.xpath("//div[@class='comment']//descendant-or-self::*/text()")

    refactored_comments = refactor_extracted_comments(comments)

    return {
        "title": "".join(title_text),
        "description": "".join(description_text),
        "comments": refactored_comments
    }


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sonnet as snt
import tensorflow as tf

__author__ = 'Yasuhiro'
__date__ = '2018/3/21'

class WhiteEffectCountLayer(snt.AbstractModule):
    def __init__(self, name='white_effect_count'):
        super().__init__(name=name)

    def _build(self, short_effect, long_effect, ou_effect):
        """
        非手番側の利きがマスごとにいくつあるかを計算する

        :param short_effect:
        :param long_effect:
        :param ou_effect:
        :return:
        """
        white_effect = [tf.to_int32(effect)
                        for effect in short_effect + long_effect + ou_effect]
        # マスごとの利きの個数を計算
        count = tf.add_n(white_effect)

        return count


import torch.nn as nn
import torch

class model(nn.Module):
    def __init__(self):
        super(model,self).__init__()
        for i in range(4):
            setattr(self,"ReLU{}".format(i),nn.SiLU(inplace=True))

    def forward(self,x):
        for i in range(4):
            x=getattr(self,)

model1=model()
print(list(model1.modules()))


"""
Logging setup.
"""
import logging

from istub.constants import LOGGER_NAME

def setup_logging(level: int = 0) -> logging.Logger:
    """
    Get Logger instance.

    Arguments:
        level -- Logging level

    Returns:
        Created Logger.
    """
    logger = logging.getLogger(LOGGER_NAME)
    stream_handler = logging.StreamHandler()
    formatter = logging.Formatter("%(levelname)s %(message)s", datefmt="%H:%M:%S")
    stream_handler.setFormatter(formatter)
    stream_handler.setLevel(level)
    logger.addHandler(stream_handler)
    logger.setLevel(level)
    return logger


from json import loads
from urllib.request import urlopen
from re import (
    compile as re_compile,
    search as re_search
)
from hashlib import sha256

def http_get_dict(url: str) -> dict:
    return loads(urlopen(url, timeout=10).read())

def get_latest_alpine() -> str:
    data = http_get_dict(
        "https://hub.docker.com/v2/repositories/library/alpine/tags/?page_size=25&page=1"
    )

    latest_digest = None
    for image in data.get("results"):
        if image.get("name") == "latest":
            latest_digest = image.get("digest")
            break

    if latest_digest is None:
        exit("Faild to get latest alpine version")

    version = None
    for image in data.get("results"):
        if len(image.get("name").split(".")) == 3:
            if image.get("digest") == latest_digest:
                version = image.get("name")
                break

    if version is None:
        exit("Faild to get latest alpine version")

    return version

def get_latest_pip() -> dict:
    data = http_get_dict("https://pypi.org/pypi/pip/json")

    version = data.get("info").get("version")

    for package in data.get("releases").get(version):
        if package.get("packagetype") == "bdist_wheel":
            return {
                "version": version,
                "url": package.get("url"),
                "sha256": package.get("digests").get("sha256"),
                "filename": package.get("filename")
            }

    exit("Faild to get latest pip version")

def get_latest_pypy() -> dict:
    data = http_get_dict(
        "https://foss.heptapod.net/api/v4/projects/76/repository/tags")

    tag_list = []
    for tag in data:
        tag_list.append(tag.get("name"))

    tags = "\n".join(tag_list)
    version_pattern = re_compile(
        r"release-pypy\d\.\d-v(?P<version>\d*\.\d*\.\d*)"
    )

    result = re_search(version_pattern, tags)
    groups = result.groupdict()

    if groups.get("version") is None:
        exit("Faild to get latest pypy version")

    return groups.get("version")

def get_pypy_archive_sha256(python_version: str, pypy_version: str):
    return sha256(
        urlopen(
            f"https://foss.heptapod.net/pypy/pypy/-/archive/release-pypy{python_version}-v{pypy_version}/pypy-release-pypy{python_version}-v{pypy_version}.tar.bz2",
            timeout=25
        ).read()
    ).hexdigest()

def main() -> None:
    latest_pip = get_latest_pip()
    latest_pypy = get_latest_pypy()
    print(f"""
ALPINE_VERSION = "{get_latest_alpine()}"

pypy_version = "{latest_pypy}"
pypy2_7_sha256 = "{get_pypy_archive_sha256("2.7", latest_pypy)}"
pypy3_8_sha256 = "{get_pypy_archive_sha256("3.8", "7.3.11")}"
pypy3_9_sha256 = "{get_pypy_archive_sha256("3.9", latest_pypy)}"
pypy3_10_sha256 = "{get_pypy_archive_sha256("3.10", latest_pypy)}"

python2_pip_version = "20.3.4"
python2_pip_sha256 = "217ae5161a0e08c0fb873858806e3478c9775caffce5168b50ec885e358c199d"
python2_pip_whl_name = "pip-20.3.4-py2.py3-none-any.whl"
python2_pip_downlaod_url = "https://files.pythonhosted.org/packages/27/79/8a850fe3496446ff0d584327ae44e7500daf6764ca1a382d2d02789accf7/pip-20.3.4-py2.py3-none-any.whl"

python3_pip_version = "{latest_pip.get("version")}"
python3_pip_sha256 = "{latest_pip.get("sha256")}"
python3_pip_whl_name = "{latest_pip.get("filename")}"
python3_pip_downlaod_url = "{latest_pip.get("url")}"

pycparser_version = "2.21"
pycparser_whl_name = "pycparser-2.21-py2.py3-none-any.whl"
pycparser_sha256 = "8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"
pycparser_downlaod_url = "https://files.pythonhosted.org/packages/62/d5/5f610ebe421e85889f2e55e33b7f9a6795bd982198517d912eb1c76e1a53/pycparser-2.21-py2.py3-none-any.whl"
""")

if __name__ == "__main__":
    main()


import random
from colorama import init, Back
from english_words import english_words_alpha_set

words = [word.upper() for word in english_words_alpha_set if len(word) == 5]
init(autoreset=True)

class Wordle:

    def __init__(self):
        self.tries = 0
        self.board = ['_____'] * 6
        self.answer = random.choice(words)
        self.alphabet = {char : Back.RESET for char in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'}
        self.output_board()
        self.run_game()

    def run_game(self):
        while self.tries < 6:
            self.make_guess()
            print(chr(27) + "[2J")
            self.output_board()
        print('Sorry, try again!\nThe word was ' + self.answer)
        exit(0)

    def output_board(self):
        for line in self.board:
            print(line)
        for key in self.alphabet:
            print(self.alphabet[key] + key, end = '')
        print()

    def make_guess(self):
        guess = input('Make a guess: ').upper()
        guess_cpy = guess
        if guess not in words:
            print(guess + ' is not a valid 5-letter word.')
            return
        # elif guess == self.answer:
        #     guess = Back.GREEN + guess
        #     self.board[self.tries] = guess
        #     print(chr(27) + "[2J")
        #     self.output_board()
        #     print('Correct, congratulations!')
        #     exit(0)
        already_used = []
        for guess_char, answer_char in zip(guess, self.answer):
            if guess_char == answer_char:
                guess = guess.replace(guess_char, Back.GREEN + guess_char, 1)
                self.alphabet[guess_char] = Back.GREEN # self.alphabet.replace(guess_char, Back.GREEN + guess_char, 1)
            elif guess_char in self.answer:
                guess = guess.replace(guess_char, Back.YELLOW + guess_char, 1)
                self.alphabet[guess_char] = Back.YELLOW #= self.alphabet.replace(guess_char, Back.YELLOW + guess_char, 1)
            else:
                if guess_char in already_used:
                    continue
                already_used.append(guess_char)
                guess = guess.replace(guess_char, Back.RESET + guess_char, 2)
                self.alphabet.pop(guess_char, 'No value')
        if guess_cpy == self.answer:
            guess = Back.GREEN + guess
            self.board[self.tries] = guess
            print(chr(27) + "[2J")
            self.output_board()
            print('Correct, congratulations!')
            exit(0)
        self.board[self.tries] = guess
        self.tries += 1

def main():
    wordle = Wordle()

main()


class Solution(object):
    def superEggDrop(self, K, N):
        """
        :type K: int
        :type N: int
        :rtype: int
        """
        dp = [[0 for _ in range(N+1)] for _ in range(K+1)]

        for i in range(1,K+1):
            for j in range(1,N+1):
                dp[i][j] = dp[i - 1][j - 1] + (dp[i][j - 1] + 1)
                if dp[i][j] >=N:
                    return j
        return 0
        # dp = [[0 for _ in range(N + 1)] for _ in range(K + 1)]
        # for i in range(1, K + 1):
        #     for step in range(1, N + 1):
        #         dp[i][step] = dp[i - 1][step - 1] + (dp[i][step - 1] + 1)
        #         print(dp[i][step])
        #         if dp[K][step] >= N:
        #             return step
        # return 0

print(Solution().superEggDrop(2,6))


#!/usr/bin/env python

import rospy
import tf
import numpy as np
from sensor_msgs.msg import LaserScan
from nav_msgs.msg import OccupancyGrid
from geometry_msgs.msg import Pose, Point, Quaternion
from visualization_msgs.msg import Marker, MarkerArray
from dynamic_reconfigure.server import Server
from test.cfg import DynamicParamConfig

class LaserToLocalGrid:

    def __init__(self):
        rospy.init_node('laser_to_local_grid')

        self.frame_id = rospy.get_param('~frame_id', '/robot0')

        # Occupancy grid parameters
        self.resolution = rospy.get_param('~resolution', 0.05)  # in meters
        self.width = rospy.get_param('~width', 10)  # in meters
        self.height = rospy.get_param('~height', 10)  # in meters

        self.noise = rospy.get_param('~noise', True)

        # Nose parameters
        self.a = rospy.get_param('~a', -106.99)
        self.b = rospy.get_param('~b', 0.336)
        self.c = rospy.get_param('~c', -0.000256)

        self.grid_pub = rospy.Publisher(self.frame_id + '/local_grid', OccupancyGrid, queue_size=1)
        rospy.Subscriber(self.frame_id + '/laser_0', LaserScan, self.scan_callback)

        self.reconfigure_server = Server(DynamicParamConfig, callback=self.server_callback)

        self.noise_threshold = 0.25

    def server_callback(self, config, level):
        # copy the values from the config to instance
        for key in config.keys():
            setattr(self, key, config[key])

        return config

    def polar_to_cartesian(self, angle, radius):
        x = radius * np.cos(angle)
        y = radius * np.sin(angle)
        return x, y

    def create_cube_marker(self, position, index, color=(1.0, 1.0, 0.0, 1.0)):
        marker = Marker()
        marker.header.frame_id = self.frame_id
        marker.type = Marker.CUBE
        marker.action = Marker.ADD
        marker.scale.x = self.resolution
        marker.scale.y = self.resolution
        marker.scale.z = 0.1
        marker.color.r, marker.color.g, marker.color.b, marker.color.a = color
        marker.pose.position.x, marker.pose.position.y = position
        marker.pose.position.z = 0.1
        marker.pose.orientation.x = 0.0
        marker.pose.orientation.y = 0.0
        marker.pose.orientation.z = 0.0
        marker.pose.orientation.w = 1.0
        marker.id = index
        return marker

    def create_occupancy_grid(self):
        grid = OccupancyGrid()
        grid.header.stamp = rospy.Time.now()
        grid.header.frame_id = self.frame_id
        grid.info.resolution = self.resolution
        grid.info.width = int(self.width / self.resolution)
        grid.info.height = int(self.height / self.resolution)
        grid.info.origin = Pose(Point(-self.width / 2, -self.height / 2, 0), Quaternion(0, 0, 0, 1))
        grid.data = np.zeros((grid.info.width, grid.info.height), dtype=np.int8).flatten().tolist()
        return grid

    def get_grid(self, ranges, angles):
        grid = self.create_occupancy_grid()
        for idx, (angle, range_value) in enumerate(zip(angles, ranges)):
            # if self.noise:
            #     range_value = self.mixed_pixel_effect(ranges, idx)

            if np.isinf(range_value) or np.isnan(range_value):
                continue

            x, y = self.polar_to_cartesian(angle, range_value)
            i = int((x - grid.info.origin.position.x) / self.resolution)
            j = int((y - grid.info.origin.position.y) / self.resolution)

            if 0 <= i < grid.info.width and 0 <= j < grid.info.height:
                index = j *  grid.info.width + i
                grid.data[index] = 100
        return grid

    def mixed_pixel_effect(self, distances, i):
        beam_width = 0.1  # A parameter representing the width of the LiDAR beam
        weights = [np.exp(-0.5 * (angle / beam_width) ** 2) for angle in range(-2, 3)]
        weights /= sum(weights)  # Normalize to sum to 1

        # Convolve distances with beam effect
        neighbors = distances[max(0, i - 2):min(len(distances), i + 3)]
        mixed_distance = sum(w * d for w, d in zip(weights, neighbors))
        return mixed_distance

    def scan_callback(self, scan):
        # Extract distances
        distances = np.array(scan.ranges)

        # Compute variance for each distance

        variances = self.a + self.b * distances + self.c * distances**2

        # Simulate noise
        if self.noise:
            noisy_distances = []
            for distance, variance in zip(distances, variances):
                if np.isinf(distance) or np.isnan(distance):
                    noisy_distances.append(distance)
                    continue
                noise = np.random.normal(loc=0, scale=np.sqrt(np.abs(variance)))
                noisy_distances.append((distance * 1000 + noise) / 1000)  # in meters
            distances = noisy_distances

        angles = np.arange(scan.angle_min, scan.angle_max + scan.angle_increment, scan.angle_increment)

        self.grid_pub.publish(self.get_grid(distances, angles))

if __name__ == '__main__':
    laser_to_local_grid = LaserToLocalGrid()
    rospy.spin()


import random
import sys
import heapq
import time

class Vertice:
	"""Representa un vertice con operaciones ver adyacentes y agregar adyacentes."""

	def __init__(self, iden, label):
		"""Crea un vertice."""
		self.iden = iden
		self.label = label
		self.dic_ady = {}

	def adyacentes(self):
		"""Devuelve un diccionario conteniendo a todos los adyacentes a un vertice."""
		return self.dic_ady

	def agregar_adyacente(self, vertice):
		"""Agrega adyacencia entre dos vertices."""
		if not vertice.iden in self.dic_ady:
			self.dic_ady[vertice.iden] = vertice
			return True

class Grafo:
	"""Representa un grafo con operaciones agregar y quitar un vertice, agregar y quitar
	una arista, obtener adyacencias, verificar si dos vertices son adyacentes, verificar
	 si un vertice existe,obtener la cantidad de vertices, obtener los identificadores
	  del grafo e iterar.."""

	def __init__(self):
		"""Crea una grafo vacío."""
		self.vertices = {}
		self.cantidad_vert = 0
		self.cantidad_aris = 0

	def obtener_dic_iden(self):
		"""Devuelve un diccionario con los vertices del Grafo"""
		return self.vertices

	def obtener_vertice(self, iden):
		"""Devuelve el Vertice correspondiente al Identificador"""
		return self.vertices.get(iden, -1)

	def agregar_vertice(self, vertice):
		"""Agrega un vertice al grafo."""
		if not vertice.iden in self.vertices:
			self.vertices[vertice.iden] = vertice
			self.cantidad_vert += 1
			return True

	def quitar_vertice(self,vertice):
		"""Quita un vertice del grafo"""
		if not vertice in self.vertices:
			return False
		for adyacente in vertice.adyacentes:
			self.vertices[adyacente].adyacentes.pop(vertice.iden)
		self.vertices.pop(vertice.iden)
		self.cantidad_vertices -= 1
		return True

	def cantidad_vertices(self):
		"""Devuelve la cantidad de vertices del grafo."""
		return self.cantidad_vert

	def agregar_arista(self, vertice, vertice_2):
		"""Agrega una arista entre dos vertices."""
		if vertice.iden in self.vertices:
			vertice = self.vertices.get(vertice.iden, -1)
		if vertice_2.iden in self.vertices:
			vertice_2 = self.vertices.get(vertice_2.iden, -1)
		if vertice.agregar_adyacente(vertice_2) and vertice_2.agregar_adyacente(vertice):
			self.cantidad_aris += 1

	def quitar_arista(self,vertice,vertice_2):
		"""Quita una arista entre dos vertices."""
		if not vertice_2 in vertice.adyacentes:
			return False
		vertice.adyacentes.pop(vertice_2.iden)
		vertice_2.adyacentes.pop(vertice.iden)
		self.cantidad_aris -= 1
		return True

	def cantidad_aristas(self):
		"""Devuelve la cantidad de aristas del grafo"""
		return self.cantidad_aris

	def verificar_conexion(self,vertice,vertice_2):
		"""Verifica si dos vertices están conectados."""
		return vertice_2 in vertice.adyacentes()

	def adyacentes_vertice(self, vertice):
		"""Devuelve un diccionario conteniendo todos los vertices adyacentes a uno."""
		return vertice.adyacentes()

	def vertice_existe(self, vertice):
		"""Verifica si un vertice existe en un grafo."""
		return vertice in self.vertices

	def obtener_identificadores(self):
		"""Devuelve todos los identificadores del grafo."""
		return list(self.vertices.keys())

	def obtener_vertices(self):
		"""Devuelve todos los identificadores del grafo."""
		return list(self.vertices.values())

class Nodo:
	"""Representa un nodo."""

	def __init__(self, dato=None, prox=None):
		self.dato = dato
		self.prox = prox

	def __str__(self):
		return str(self.dato)

class Cola:
	"""Representa una cola con funciones, encolar, desencolar y está vacia."""

	def __init__(self):
		"""Crea una cola vacía."""
		self.primero = None
		self.ultimo = None

	def encolar(self, x):
		"""Encola el elemento x."""
		nuevo = Nodo(x)
		if self.ultimo is not None:
			self.ultimo.prox = nuevo
			self.ultimo = nuevo
		else:
			self.primero = nuevo
			self.ultimo = nuevo

	def desencolar(self):
		"""Desencola el primer elemento y devuelve su valor. Si la cola está vacía, levanta ValueError."""
		if self.primero is None:
			raise ValueError("La cola está vacía")
		valor = self.primero.dato
		self.primero = self.primero.prox
		if not self.primero:
			self.ultimo = None
		return valor

	def esta_vacia(self):
		"""Devuelve True si la cola esta vacía, False si no."""
		return self.primero is None

##########################################################################################
##########################################################################################
##########################################################################################

def procesar_archivo(grafo, archivo):
	"""Función que abre el archivo y linea por linea va generando vertices y aristas."""
	lineas_de_cabecera = 4
	try:
		with open(archivo, "r") as archivo:
			i = 0
			for linea in archivo:
				if (i <= lineas_de_cabecera):
					i += 1
					continue
				id_1, id_2 = linea.rstrip("\n").split("\t")
				vertice_1 = Vertice(id_1, id_1)
				vertice_2 = Vertice(id_2, id_2)
				grafo.agregar_vertice(vertice_1)
				grafo.agregar_vertice(vertice_2)
				grafo.agregar_arista(vertice_1, vertice_2)
	except FileNotFoundError:
		print("El archivo no fue creado aún.")
		return False
	except IOError:
		print("Error al intentar abrir o guardar el archivo.")
		return False
	return True

def generar_grafo(archivo):
	"""Función que crea un grafo y le agrega todos los vertices y aristas."""
	grafo = Grafo()
	if not procesar_archivo(grafo, archivo):
		return False
	return grafo

def get_rand_vert(lista):
	"""Devuelve una clave aleatoria del diccionario.
	Pre: El diccionario no esta vacio"""
	posicion = random.randint(0, len(lista)-1)
	return lista[posicion]

def validar_vertice(vertice, grafo):
	"""Verifica si el vértice se encuentra en el grafo."""
	if vertice == -1:
		print("El vertice no se encuentra en el grafo.")
		return False
	return True

def validar_cantidad(cantidad, grafo):
	"""Verifica que el numero pasado por parametro esté dentro del rango del grafo."""
	return cantidad < grafo.cantidad_vertices()

def verificar_elecciones(vertice, n, grafo):
	"""Verifica las elecciones pasadas por parametro."""
	if not validar_cantidad(n, grafo):
		print("Ingreso una cantidad mayor a la del grafo.")
		return False
	return validar_vertice(vertice, grafo)

def establecer_cantidad(n, grafo):
	"""Establece la cantidad de random walks a hacer según el numero pasado por parametro."""
	cantidad = n*100
	if not validar_cantidad(cantidad, grafo):
		cantidad = grafo.cantidad_vertices()
	return cantidad

def random_walk(largo, cantidad, vertice, grafo):
	"""Genera un random walk."""
	if not validar_cantidad(largo, grafo):
		return -1
	apariciones = {}
	v = vertice
	for c in range(cantidad):
		i = 0
		while i < largo:
			w = get_rand_vert(list(grafo.adyacentes_vertice(v).values()))
			cant = apariciones.get(w.iden, 0)
			apariciones[w.iden] = cant+1
			i += 1
			v = w
	return apariciones

def crear_heap_menores(diccionario, largo):
	"""Genera un heap de menores de largo dado lleno de vertices similares
	al recibido por parametro."""
	heap_min = []
	heapq.heapify(heap_min)
	for vertice in list(diccionario.keys()):
		tupla = (diccionario[vertice], vertice)
		if len(heap_min) < largo:
			heapq.heappush(heap_min, tupla)
		else:
			if heap_min[0][0] < tupla[0]:
				heapq.heapreplace(heap_min,tupla)
	return heap_min

def recorrido_BFS(grafo, origen, destino):
	"""Hace recorrido BFS desde un origen hasta un destino si se desea agregarle."""
	visitados = {}
	padre = {}
	orden = {}
	padre[origen] = None
	orden[origen] = 0
	q = Cola()
	q.encolar(origen)
	visitados[origen] = True
	while not q.esta_vacia():
		v = q.desencolar()
		vertice = grafo.obtener_vertice(v)
		for w in grafo.adyacentes_vertice(vertice):
			if w not in visitados:
				visitados[w] = True
				padre[w] = v
				orden[w] = orden[v] + 1
				if(destino != None ):
					if(w == destino):
						break;
				q.encolar(w)
	return padre, orden

def verificar_parametros(comando, ingreso):
	"""Verifica que los parametros ingresados por el usuario sean correctos."""
	if len(ingreso) == 0 or len(ingreso) > 3:
		print("Ingresó una cantidad incorrecta de parametros.")
		return False
	if (comando == 0):
		print("El comando ingresado es incorrecto.")
		return False
	if (comando == 1 or comando == 2 or comando == 3):
		if (len(ingreso) != 3):
			print("Ingresó una cantidad incorrecta de parametros.")
			return False
	if (comando == 4 or comando == 5):
		if (len(ingreso) != 2):
			print("Ingresó una cantidad incorrecta de parametros.")
			return False
	if (comando == 6 or comando == 7 or comando == 8):
		if (len(ingreso) != 1):
			print("Ingresó una cantidad incorrecta de parametros.")
			return False
	return True

def menu(grafo):
	"""Corre el menú que interactúa con el usuario."""
	opciones = {"similares":1, "recomendar":2, "camino":3, "centralidad":4, "distancias":5, "estadisticas":6, "comunidades":7, "s":8}
	while True:
		ingreso = input("Ingrese el comando deseado o 'S' para salir: ").split(" ")
		comando = opciones.get(ingreso[0].lower(), 0)
		if not verificar_parametros(comando, ingreso):
			menu(grafo)
		for i in range (len(ingreso)-1, 0, -1):
			parametro = ingreso[i]
			if not parametro.isdigit():
				print("Los parametros deben ser dígitos.")
				menu(grafo)
		if(comando == 1):
			similares(ingreso[1], int(ingreso[2]), grafo)
		if(comando == 2):
			recomendar(ingreso[1], int(ingreso[2]), grafo)
		if(comando == 3):
			camino(ingreso[1], ingreso[2], grafo)
		if(comando == 4):
			centralidad(grafo, int(ingreso[1]))
		if(comando == 5):
			distancia(grafo, ingreso[1])
		if(comando == 6):
			estadisticas(grafo)
		if(comando == 7):
			ini = time.time()
			comunidades(grafo)
			fin = time.time()
			print("Tiempo:", (fin-ini))
		if(comando == 8):
			sys.exit()

##########################################################################################

def similares(iden, n, grafo):
	"""Dado un usuario, encontrar los personajes más similares a este."""
	LARGO_RAN_WALKS = 30
	vertice = grafo.obtener_vertice(iden)
	if not verificar_elecciones(vertice, n, grafo):
		return False
	cantidad = establecer_cantidad(n, grafo)
	apariciones = random_walk(cantidad, LARGO_RAN_WALKS, vertice, grafo)
	heap_min = crear_heap_menores(apariciones, n)
	for i in range(len(heap_min)):
		print("{} ".format(heap_min[i][1]),end=" ")
	print("\n")

def recomendar(iden, n, grafo):
	"""Dado un usuario, recomienda otro (u otros) usuario con el cual aún no tenga relación,
	 y sea lo más similar a él posible. Si la cantidad de similares < n, imprime la cantidad maxima."""
	LARGO_RAN_WALKS = 30
	vertice = grafo.obtener_vertice(iden)
	if not verificar_elecciones(vertice, n, grafo):
		return False
	cantidad = n*100
	cantidad = establecer_cantidad(n, grafo)
	apariciones = random_walk(cantidad, LARGO_RAN_WALKS, vertice, grafo)
	heap_min = crear_heap_menores(apariciones, n*2)
	i = 0
	lista = []
	while(i < n):
		cant, vertice_aux = heapq.heappop(heap_min)
		if not vertice_aux in list(grafo.adyacentes_vertice(vertice).keys()):
			lista.append(vertice_aux)
			i += 1
	for c in range(len(lista)-1,-1,-1):
		print("{} ".format(lista[(c)]),end=" ")
	print("\n")

def camino(id_1, id_2, grafo):
	"""Busca el camino mas corto entre dos vertices."""
	vertice = grafo.obtener_vertice(id_1)
	vertice_2 = grafo.obtener_vertice(id_2)
	if vertice == -1 or vertice_2 == -1:
		print("Algún vertice no se encuentra en el grafo.")
		return False
	padre, orden = recorrido_BFS(grafo, id_1, id_2)
	camino = []
	v = id_2
	while padre[v] != None:
		camino.append(v)
		v = padre[v]
	camino.append(id_1)
	print("Camino: ",end="")
	for i in range(len(camino)-1):
		print("{}->".format(camino[len(camino)-1-i]),end="")
	print("{}".format(id_2))

def centralidad(grafo, n):
	"""Obtiene lo "n" vertices con mayor influencia del grafo utilizando random walks."""
	CANT_RAN_WALKS = 10
	LARGO_RAN_WALKS = 5
	FACTOR_DE_CENTRALIDAD = 50
	globales = {}
	for i in range(n*FACTOR_DE_CENTRALIDAD):
		vertice = random.choice(grafo.obtener_vertices())
		apariciones = random_walk(CANT_RAN_WALKS, LARGO_RAN_WALKS, vertice, grafo)
		for vertice in apariciones.keys():
			apariciones = globales.get(vertice, 0) + 1
			globales[vertice] = apariciones
	heap = crear_heap_menores(globales, n)
	for i in range(len(heap)):
		print("{} ".format(heap[i][1]),end=" ")
	print("\n")

def distancia(grafo, iden):
	"""Obtiene la cantidad de elementos que hay para cada distancia del vertice ingresado."""
	padre, orden = recorrido_BFS(grafo, iden, None)
	distancias = {}
	for vertice in orden.keys():
		if(orden[vertice] == 0):
			continue
		if not orden[vertice] in distancias:
			distancias[orden[vertice]] = []
		distancias[orden[vertice]].append(vertice)
	lista = list(distancias.keys())
	lista.sort()
	for i, elemento in enumerate(lista):
		print("Distancia {}: {}".format(elemento, len(distancias[elemento])))

def estadisticas(grafo):
	"""Imprime por pantalla las estadisticas del grafo."""
	acumulador = 0
	for vertice in grafo.obtener_vertices():
		acumulador += len(grafo.adyacentes_vertice(vertice))
	promedio = acumulador/grafo.cantidad_vertices()
	densidad = grafo.cantidad_vertices()/grafo.cantidad_aristas()
	print("Estadisticas:")
	print("Cantidad de Vertices: {} ".format(grafo.cantidad_vertices()))
	print("Cantidad de Aristas: {}".format(grafo.cantidad_aristas()))
	print("Densidad del Grafo: {} ".format(densidad))
	print("Promedio de grado de entrada de cada vértice: {}".format(promedio))

def comunidades(grafo):
	comunidades_d = {}
	for i in range(5):
		for vertice in grafo.obtener_vertices():
			labels = {}
			mas_aparece = 0
			label_mas_presente = 0
			for adyacente in list(grafo.adyacentes_vertice(vertice).values()):
				apariciones = labels.get(adyacente.label, 0)
				labels[adyacente.label] = apariciones + 1
			for label in labels:
				if labels[label] > mas_aparece:
					mas_aparece = labels[label]
					label_mas_presente = label
			vertice.label = label_mas_presente

	comunidades_d = {}
	for vertice in grafo.obtener_vertices():
		lista = comunidades_d.get(vertice.label, [])
		lista.append(vertice.iden)
		comunidades_d[vertice.label] = lista

	cantidad = 0
	for label in comunidades_d:
		if len(comunidades_d[label]) <= 4 or len(comunidades_d[label]) > 2000:
			continue
		cantidad+=1
		print("Comunidad {}, {} integrantes: {}".format(label, len(comunidades_d[label]),comunidades_d[label]))
	print("Cantidad de comunidades: {}".format(cantidad))

def main():
	"""Función que corre el programa."""
	archivo = input("Ingrese el nombre del archivo: ")
	print("Generando grafo.")
	grafo = generar_grafo(archivo)
	print("Grafo generado.")
	if not grafo:
		sys.exit()
	menu(grafo)

main()


import pandas as pd
import numpy as np
import cv2
import os
from sklearn.model_selection import train_test_split
from sklearn.utils import shuffle
from keras.callbacks import EarlyStopping, ModelCheckpoint
from keras.layers import Convolution2D, MaxPooling2D, Dense, Flatten, Dropout, Lambda
from keras.models import Sequential
from keras.optimizers import Adam

def load_data(data_folder, correction):

  data = pd.read_csv(os.path.join(data_folder, "driving_log.csv"))

  # Get the file paths from csv and extract the filename only
  X = data[['center', 'left', 'right']].values
  # Get the steering values and adjust them for the left and right images
  y = data['steering'].values

  # Get indices of training and validation data
  train_ind, valid_ind = train_test_split(range(X.shape[0]), test_size=0.2, random_state=42)

  # For training use center, left, and right images
  X_train = np.array([fname.replace("\\", "/").split("/")[-1] for fname in X[train_ind].flatten(order="F")])
  y_train = np.hstack((y[train_ind], y[train_ind]+correction, y[train_ind]-correction))
  # For validation only use center images
  X_valid = np.array([fname.replace("\\", "/").split("/")[-1] for fname in X[valid_ind][:,0].flatten(order="F")])
  y_valid = y[valid_ind]

  return X_train, X_valid, y_train, y_valid

def read_img(file_name):
  # Read in image as BGR format
  img = cv2.cvtColor(cv2.imread(file_name), cv2.COLOR_BGR2RGB)
  return img

def preprocess_img(img):
  # Remove the sky and the car
  crop = img[65:-25, :, :]
  # Resize to fit for Nvidia model and change color space accordingly
  resize = cv2.resize(crop, (200, 66))
  color = cv2.cvtColor(resize, cv2.COLOR_RGB2YUV)
  return color

def flip_img(img, steering):
  # Flip image horizontally and adjust steering angle
  flip = cv2.flip(img, 1)
  angle = -steering
  return flip, angle

def data_generator(img_folder, X_data, y_data, batch_size=128, is_training=True):
  num_samples = len(X_data)
  while 1:
    # Shuffle the data
    X_data, y_data = shuffle(X_data, y_data)
    # Loop over the dataset with batch size step
    for offset in range(0, num_samples, batch_size):
      X_batch = X_data[offset:offset+batch_size]
      y_batch = y_data[offset:offset+batch_size]

      images = []
      angles = []

      # Loop over each sample in teh batch
      for fname, angle in zip(X_batch, y_batch):
        img = read_img(os.path.join(img_folder, fname))
        ang = angle
        # If training randomly augment the image by flipping it
        if is_training and np.random.rand() < 0.5:
          img, ang = flip_img(img, angle)
        # Preprocess data for the model
        img = preprocess_img(img)
        images.append(img)
        angles.append(ang)

      X = np.array(images)
      y = np.array(angles)

      yield shuffle(X, y)

def build_model():
  # Build the Nvidia model with small adaptions
  model = Sequential()
  model.add(Lambda(lambda x: x/255.0-0.5, input_shape=(66,200,3)))
  model.add(Convolution2D(24, 5, 5, activation='elu', subsample=(2, 2), init="he_normal"))
  model.add(Convolution2D(36, 5, 5, activation='elu', subsample=(2, 2), init="he_normal"))
  model.add(Convolution2D(48, 5, 5, activation='elu', subsample=(2, 2), init="he_normal"))
  model.add(Convolution2D(64, 3, 3, activation='elu', init="he_normal"))
  model.add(Convolution2D(64, 3, 3, activation='elu', init="he_normal"))
  model.add(Dropout(0.5))
  model.add(Flatten())
  model.add(Dense(100, activation='elu', init="he_normal"))
  model.add(Dense(50, activation='elu', init="he_normal"))
  model.add(Dense(10, activation='elu', init="he_normal"))
  model.add(Dense(1, init="he_normal"))

  return model

def train_model(model, X_train, X_valid, y_train, y_valid, batch_size, epochs, learning_rate, samples_per_epoch, img_folder):
  # Create callbacks for saving checkpoints and early stopping
  checkpoint = ModelCheckpoint("model.h5", save_best_only=True)
  early_stopping = EarlyStopping(min_delta=0.001, patience=3)

  model.compile(loss='mse', optimizer=Adam(lr=learning_rate))
  # Number of batches to yield from generator for one epoch
  train_steps = samples_per_epoch
  valid_steps = (X_valid.shape[0]//batch_size)*batch_size
  # Train the model
  model.fit_generator(data_generator(img_folder, X_train, y_train, batch_size, True), train_steps, max_q_size=1,
                      nb_epoch=epochs, validation_data=data_generator(img_folder, X_valid, y_valid, batch_size, False),
                      nb_val_samples=valid_steps, callbacks=[checkpoint])

def main():
  data_folder = os.path.join(os.path.curdir, "data")
  img_folder = os.path.join(data_folder, "IMG")
  batch_size = 256
  learning_rate = 0.0001
  correction = 0.2
  epochs = 10
  samples_per_epoch = 25600

  X_train, X_valid, y_train, y_valid = load_data(data_folder, correction)
  model = build_model()
  train_model(model, X_train, X_valid, y_train, y_valid, batch_size, epochs, learning_rate, samples_per_epoch, img_folder)

if __name__ == "__main__":
  main()


# users/朔/PycharmProjects
# -*- coding:utf-8 -*-
# author:YQRY time:2018/9/29
import random
secret = random.randint(1, 10)
print('猜数小游戏')
temp = input('猜我想的是那个数：')
guess = int(temp)
while guess != secret:
    temp = input('猜错了，再猜：')
    guess = int(temp)
    if guess == secret:
        print('牛逼！猜对了！')
    else:
        if guess > secret:
            print('大了')
        else:
            print('小了')
print('游戏结束！')


from unittest.mock import MagicMock, patch

from django.urls import reverse
from rest_framework import status
from rest_framework.test import APITestCase

from graphs import graphs
from dasktasks.daskmanager import DaskManager
from dasktasks.models import DaskTask
from oaze.models import CsvDocument
from oaze import views

class CsvDocumentTests(APITestCase):
    csv_path = 'testfiles/1000.csv'

    def test_create(self):
        task = DaskTask(task_key='123')
        task.save()

        compute_mock = MagicMock()
        compute_mock.compute.return_value = task
        views.DaskManager = MagicMock(return_value=compute_mock)
        views.sum_csv_graph = MagicMock()
        url = reverse('csvdocument-list')
        with open(self.csv_path, 'rb') as image_data:
            data = {'file': image_data}
            response = self.client.post(url, data, format='multipart')
        self.assertEqual(response.status_code, status.HTTP_201_CREATED)
        new_document = CsvDocument.objects.get(id=response.data['id'])
        views.sum_csv_graph.assert_called_with(new_document.file.url)

    def test_create_no_file(self):
        url = reverse('csvdocument-list')
        data = {}
        response = self.client.post(url, data, format='multipart')
        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)


import numpy as np
import csv
import matplotlib.pyplot as plt

def plot_learning_curves(x, scores, goals, lengths, optim, figure_file):
    running_avg = np.zeros(len(scores))
    for i in range(len(running_avg)):
        running_avg[i] = np.mean(scores[max(0, i-100):(i+1)])
    plt.plot(x, running_avg)
    plt.xlabel('Episodes')
    plt.ylabel('Cost')
    plt.title('Running average of previous 100 scores')
    plt.savefig(figure_file + '_score.pdf')
    plt.close()
    for i in range(len(running_avg)):
        running_avg[i] = np.sum(goals[max(0, i-100):(i+1)])
    plt.plot(x, running_avg)
    plt.xlabel('Episodes')
    plt.ylabel('Goals')
    plt.title('Running average of previous 100 goals')
    plt.savefig(figure_file + '_goals.pdf')
    plt.close()
    optim = np.array(optim)
    lengths = np.array(lengths)
    ratio = optim/lengths
    ratio[np.array(goals)==0] = 0
    for i in range(len(running_avg)):
        running_avg[i] = np.mean(ratio[max(0, i-100):(i+1)])
    plt.plot(x, running_avg)
    plt.xlabel('Episodes')
    plt.ylabel('Normalized score')
    plt.yticks([0,1])
    plt.title('Running average of normalized cost')
    plt.savefig(figure_file + '_normalized_score.pdf')
    plt.close()

def save_data(scores, goals, lengths, opts, data_file):
    file = open(data_file, 'w', newline='')
    writer = csv.writer(file)
    writer.writerow(['score', 'avg_score', 'goal', 'avg_goals', 'path_length',
                    'optimal_length', 'normalized_score'])
    for w in range(len(scores)):
        writer.writerow([scores[w], np.mean(scores[max(0, w-100):(w+1)]),
                    goals[w], np.sum(goals[max(0, w-100):(w+1)]), lengths[w],
                    opts[w], opts[w]/lengths[w] if goals[w] else 0])
    file.close()