import numpy as np
import os, time, shutil
import imageio
import cv2
# import pcnn_util as pu
from pcnn_util import *

class pcnn:
    '''
    A class implementing various methods using pulse-coupled neural networks (PCNN)

    Attributes:
        kernel (int or np.ndarray):
            A kernel of ones to implement local summation using convolution. When an int is given a ndarray of shape (int,int) is used. One can also give a kernel of choice to implement weighted sums in the receptive field. You can also give 1, so no receptive field is used. Default is a square array of ones with shape (3,3).

        epochs (int):
            The number of iterations to run the PCNN. Default is 10.

        brightness_is_normed (bool):
            If True the read images brightness is normalized using a Ratio of Gaussian (rog) approach. Default is False.

        V (int or np.ndarray):
            parameter to control the adaption of the threshold. Give a ndarray to specify a value for each pixel. Default is 0.1

        alpha (int or np.ndarray):
            parameter to control the adaption of the threshold. Give a ndarray to specify a value for each pixel. Default is 0.005

        beta (int or np.ndarray):
            linking strength. Determines how much of the activations from the linking channel is used for the internal activation. Give a ndarray to specify a value for each pixel. Default is 0.2

        dropout (bool):
            If True, a random number of kernel entries is set to 0 in each epoch. Default is False.

        scales (tuple of int):
            A tuple of length 2, that defines a downscaling factor for each dimension of the input image. Default is (1,1), meaning no rescaling.

        t (list of int):
            Sigma values by which the image is blurred for the scale space representation. Default is [2,4,6,8]

        write_gif (bool):
            set to True if the pulses of each epoch should be written to a gif in the current directory. Uses imageio package. Default is False.

        show_epoch (bool):
            If True, each frame of the written gif shows the number of the current epoch. Default is false.

        out_path (str):
            The path where outputs, like the gif, are written to. The folder will be deleted if it already exists, otherwise it is created. Default is "out/" in the current dir.

    '''
    def __init__(self, kernel=None, epochs=None, brightness_is_normed=False, V=None, alpha=None, beta=None,
                 F_init=None, Y_init=None, dropout=False, scales=(1,1), t=[2,4,6,8],
                 write_gif=False, show_epoch=False, out_path='out'):
        ### input check
        if kernel is None:
            self.kernel = np.ones((3,3))
        elif isinstance(kernel, int):
            assert kernel >= 1, 'Kernel size can not be 0 or negative!'
            self.kernel = np.ones((kernel,kernel))
        elif isinstance(kernel, np.ndarray):
            self.kernel = kernel
        elif isinstance(kernel, list):
            assert all(isinstance(x,np.ndarray) for x in kernel), 'all elements of the list must be kernels of type ndarray!'
            self.kernel = kernel
        else:
            raise ValueError('Invalid datatype for kernel! Pass an integer, a numpy ndarray or a list of arrays')

        if epochs is None:
            self.epochs = 10
        else:
            assert isinstance(epochs,int), 'epochs must be given as integer!'
            self.epochs = epochs

        assert isinstance(brightness_is_normed, bool), 'brightness_is_normed must be a boolean!'
        self.brightness_is_normed = brightness_is_normed

        if alpha is None:
            self.alpha = 0.005
        else:
            assert isinstance(alpha,(float,np.ndarray)), 'alpha must be given as float or ndarray!!'
            self.alpha = alpha

        if beta is None:
            self.beta = 0.2
        else:
            assert isinstance(beta,(float,np.ndarray)), 'beta must be given as float or ndarray!!'
            self.beta = beta

        if V is None:
            self.V = 1.0
        else:
            assert isinstance(V,(float, int, np.ndarray)), 'V must be given as float, int or ndarray!'
            self.V = V

        assert isinstance(dropout, bool), 'dropout must be bool!'
        self.dropout = dropout

        assert isinstance(scales, tuple) and all(isinstance(x, int) for x in scales) and len(scales) == 2,\
            'scales must be a tuple of 2 ints: (dh, dw)'
        assert all(scales) >= 0, 'a scaling factor can not be negative!'
        self.scales = scales

        assert isinstance(t, list) and all(isinstance(x, int) for x in t), 't must be a list of ints!'
        self.t = t

        assert isinstance(write_gif, bool), 'write_gif must be bool!'
        self.write_gif = write_gif

        assert isinstance(show_epoch, bool), 'show_epoch must be bool!'
        self.show_epoch = show_epoch

        assert isinstance(out_path, str), 'out_path must be given as string'
        self.out_path = out_path
        # create folder later, when it is needed
        ###

    def _create_out_folder(self):
        out_path = self.out_path
        if os.path.isdir(out_path):
            shutil.rmtree(out_path)
            os.makedirs(out_path)
        else:
            os.makedirs(out_path)

    def gray_pcnn(self, img, Y_init=None, F_init=None, thresh=None, gif_name=None):
        '''
        Compute the response of a Unit-linking Pulse-Coupled Neural Network (PCNN) on a given gray image.

        The neurons will be structured in 2D. Thus this implementation can be used with any 2-dimensional np.ndarray.

        Args:
            img (np.ndarray):
                The image to run the PCNN on. Is asumed to be gray-scale.

            F_init (np.ndarray):
                The feeding channel. Has the same shape as img. Defaults to the values of the given image.

            Y_init (np.ndarray):
                The initial pulse values for every neuron. Has the same shape as img. Default is zeros.

            thresh (int or np.ndarray):
                Initial threshold. Give a ndarray to specify a value for each pixel. Defaults to the values of the given image.

        Returns:
            Y_out (np.ndarray):
                2D Array of pulses, having the highest entropy.

        '''
        pcnn_gen = self.gray_pcnn_gen(img, Y_init, F_init, thresh)
        h,w = img.shape
        write_gif = self.write_gif
        out_path = self.out_path
        if gif_name is None:
            gif_name = 'movie.gif'
        assert isinstance(gif_name, str), 'The gif_name must be a valid string!'
        assert gif_name.endswith('.gif'), 'The .gif extension must be given explicitly!'
        max_ent = -np.Inf
        if write_gif:
            out_list=[]
        for epoch in range(self.epochs):
            Y = next(pcnn_gen)
            # compute entropy of current segmentation
            # TODO avoid divide by zero warnings
            P_0 = np.sum(Y == 0) / h*w
            P_1 = np.sum(Y == 1) / h*w
            ent = -P_0*np.log2(P_0) - P_1*np.log2(P_1)
            # and store the result with the highest entropy
            if ent > max_ent:
                max_ent = ent
                Y_out = Y
            # spatial_hist = spatial_hist + Y
            if write_gif:
                # convert to float for correct behaviour with opencv
                out_list.append((255*Y).astype(np.uint8))
        if write_gif:
            with imageio.get_writer(os.path.join(out_path, gif_name), mode='I',fps=8) as writer:
                for idx, Y in enumerate(out_list):
                    out = np.dstack((Y,)*3)
                    if self.show_epoch:
                        out = cv2.putText(out, text=str(idx), org=(int(w-w*0.1),int(h-h*0.05)), fontFace=3, fontScale=1.1, color=(0,0,1), thickness=4)
                    writer.append_data(out)
                print('gif {} has been written'.format(os.path.join(out_path, gif_name)))
        # TODO explore different metrics to select a final result
        return Y_out

    def get_gray_pcnn_descriptor(self, img, Y_init=None, F_init=None, thresh=None):
        '''
        Compute a descriptor based on the local pulse frequencies in a given image
        '''
        # TODO implement variants
        #   1. rectangles of size n*m counting the pulses
        #   2. binary or angular coding of local pulses (quadtree-like representaion?)

    def get_spatial_histogram(self, img, Y_init=None, F_init=None, thresh=None):
        '''
        Run the gray_pcnn generator to obtain a spatial histogram, counting the pulses at each pixel

        Args:
            img (np.ndarray):
                the image to run the PCNN on. Is asumed to be gray-scale.

            Y_init (np.ndarray):
                The initial pulse values for every neuron. Has the same shape as img. Default is zeros.

            F_init (np.ndarray):
                The feeding channel. Has the same shape as img. Defaults to the values of the given image.

            thresh (int or np.ndarray):
                Initial threshold. Give a ndarray to specify a value for each pixel. Defaults to the values of the given image.

        Returns:
            spatial_hist (np.ndarray):
                An array that counts the number of pulses in each pixel, normed by the number of epochs
        '''
        spatial_hist = np.zeros_like(img)
        pcnn_gen = self.gray_pcnn_gen(img, Y_init, F_init, thresh)
        for _ in range(self.epochs):
            spatial_hist = spatial_hist + next(pcnn_gen)
        return spatial_hist / self.epochs

    def gray_pcnn_gen(self, img, Y_init=None, F_init=None, thresh=None):
        '''
        A generator yielding the next state of pulsing neurons on each call of next().

        The neurons will be structured in 2D. Thus this implementation can be used with any 2-dimensional np.ndarray.
        This generator runs in an endless loop, so its termination must be implemented by its caller.

        Args:
            img (np.ndarray):
                the image to run the PCNN on. Is asumed to be gray-scale.

            Y_init (np.ndarray):
                The initial pulse values for every neuron. Has the same shape as img. Default is zeros.

            F_init (np.ndarray):
                The feeding channel. Has the same shape as img. Defaults to the values of the given image.

            thresh (int or np.ndarray):
                Initial threshold. Give a ndarray to specify a value for each pixel. Defaults to the values of the given image.

        Yields:
            Y (np.ndarray):
                The current state of all pulses.

        '''
        h,w = img.shape
        if Y_init is None:
            Y = np.zeros((h,w))
        else:
            assert isinstance(Y_init, np.ndarray), 'Y_init must be given as ndarray, if it is given!!'
            Y = Y_init

        if F_init is None:
            F = img
        else:
            assert isinstance(F_init, np.ndarray), 'F_init must be given as ndarray, if it is given!!'
            F = F_init

        if thresh is None:
            # thresh = F
            thresh = np.ones_like(img)
        else:
            assert isinstance(thresh,(float,np.ndarray)), 'thresh must be given as float or ndarray!!'

        V = self.V
        alpha = self.alpha
        beta = self.beta

        while True:
            k = self.kernel
            if self.dropout:
                k = k * np.random.randint(2, size=k.shape)
            Y_sum = cv2.filter2D(Y.astype(np.float), -1, k, borderType=cv2.BORDER_REFLECT_101)
            L = Y_sum > 0
            # compute inner activation
            U = F*(1 + beta*L)
            # compute each neurons pulse state
            Y = ((U-thresh) > 0)
            thresh = thresh + (-1*alpha + V * Y)
            yield Y

    def get_gen(self, img_name, color=False):
        '''
        This function returns a pcnn generator, expecting an images name instead of an actual image

        Args:
            img_name (str):
                Path to the image on which the pcnn will be run.

        Returns:
            Python generator object, yielding a pulse state for the given image on each next() call.

        '''
        return self.gray_pcnn_gen(read_img(img_name, color, self.scales, self.brightness_is_normed))

    def run(self, input, color=False, write_results=True, use_scale_space=False,
            use_hessian=False, Y_init=None, F_init=None, thresh=None):
        '''
        Wrapper for all the different models of pcnn

        Use this function as access point for all functionalities

        Args:
            input(str):
                Give an images filename to be computed or give a folder to compute the result for all images in that folder.

            color (bool):
                if True the image will be loaded as RGB otherwise as grayscale. Default is False.

            write_results (bool):
                if True all results will be written to disk. Default is True.

            use_scale_space (bool):
                If True the input images scale space is computed and the pcnn is applied to each of the resulting images. Default is False.

            use_hessian (bool):
                If True the input images scale space and the hessian determinant for each scale is computed and the pcnn is applied to each of the resulting images. Default is False.

            Y_init (np.ndarray):
                The initial pulse values for every neuron. Has the same shape as img. Default is zeros.

            F_init (np.ndarray):
                The feeding channel. Has the same shape as img. Defaults to the values of the given image.

            thresh (int or np.ndarray):
                Initial threshold. Give a ndarray to specify a value for each pixel. Defaults to the values of the given image.

        Returns:
            result (list of np.ndarray):
                The pulse state of maximum entropy for each given image.

        '''
        result = []
        self._create_out_folder()
        if os.path.isdir(input):
            img_path = input
            img_name_list = []
            for dirpath, dirnames, files in os.walk(img_path):
                for f in files:
                    if f.endswith(('.png', '.bmp', '.jpg', '.jpeg')):

                        img_name_list.append(os.path.join(dirpath, f))

            for img_name in img_name_list:
                img = read_img(img_name, color, self.scales, self.brightness_is_normed)
                out_name = img_name.split('/')[-1]
                out_sub_path = os.path.join(self.out_path, os.path.splitext(out_name)[0])
                # make a subfolder for the current image
                os.makedirs(out_sub_path)
                if use_scale_space or use_hessian:
                    sc_sp, hes_det = get_scale_space(img, self.t, use_hessian)
                    inter_list = hes_det if use_hessian else sc_sp
                    for idx, scale in enumerate(inter_list):
                        r = self.gray_pcnn(scale, Y_init, F_init, thresh,
                                           gif_name=os.path.splitext(out_name)[0] + 'scale_{}.gif'.format(self.t[idx]))
                        if write_results:
                            cv2.imwrite(os.path.join(out_sub_path, 'res_sc{}.png'.format(self.t[idx])), (r*255).astype(np.uint8))
                        result.append(r)
                r = self.gray_pcnn(img, Y_init, F_init, thresh, gif_name=os.path.splitext(out_name)[0]+'.gif')
                if write_results:
                    cv2.imwrite(os.path.join(out_sub_path, 'res.png'), (r*255).astype(np.uint8))
                result.append(r)
        else:
            img_name = input
            img = read_img(img_name, color, self.scales, self.brightness_is_normed)
            out_name = img_name.split('/')[-1]
            if use_scale_space or use_hessian:
                sc_sp, hes_det = get_scale_space(img, self.t, use_hessian)
                inter_list = hes_det if use_hessian else sc_sp
                for idx, scale in enumerate(inter_list):
                    r = self.gray_pcnn(scale, Y_init, F_init, thresh, gif_name=os.path.splitext(out_name)[0] + 'scale_{}.gif'.format(self.t[idx]))
                    if write_results:
                        cv2.imwrite(os.path.join(self.out_path, os.path.splitext(out_name)[0] + 'res_sc{}.png'.format(self.t[idx])), (r*255).astype(np.uint8))
                    result.append(r)
            r = self.gray_pcnn(img, Y_init, F_init, thresh, gif_name=os.path.splitext(out_name)[0]+'.gif')
            if write_results:
                cv2.imwrite(os.path.join(self.out_path, os.path.splitext(out_name)[0] + 'res.png'), (r*255).astype(np.uint8))
            result.append(r)
        return result


import re
import struct
import ctypes
import itertools

def tokenize(string):
	string_ = string
	#print 'tokenizing: %s' % string
	result = []

	# pick off opcode
	opcode = re.match(r'^(\w+)', string).group(1)
	result.append(['OPC', opcode])
	string = string[len(opcode):]

	# pick off the rest
	while string:
		eat = 0
		if string.startswith(' '):
			eat = 1
		elif re.match(r'^r\d+', string):
			tmp = re.match(r'^(r\d+)', string).group(1) # eg: r0
			result.append( ('GPR',int(tmp[1:])) )
			eat = len(tmp)
		elif re.match(r'^v\d+', string):
			tmp = re.match(r'^(v\d+)', string).group(1) # eg: v0
			result.append( ('VREG',int(tmp[1:])) )
			eat = len(tmp)
		elif len(string) >= 2 and string[0:2] in ['lt','gt','eq','so'] and \
		  (len(string)==2 or (re.match(r'[^\w]',string[2]))):
			result.append( ('FLAG',string[0:2]) )
			eat = 2
		elif re.match(r'^cr\d+', string):
			tmp = re.match(r'^(cr\d+)', string).group(1) # eg: cr1
			result.append( ('CREG',int(tmp[2:])) )
			eat = len(tmp)
		elif re.match(r'^vs\d+', string):
			tmp = re.match(r'^(vs\d+)', string).group(1) # eg: vs0
			result.append( ('VSREG',int(tmp[2:])) )
			eat = len(tmp)
		elif re.match(r'^f\d+', string):
			tmp = re.match(r'^(f\d+)', string).group(1) # eg: f0
			result.append( ('FREG',int(tmp[1:])) )
			eat = len(tmp)
		elif re.match(r'^-?0x[a-fA-F0-9]+', string):
			tmp = re.match(r'^(-?0x[a-fA-F0-9]+)', string).group(1)
			result.append( ('NUM', int(tmp,16)) )
			eat = len(tmp)
		elif re.match(r'^-?\d+', string):
			tmp = re.match(r'(^-?\d+)', string).group(1)
			result.append( ('NUM', int(tmp)) )
			eat = len(tmp)
		elif string[0] in list('(),.*+-'):
			result.append( (string[0],string[0]) )
			eat = 1
		else:
			raise Exception('dunno what to do with: %s (original input: %s)' % (string, string_))

		string = string[eat:]

	return result

def tokenize_types(string):
	tokens = tokenize(string)
	types, values = zip(*tokens)
	return ' '.join((values[0],) + types[1:])

(adapt, cbuf, ibuf) = (None, None, None)
def disasm(word):
	global adapt, cbuf, ibuf

	if not adapt:
		adapt = ctypes.CDLL("gofer.so")
		cbuf = ctypes.create_string_buffer(256)
		ibuf = ctypes.create_string_buffer(4)

	# form input buffer
	data = struct.pack('<I', word)

	# ask capstone
	adapt.get_disasm_capstone(data, 4, ctypes.byref(cbuf))
	return cbuf.value

def syntax_from_string(instr):
	tokens = tokenize(instr)
	syntax = tokens[0][1];

	if tokens[1:]:
		syntax += ' ' + ' '.join(map(lambda x: x[0], tokens[1:]))

	return syntax

def syntax_from_insword(insword):
	return syntax_from_string(disasm(insword))

# return all 32-bit values that have 4, 3, 2, 1 and 0 bits set
def fuzz4():
	fuzz = [0]

	for positions in itertools.combinations(range(32), 1):
		mask = (1<<positions[0])
		fuzz.append(mask)

	for positions in itertools.combinations(range(32), 2):
		mask = (1<<positions[0])|(1<<positions[1])
		fuzz.append(mask)

	for positions in itertools.combinations(range(32), 3):
		mask = (1<<positions[0])|(1<<positions[1])|(1<<positions[2])
		fuzz.append(mask)

	for positions in itertools.combinations(range(32), 4):
		mask = (1<<positions[0])|(1<<positions[1])|(1<<positions[2])|(1<<positions[3])
		fuzz.append(mask)

	# fuzz should have all 4-bit subsets, 3-bit subsets, 2-bit, 1-bit, 0-bit
	assert len(fuzz) == 32*31*30*29/24 + 32*31*30/6 + 32*31/2 + 32 + 1

	return fuzz

def fuzz5():
	fuzz = fuzz4()

	for positions in itertools.combinations(range(32), 5):
		mask = (1<<positions[0])|(1<<positions[1])|(1<<positions[2])|(1<<positions[3])|(1<<positions[4])
		fuzz.append(mask)

	# fuzz should have all 4-bit subsets, 3-bit subsets, 2-bit, 1-bit, 0-bit
	assert len(fuzz) == 32*31*30*29*28/120 + 32*31*30*29/24 + 32*31*30/6 + 32*31/2 + 32 + 1

	return fuzz

def fuzz6():
	fuzz = fuzz5()

	for positions in itertools.combinations(range(32), 6):
		mask = (1<<positions[0])|(1<<positions[1])|(1<<positions[2])|(1<<positions[3])|(1<<positions[4])|(1<<positions[5])
		fuzz.append(mask)

	# fuzz should have all 5-bit subets, 4-bit subsets, 3-bit subsets, 2-bit, 1-bit, 0-bit
	assert len(fuzz) == 32*31*30*29*28*27/720 + 32*31*30*29*28/120 + 32*31*30*29/24 + 32*31*30/6 + 32*31/2 + 32 + 1

	return fuzz


#linear search
def linear_search(arr, target):
    for i in range(len(arr)):
        if arr[i] == target:
            return i

    return -1

arr = [4, 2, 7, 1, 9, 5]
target = 7

result = linear_search(arr, target)

if result != -1:
    print("Element found at index", result)
else:
    print("Element not found")

#Binary search
def binary_search(arr, target):
    low = 0
    high = len(arr) - 1

    print(high)

    while low <= high:
        mid = (low + high) // 2

        if arr[mid] == target:
            return mid
        elif arr[mid] < target:
            low = mid + 1

        else:
            high = mid - 1

    return -1

arr = [2, 4, 6, 8, 10, 12, 14, 16, 18, 20]
target = 12

result = binary_search(arr, target)

if result != -1:
    print("Element found at index", result)
else:
    print("Element not found")


import os
import configparser

class Config:
    # _CONFIG_NAME = None
    _CONFIG_NAME = 'config.ini'

    def __init__(self):

        self.options = {
            'logging': {
                'log_path': None,
                'log_level': 'DEBUG',
                'log_format': '%(asctime)s - %(levelname)s - %(message)s'
            },
            'db': {
                'postgres': None
            },
            'general': {
                'chunk_step': None,
                'promts_table': 'promts',
                'promts_files_table':'promt_files'
            }
        }
        self._parseini()

    def _parseini(self):
        if self._CONFIG_NAME is None:
            return

        path = os.path.dirname(os.path.realpath(__file__))
        if not os.path.exists(path + '/' + self._CONFIG_NAME):
            return

        conf = configparser.RawConfigParser()
        conf.read(path + '/' + self._CONFIG_NAME)

        for key, value in conf.items():
            if key in self.options:
                self.options[key] = dict(value)

    def __getitem__(self, key):
        if key in self.options:
            return self.options[key]
        return {}


import cluster_generator as cg
import unyt as u
from numpy.random import RandomState
import numpy as np

# Note that cluster_generator does not use unyt units for speed and simplicity,
# so mass units are Msun, length units are kpc, and time units are Myr

# Put the two clusters at a redshift z = 0.1
z = 0.1

# M200 for both clusters
M200_1 = 6.0e14 # in Msun
M200_2 = 2.0e14 # in Msun

conc = 4.0 # A good approximation to the concentration parameter for both clusters

# Find r200 for both clusters
r200_1 = cg.find_overdensity_radius(M200_1, 200.0, z=z)
r200_2 = cg.find_overdensity_radius(M200_2, 200.0, z=z)

# Scale radii to be used for the sNFW profiles
a1 = r200_1/conc
a2 = r200_2/conc

# For the total mass density profile, we will use a "super-NFW" profile, which
# is very similar to the NFW profile but falls off slightly faster (Lilley, E. J.,
# Wyn Evans, N., & Sanders, J.L. 2018, MNRAS)

# Determine the total mass for each sNFW profile
M1 = cg.snfw_total_mass(M200_1, r200_1, a1)
M2 = cg.snfw_total_mass(M200_2, r200_2, a2)

# Use this total mass to construct total mass profiles for each cluster
Mt1 = cg.snfw_mass_profile(M1, a1)
Mt2 = cg.snfw_mass_profile(M2, a2)

# Use the total mass profiles to determine r500/M500 and r2500/M2500 for
# each cluster
r500_1, M500_1 = cg.find_radius_mass(Mt1, z=z, delta=500.0)
r2500_1, M2500_1 = cg.find_radius_mass(Mt1, z=z, delta=2500.0)
r500_2, M500_2 = cg.find_radius_mass(Mt2, z=z, delta=500.0)
r2500_2, M2500_2 = cg.find_radius_mass(Mt2, z=z, delta=2500.0)

# Total mass density profiles for each cluster
rhot1 = cg.snfw_density_profile(M1, a1)
rhot2 = cg.snfw_density_profile(M2, a2)

# Sprinkle some stars in--2% of the total mass for each cluster
rhos1 = 0.02*rhot1
rhos2 = 0.02*rhot2

# Find the gas mass fraction within R500 (using the relationship between
# M500 and fgas from Vikhlinin, A., et al. 2009, ApJ, 692, 1033
f_g1 = cg.f_gas(M500_1)
f_g2 = cg.f_gas(M500_2)

# This sets the gas density profile using the functional form from Vikhlinin, A.,
# Kravtsov, A., Forman, W., et al. 2006, ApJ, 640, 691 for the first cluster. We
# set the scale density to 1.0 first and will rescale it in the next line by the
# gas mass within r500
rhog1 = cg.vikhlinin_density_profile(1.0, 0.2*r2500_1, 0.67*r200_1, 1.0, 0.67, 3.0)
rhog1 = cg.rescale_profile_by_mass(rhog1, f_g1*M500_1, r500_1)

# Same as above for the second cluster
rhog2 = cg.vikhlinin_density_profile(1.0, 0.2*r2500_2, 0.67*r200_2, 1.0, 0.67, 3.0)
rhog2 = cg.rescale_profile_by_mass(rhog2, f_g2*M500_2, r500_2)

# This is the plasma beta parameter for the ratio of the thermal pressure to the
# magnetic pressure
beta = 100.0

# This sets up the profiles for the first cluster assuming hydrostatic equilibrium,
# taking the gas density, total mass density, and stellar density as input
hse1 = cg.ClusterModel.from_dens_and_tden(0.1, 20000.0, rhog1, rhot1,
                                          stellar_density=rhos1)

# This sets a radial magnetic field strength profile using the beta parameter and
# the pressure in the profile, assuming p_B = B^2/s (thus gaussian=False)
hse1.set_magnetic_field_from_beta(beta, gaussian=False)

# These lines are the same as above for the second cluster
hse2 = cg.ClusterModel.from_dens_and_tden(0.1, 20000.0, rhog2, rhot2,
                                          stellar_density=rhos2)
hse2.set_magnetic_field_from_beta(beta, gaussian=False)

# Write the profiles for each cluster to files
hse1.write_model_to_h5("profile1.h5", overwrite=True)
hse2.write_model_to_h5("profile2.h5", overwrite=True)

# Set a random number generator for the generation of the magnetic field
# vector potential in 3D
prng = RandomState(24)

# This is the width of the GAMER simulation box and its center
w = 15000.0 # in kpc
center = np.array([0.5*w]*3)

# This determines the centers of the clusters, assuming a distance of
# 3 Mpc and zero impact parameter, centered on the box center
d = 3000.0 # in kpc
b = 0.0 # in kpc
center1, center2 = cg.compute_centers_for_binary(center, d, b)

# This sets up a 3D magnetic vector potential which GAMER will take the curl
# of on the AMR grid to get the initial B-field. It is a tangled field which
# uses a Kolmogorov spectrum with a large-scale cutoff of 500 kpc, a
# small-scale cutoff of 10 kpc, and is proportional on average to the pressure
# everywhere (given by the magnetic field profile of the clusters from above).
# Outside of r_max = 5000.0 kpc from each cluster center the average B-field
# is constant
left_edge = center-0.5*w
right_edge = center+0.5*w
dims = (256,)*3
bfield = cg.RadialRandomMagneticVectorPotential(left_edge, right_edge, dims,
                                                10.0, 500.0, center1,
                                                "profile1.h5", ctr2=center2,
                                                profile2="profile2.h5", r_max=5000.0)

# Write the 3D vector potential to the B_IC file
bfield.write_to_h5("B_IC", overwrite=True, length_unit="Mpc",
                   field_unit="sqrt(1e14*Msun/Mpc**3)*Mpc/(10*Gyr)")

# We now set up the velocities of the two clusters. Assume 1500 km/s
# relative velocity, and then use the M200 of the two clusters to
# set velocity vectors in roughly the CM frame. The velocity is in
# the x-direction only
velocity = (1500.0*u.km/u.s).to_value("kpc/Myr")
velocity1 = np.array([velocity*M200_2/(M200_1+M200_2), 0.0, 0.0])
velocity2 = np.array([-velocity*M200_1/(M200_1+M200_2), 0.0, 0.0])

# Now we set up the cluster initial conditions. use 2e6 DM particles,
# 4e4 star particles. At r_max = 5000.0 kpc, the profiles of each cluster
# are constant
num_particles = {"dm": 2_000_000, "star": 40_000}

ics = cg.ClusterICs("1to3_b0.0", 2, ["profile1.h5", "profile2.h5"],
                    [center1, center2], [velocity1, velocity2],
                    num_particles=num_particles, mag_file="B_IC", r_max=5000.0)

# This writes the GAMER-specific IC files that are needed, generates
# the particles, and prints out the contents of the Input__TestProb
# file which should be used
cg.setup_gamer_ics(ics)


import mne.time_frequency.tfr
from mne.io import read_raw_bdf
from mne.preprocessing import ICA
import matplotlib.pyplot as plt
import numpy as np
import os

#path = '/Users/carme/Desktop/ML for physiological time series/Project/EEG/sub-001_eeg_sub-001_task-med1breath_eeg.bdf'
path = '/Users/carme/Desktop/ML for physiological time series/Project/EEG'

for file in os.listdir(path):
    f = os.path.join(path, file)
    raw_eeg = read_raw_bdf(f,preload=True)

#for i in [0, 5, 50, 75]:
#    channel_i = raw_eeg[raw_eeg.ch_names[i]]
#    plt.plot(channel_i[1], np.squeeze(channel_i[0]))
#    plt.show()

    #We will use only 180 seconds of the signal - in the paper they used 30sec, however, when filtering the data edge artifacts may rise, therefore, we perform the preprocessing on longer data period
    raw_eeg.crop(tmin=60, tmax=240)

    #Change the reference of the signal to be the average reference across EEG electrodes
    raw_eeg.set_eeg_reference('average', projection=True, ch_type='eeg')

    channel_i = raw_eeg[raw_eeg.ch_names[37]]
    plt.plot(channel_i[1], np.squeeze(channel_i[0]))
    plt.title('Fz electrode before pre-processing')
    plt.xlabel('Time [$Sec$]')
    plt.ylabel('Amplitude ['r'$\mu$V]')
    plt.show()

    channels_to_filter = np.arange(0,72)

    #As can be seen, the data is noisy, therefore we will first perform pre-processing.
    #First step - band pass filtering of 1-45Hz (different from the paper which performed 4-45Hz filtering) since this is a meditation task therefore lower frequencies should be kept in the data
    #We will use IIR filter (backward-foreward/filtfilt) so no phase response

    Filtered_eeg = raw_eeg.filter(l_freq=1,h_freq=45,picks=channels_to_filter,filter_length='auto',method='iir')

    channel_i_filt = Filtered_eeg[Filtered_eeg.ch_names[37]]
    plt.plot(channel_i_filt[1], np.squeeze(channel_i_filt[0]))
    plt.title('Fz electrode after band-pass filtering')
    plt.xlabel('Time [$Sec$]')
    plt.ylabel('Amplitude ['r'$\mu$V]')
    plt.show()

    #50Hz notch filter (India's power line frequency)
    Filtered_eeg_notch = Filtered_eeg.notch_filter(np.arange(50, 251, 50),method='fir',picks=channels_to_filter,phase='zero-double')

    channel_i_notch = Filtered_eeg_notch[Filtered_eeg_notch.ch_names[37]]

    plt.plot(channel_i_filt[1], np.squeeze(channel_i_filt[0]),label='Band pass filtered')
    plt.plot(channel_i_notch[1], np.squeeze(channel_i_notch[0]),label='Band pass and notch filtered')
    #plt.xlim(0,10)
    plt.legend(bbox_to_anchor=(1.124, 0.4))
    plt.title('Fz electrode after filtering')
    plt.xlabel('Time [$Sec$]')
    plt.ylabel('Amplitude ['r'$\mu$V]')
    plt.show()

    #EOG + ECG artifacts removal
    #First lets examine the extrenal channels:

    for i in [64, 65, 66, 67, 68, 69, 70, 71]:
        channel_i_i = Filtered_eeg_notch[Filtered_eeg_notch.ch_names[i]]
        plt.plot(channel_i_i[1], np.squeeze(channel_i_i[0]))
        plt.xlim(0, 5)
        plt.show()

    #As can be seen, electrode 70 recorded ECG data, according to the paper 6 electrodes were used to record EOG data
    #Therefore, we will use electrode 70 for ECG artifact removal and electrodes 64-69 for EOG artifact removal

    ica = ICA(n_components=15, max_iter='auto', random_state=15)
    ica.fit(Filtered_eeg_notch)
    ica
    ica.plot_sources(Filtered_eeg_notch, show_scrollbars=False)

    EOG_ch_names =  ["EXG1", "EXG2", "EXG3","EXG4","EXG5","EXG6"]
    ica.exclude = []
    # find which ICs match the EOG pattern
    eog_indices, eog_scores = ica.find_bads_eog(Filtered_eeg_notch,ch_name=EOG_ch_names)
    ica.exclude = eog_indices

    #barplot of ICA component "EOG match" scores
    ica.plot_scores(eog_scores)

    # plot ICs applied to raw data, with EOG matches highlighted
    ica.plot_sources(Filtered_eeg_notch, show_scrollbars=False)

    ica.exclude = []
    # find which ICs match the EOG pattern
    ecg_indices, ecg_scores = ica.find_bads_ecg(Filtered_eeg_notch,ch_name='EXG7')
    ica.exclude = ecg_indices

    #barplot of ICA component "EOG match" scores
    ica.plot_scores(ecg_scores)

    # plot ICs applied to raw data, with ECG matches highlighted
    ica.plot_sources(Filtered_eeg_notch, show_scrollbars=False)

    Filtered_eeg_notch_ICA = ica.apply(Filtered_eeg_notch)
    channel_i_i = Filtered_eeg_notch[Filtered_eeg_notch.ch_names[37]]
    plt.plot(channel_i_i[1], np.squeeze(channel_i_i[0]))
    plt.title('Fz electrode after filtering and ICA - EOG and ECG rejection')
    plt.xlabel('Time [$Sec$]')
    plt.ylabel('Amplitude ['r'$\mu$V]')
    plt.show()

    # Perform a CWT on Fz electrode
    Fz_signal = Filtered_eeg_notch_ICA[Filtered_eeg_notch_ICA.ch_names[37]][0]
    #t = Filtered_eeg_notch[Filtered_eeg_notch.ch_names[37]][1]

    freq_range = np.arange(1,46)
    Ws = mne.time_frequency.tfr.morlet(1024, freq_range)
    Fz_tfr = abs(np.squeeze(mne.time_frequency.tfr.cwt(Fz_signal, Ws)))

    matrix = np.zeros((45,1))
    for i in range(0,45):
        max_array = np.array([np.amax(Fz_tfr[:,i*4000:i*4000+4000], axis=1)]).T
        if i==0:
            matrix = max_array
        else:
            matrix = np.append(matrix, max_array, axis=1)

    plt.imshow(matrix, 'gray')
    plt.title('Grayscale image after performing CWT')
    plt.xlabel('Scale')
    plt.ylabel('Frequency [Hz]')
    plt.show()

    np.save(f,matrix)

#After all the matrices were saved perform train-test split, X should be the data (the matrices) and Y should be the labels


from collections import Counter, defaultdict, deque
from functools import lru_cache
from typing import (
    List,
    Tuple,
    Set,
    Dict,
    Iterable,
    DefaultDict,
    Optional,
    Union,
    Generator,
)
from copy import deepcopy
import re
from math import floor, ceil

def parse(filename: str) -> List[str]:
    with open(filename, "r") as f:
        strings = f.read().strip().split("\n")

    return strings

def is_explosive(
    string,
) -> Tuple[bool, Tuple[Optional[int], Optional[int]]]:
    opening_brackets_stack = []
    is_exploding = False
    explode_start = None
    explode_end = None
    for i, char in enumerate(string):
        if char == "[":
            opening_brackets_stack.append(char)
            if len(opening_brackets_stack) == 5:
                explode_start = i
                is_exploding = True
                continue
        elif char == "]":
            opening_brackets_stack.pop()
            if is_exploding:
                explode_end = i
                break

    return is_exploding, (explode_start, explode_end)

def explode_and_reformat(
    string: str, explosion_span: Tuple[int, int]
) -> str:
    # Get exploding tuple values
    left, right = re.findall(
        "\d+", string[explosion_span[0]: explosion_span[1]]
    )
    new_str = ""
    # Replace exploded with 0
    string = (
        string[:explosion_span[0]] + "0" + string[explosion_span[1] + 1:]
    )

    prev_nums = list(re.finditer("\d+", string[0: explosion_span[0]]))
    next_nums = list(re.finditer("\d+", string[explosion_span[0] + 1:]))

    # Replace left, right numbers in the string where the explosion pair has been
    # replaced by 0
    if not prev_nums and next_nums:
        next_num = next_nums[0]
        new_str = (
            string[:explosion_span[0] + 1 + next_num.start()]
            + f"{int(next_num.group()) + int(right)}"
            + string[explosion_span[0] + 1 + next_num.end():]
        )
    elif prev_nums and not next_nums:
        prev_num = prev_nums[-1]
        new_str += (
            string[:prev_num.start()]
            + f"{int(prev_num.group()) + int(left)}"
            + string[prev_num.end():]
        )

    else:
        prev_num, next_num = prev_nums[-1], next_nums[0]
        new_str = (
            string[:prev_num.start()]
            + f"{int(prev_num.group()) + int(left)}"
            + string[prev_num.end(): next_num.start() + explosion_span[0] + 1]
            + f"{int(next_num.group()) + int(right)}"
            + string[explosion_span[0] + 1 + next_num.end():]
        )
    return new_str

def split_check_and_reformat(string: str) -> Tuple[bool, str]:
    for num in re.finditer("\d+", string):
        val, span = int(num.group()), num.span()
        if val >= 10:
            replacement = f"[{floor(val / 2)},{ceil(val / 2)}]"
            new_string = string[:span[0]] + replacement + string[span[1]:]
            return True, new_string
    return False, ""

def reduce(string: str) -> str:
    is_exploding, explosion_span = is_explosive(string)
    if is_exploding:
        string = explode_and_reformat(string, explosion_span)  # type: ignore
        string = reduce(string)
    is_split, split_string = split_check_and_reformat(string)
    if is_split:
        string = split_string
        string = reduce(string)
    return string

def addition(strings: List[str]) -> int:
    first = strings.pop(0)

    while strings:
        second = strings.pop(0)
        full = "[" + first + "," + second + "]"
        first = reduce(full)
    return evaluate_expression(first)

def evaluate_expression(string):
    if string.isdigit():
        return int(string)
    split_idx = get_split_idx(string)
    string1 = string[1:split_idx]
    string2 = string[split_idx + 1: -1]

    return 3 * evaluate_expression(string1) + 2 * evaluate_expression(string2)

def get_split_idx(string) -> Optional[int]:

    opening_brackets_stack = []
    comma_idx: List[int] = []
    for i, char in enumerate(string[1:-1], 1):
        if i == len(string) - 1:
            break
        elif char == "[":
            opening_brackets_stack.append(char)
        elif char == "]":
            opening_brackets_stack.pop()
            comma_idx.pop()
        elif char == ",":
            comma_idx.append(i)

    assert len(comma_idx) == 1
    return comma_idx[0]

def get_pair_with_max_addition(strings: List[str]) -> int:
    max_score = 0
    for s1 in strings:
        for s2 in strings:
            if s1 == s2:
                continue
            score = addition([s1, s2])
            if score > max_score:
                max_score = score
    return max_score

def main(filename: str) -> Tuple[Optional[int], Optional[int]]:
    from time import time

    start = time()
    answer_a, answer_b = None, None
    # p1
    strings = parse(filename)
    addition_score = addition(strings)
    # p2
    strings = parse(filename)
    max_pair_score = get_pair_with_max_addition(strings)
    answer_a = addition_score
    answer_b = max_pair_score
    end = time()
    print(end - start)
    return answer_a, answer_b

if __name__ == "__main__":

    from utils import submit_answer
    from aocd.exceptions import AocdError

    sample = "sample.txt"
    input = "input.txt"

    sample_a_answer = 4140
    sample_b_answer = 3993

    answer_a, answer_b = main(sample)
    assert (
        answer_a == sample_a_answer
    ), f"AnswerA incorrect: Actual: {answer_a}, Expected: {sample_a_answer}"
    print("sampleA correct")
    if answer_b:
        assert (
            answer_b == sample_b_answer
        ), f"AnswerB incorrect: Actual: {answer_b}, Expected: {sample_b_answer}"
        print("sampleB correct")

    # Test on your input and submit
    answer_a, answer_b = main(input)
    print(f"Your input answers: \nA: {answer_a}\nB: {answer_b}")
    try:
        submit_answer(answer_a, "a", 18, 2021)
    except AocdError:
        submit_answer(answer_b, "b", 18, 2021)


#Data analysis
import os
import io
import pandas as pd
import pickle
import numpy as np
from datetime import datetime, date,timedelta
import streamlit as st

#custom functions
from Funzioni_utili import bestclassifier,train,save_pickle,team_metrics,champions_metrics,d_in_future, confMatrix,\
download_excel,file_selector,doyourstupidthings,talk, prediction

###MAIN###
st.sidebar.markdown("# Previsioni per la champions del brasileðŸŽˆ")

with st.sidebar:
    st.write('Qui facciamo le predizioni. ')
st.title('Brasile 2023')

st.subheader("""Cosa vorresti prevedere?""")
st.write('Il modello Ã¨ allenato per il periodo 2012-2022 della championship del Brasile.')
#uploaded_file = st.file_uploader("Upload Excel to explore", type=".xlsx")
#path=os.getcwd()

#name = st.text_input("Nome e percorso del file", 'BRA_2012-2022hst_2023og_xVERO.xlsx')
year_col = st.text_input("Nome della colonna dell'anno", 'Season')
col_day = st.text_input("Nome della colonna della giornata", 'giornata')
start_year=st.text_input("Anno iniziale del dataset completo",2012)
start_year=int(start_year)
end_year=st.text_input("Anno finale del dataset completo",2022)
end_year=int(end_year)
anno_val=st.text_input("Anno in corso",2023)
anno_val=int(anno_val)
day=st.text_input('Giornata che si vuole predire',10)
day=int(day)
#Campionato brasile. Un anno per ogni foglio
#Alleno su tutti gli anni, passando tutti gli anni sia per la media che per la predizione.

anni=[*range(start_year,end_year+1)] #esclude 'l'ultimo anno
#valido sul 2022 che non ha mai visto.

#col_raw=['Giornata','Date','Time','HomeTeam','AwayTeam','FTHG','FTAG','FTR']
col_raw=['Country','League','Season','Date','Time','Home','Away','HG','AG','Res']

uploaded_model = st.file_uploader("Carica modello. Limite: 200MB")

#'D_in_1iter', 'D_in_2iter', 'D_in_3iter',
outputs=['D_in_4iter','D_in_2iter','D_in_1iter']
uploaded_file = st.file_uploader("Carica excel", type=".xlsx")

if st.button('Prevedi for Braaasil',disabled=not (uploaded_file and uploaded_model), type='primary'):

    st.write(':leaves:')
    save=pickle.load(uploaded_model)
    output_select=save['Predict_future']
    st.write('Il modello prevede la probabilitÃ  che una squadra faccia almeno un pareggio nelle prossime {} giornate.'.format(output_select))

    output_choice=save['Output']

    [raw,final_df,int_df]=doyourstupidthings(uploaded_file,year_col,col_day,anni,anno_val,day=day)
    [final_df,alg_w]=prediction(save,output_choice, final_df)
    talk(day,output_choice,final_df)

    st.write('Ecco i dati completi per la giornata {}.'.format(day))
    download_excel(final_df,name_exc='Prediction_Day{}'.format(day))
    st.balloons()

#######PROSSIMI SVILUPPI##########
## NEXT: PER I DATI CHE HO NEL PASSATO RENDERE DISPONIBILE VALIDAZIONE. XAI PER CAPIRE COME MAI.
## SQUADRE CAMBIANO: 3 ALL'ANNO.

##ANALISI
# QUALE DISTRIBUZIONE HA LA SINGOLA SQUADRA NEI PAREGGI QUEST'ANNO? QUANTO NEGLI ANNI PRECEDENTI?
# SOLO 3-4 ANNI NEL TRAINING
# PUNTEGGIO: DOVE SI TROVA LA SQUADRA RISPETTO ALLE ALTRE?
# PUNTEGGIO NEL PASSATO?
# QTY PAREGGI MASSIMI E MEDI!

####NUOVI INPUT
## INTEGRARE ANCHE I PUNTI DELLA SQUADRA GG PER GIORNO E LA MEDIA LA DEV STD E MAX MIN DI TUTTE LE ALTRE SQUADRE NELLA GIORNATA CORRENTE.
## MEDIA E DEV STD DI PUNTI DELLA CHAMPIONSHIP E DI QUELLE PRECEDENTI
## E ANCHE MEDIA E DEV STD DEGLI ANNI PRECEDENTI DELLA STESSA SQUADRA.
## INFO SULLA DISTRIBUZIONE DEI PAREGGI NEL TEMPO. COME RAPPRESENTARLA? NON SOLO MAC DEI GIORNI MA MEDIA SU PARTITA E DEV STD SU PARTITA DI MAC NON PARI.
## COPPIA DI SQUADRE: QUANDO PAREGGIANO INSIEME RENDILO. 1. INCROCIARE PROB PAREGGIO E MOSTRARE LE COPPIE PIÃ¹ ALTE.
## MODELLO CHE PREVEDE LA PROBABILITA' SHARP DI PAREGGIO AL GG 1, 2 3 O 4, NON LA SOMMA.
## MODELLO PER COPPIE: CASA/FUORI
## INTEGRARE MEDIA E DEV STD DELLA DISTRIBUZIONE DI GOAL NELLE ULTIME PARTITE PER SQUADRA PER COFRONTARE LE DISTRIBUZIONI
## MODELLINO MULTICLASSE CHE PREVEDE TUTTI E QUATTRO.


import gradio as gr
from PIL import Image
from clip_interrogator import Config, Interrogator
from transformers import (
    AutoProcessor,
    AutoModelForCausalLM,
    BlipForConditionalGeneration,
    Blip2ForConditionalGeneration,
)
import pandas as pd
import torch
import os

title = "Interrogator"

def setup(kubin):
    cache_dir = kubin.params("general", "cache_dir")

    CAPTION_MODELS = {
        "blip-base": "Salesforce/blip-image-captioning-base",  # 990MB
        "blip-large": "Salesforce/blip-image-captioning-large",  # 1.9GB
        "blip2-2.7b": "Salesforce/blip2-opt-2.7b",  # 15.5GB
        "blip2-flan-t5-xl": "Salesforce/blip2-flan-t5-xl",  # 15.77GB
        "git-large-coco": "microsoft/git-large-coco",  # 1.58GB
    }

    def patched_load_caption_model(self):
        if self.config.caption_model is None and self.config.caption_model_name:
            if not self.config.quiet:
                print(f"Loading caption model {self.config.caption_model_name}...")

            model_path = CAPTION_MODELS[self.config.caption_model_name]
            if self.config.caption_model_name.startswith("git-"):
                caption_model = AutoModelForCausalLM.from_pretrained(
                    model_path, torch_dtype=torch.float32, cache_dir=cache_dir
                )
            elif self.config.caption_model_name.startswith("blip2-"):
                caption_model = Blip2ForConditionalGeneration.from_pretrained(
                    model_path, torch_dtype=self.dtype, cache_dir=cache_dir
                )
            else:
                caption_model = BlipForConditionalGeneration.from_pretrained(
                    model_path, torch_dtype=self.dtype, cache_dir=cache_dir
                )
            self.caption_processor = AutoProcessor.from_pretrained(
                model_path, cache_dir=cache_dir
            )

            caption_model.eval()
            if not self.config.caption_offload:
                caption_model = caption_model.to(self.config.device)
            self.caption_model = caption_model
        else:
            self.caption_model = self.config.caption_model
            self.caption_processor = self.config.caption_processor

    Interrogator.load_caption_model = patched_load_caption_model

    ci = None
    ci_config = None
    source_image = gr.Image(type="pil", label="Input image", elem_classes=[])

    def get_interrogator(clip_model, blip_type, cache_path, chunk_size):
        nonlocal ci
        nonlocal ci_config

        if ci is None or [clip_model, blip_type] != ci_config:
            ci_config = [clip_model, blip_type]
            ci = Interrogator(
                Config(
                    clip_model_name=clip_model,
                    caption_model_name=blip_type,
                    clip_model_path=cache_path,
                    cache_path=cache_path,
                    download_cache=True,
                    chunk_size=chunk_size,
                )
            )

        return ci

    def interrogate(image, mode, clip_model, blip_type, chunk_size):
        image = image.convert("RGB")
        interrogated_text = ""

        interrogator = get_interrogator(
            clip_model=clip_model,
            blip_type=blip_type,
            cache_path=f"{cache_dir}/clip_cache",
            chunk_size=chunk_size,
        )
        if mode == "best":
            interrogated_text = interrogator.interrogate(image)
        elif mode == "classic":
            interrogated_text = interrogator.interrogate_classic(image)
        elif mode == "fast":
            interrogated_text = interrogator.interrogate_fast(image)
        elif mode == "negative":
            interrogated_text = interrogator.interrogate_negative(image)

        return interrogated_text

    def batch_interrogate(
        image_dir,
        batch_mode,
        image_extensions,
        output_dir,
        caption_extension,
        output_csv,
        mode,
        clip_model,
        blip_type,
        chunk_size,
        progress=gr.Progress(),
    ):
        if output_dir == "":
            output_dir = image_dir
        os.makedirs(output_dir, exist_ok=True)

        relevant_images = []

        progress(0, desc="Starting batch interrogation...")
        if not os.path.exists(image_dir):
            return f"Error: image folder {image_dir} does not exists"

        for filename in os.listdir(image_dir):
            if filename.endswith(tuple(image_extensions)):
                relevant_images.append([filename, f"{image_dir}/{filename}", ""])

        print(f"found {len(relevant_images)} images to interrogate")
        image_count = 0
        for _ in progress.tqdm(relevant_images, unit="images"):
            filename = relevant_images[image_count][0]
            filepath = relevant_images[image_count][1]

            image = Image.open(filepath)
            caption = interrogate(image, mode, clip_model, blip_type, chunk_size)

            if batch_mode == 0:
                caption_filename = os.path.splitext(filename)[0]
                with open(
                    f"{output_dir}/{caption_filename}{caption_extension}",
                    "w",
                    encoding="utf-8",
                ) as file:
                    file.write(caption)
            elif batch_mode == 1:
                relevant_images[image_count][2] = caption

            image_count += 1

        if batch_mode == 1:
            captions_df = pd.DataFrame(
                [i[1:] for i in relevant_images], columns=["image_name", "caption"]
            )
            csv_path = f"{output_dir}/{output_csv}"
            captions_df.to_csv(csv_path, index=False)
            print(f"CSV file with captions saved to {csv_path}")

        return f"Captions for {len(relevant_images)} images created"

    def interrogator_ui(ui_shared, ui_tabs):
        with gr.Row() as interrogator_block:
            with gr.Column(scale=1) as interrogator_params_block:
                with gr.Row():
                    clip_model = gr.Dropdown(
                        choices=["ViT-L-14/openai", "ViT-H-14/laion2b_s32b_b79k"],
                        value="ViT-L-14/openai",
                        label="CLIP model",
                    )
                with gr.Row():
                    mode = gr.Radio(
                        ["best", "classic", "fast", "negative"],
                        value="fast",
                        label="Mode",
                    )
                with gr.Row():
                    blip_model_type = gr.Radio(
                        ["blip-base", "blip-large", "git-large-coco"],
                        value="blip-large",
                        label="Caption model name",
                    )
                with gr.Row():
                    chunk_size = gr.Slider(
                        512, 2048, 2048, step=512, label="Chunk size"
                    )

            with gr.Column(scale=1):
                with gr.Tabs():
                    with gr.TabItem("Single image"):
                        with gr.Column(scale=1):
                            with gr.Row():
                                source_image.render()

                        with gr.Column(scale=1):
                            interrogate_btn = gr.Button(
                                "Interrogate", variant="primary"
                            )
                            target_text = gr.Textbox(
                                lines=5,
                                label="Interrogated text",
                                show_copy_button=True,
                            )

                            kubin.ui_utils.click_and_disable(
                                interrogate_btn,
                                fn=interrogate,
                                inputs=[
                                    source_image,
                                    mode,
                                    clip_model,
                                    blip_model_type,
                                    chunk_size,
                                ],
                                outputs=[target_text],
                                js=[
                                    f"args => kubin.UI.taskStarted('{title}')",
                                    f"args => kubin.UI.taskFinished('{title}')",
                                ],
                            )
                    with gr.TabItem("Batch"):
                        image_dir = gr.Textbox(label="Directory with images")

                        with gr.Row():
                            image_types = gr.CheckboxGroup(
                                [".jpg", ".jpeg", ".png", ".bmp"],
                                value=[".jpg", ".jpeg", ".png", ".bmp"],
                                label="Files to interrogate",
                            )

                        caption_mode = gr.Radio(
                            choices=["text files", "csv dataset"],
                            info="Save captions to separate text files or to a single csv file",
                            value="text files",
                            label="Caption save mode",
                            type="index",
                        )
                        output_dir = gr.Textbox(
                            label="Output folder",
                            info="If empty, the same folder will be used",
                        )

                        caption_extension = gr.Textbox(
                            ".txt", label="Caption files extension", visible=True
                        )
                        output_csv = gr.Textbox(
                            value="captions.csv",
                            label="Name of csv file",
                            visible=False,
                        )

                        caption_mode.select(
                            fn=lambda m: [
                                gr.update(visible=m != 0),
                                gr.update(visible=m == 0),
                            ],
                            inputs=[caption_mode],
                            outputs=[caption_extension, output_csv],
                        )

                        batch_interrogate_btn = gr.Button(
                            "Interrogate", variant="primary"
                        )
                        progress = gr.HTML(label="Interrogation progress")

                        kubin.ui_utils.click_and_disable(
                            batch_interrogate_btn,
                            fn=batch_interrogate,
                            inputs=[
                                image_dir,
                                caption_mode,
                                image_types,
                                output_dir,
                                caption_extension,
                                output_csv,
                                mode,
                                clip_model,
                                blip_model_type,
                                chunk_size,
                            ],
                            outputs=[progress],
                            js=[
                                f"args => kubin.UI.taskStarted('{title}')",
                                f"args => kubin.UI.taskFinished('{title}')",
                            ],
                        )

            interrogator_params_block.elem_classes = ["block-params"]
        return interrogator_block

    return {
        "title": title,
        "send_to": f"ðŸ“„ Send to {title}",
        "tab_ui": lambda ui_s, ts: interrogator_ui(ui_s, ts),
        "send_target": source_image,
        "api": {
            interrogate: lambda image, mode="fast", clip_model="ViT-L-14/openai", blip_type="large", chunks=2048: interrogate(
                image, mode, clip_model, blip_type, chunks
            )
        },
    }


from remail.mail import msg_set_payload, msg_set_header
from remail.mail import msg_get_payload_as_string
from remail.mail import msg_from_string, msg_set_gpg_payload

import hashlib
import string
import gnupg
import time
import os

class RemailGPGException(Exception):
    pass

class RemailGPGKeyException(Exception):
    pass

class gpg_crypt(object):
    def __init__(self, gpgcfg, account, checkkey=True):
        self.config = gpgcfg
        self.account = account
        self.gpg = gnupg.GPG(gnupghome=self.config.home,
                             gpgbinary=self.config.gpgbinary,
                             keyring=self.config.keyring)
        self.keys = self.gpg.list_keys()

        if not checkkey:
            return

        self._check_key(self.keys, self.account)
        # Verify that the private key is there as well
        self._check_key(self.gpg.list_keys(True), self.account, private=True)

    def _check_key_addr(self, key, addr):
        maddr = '<%s>' %addr
        maddr = maddr.lower()
        for uid in key['uids']:
            uid = uid.lower()
            if uid.find('<') >= 0:
                if uid.find(maddr) >= 0:
                    return
            elif uid == addr:
                return

        txt = 'Account address/alias %s' % addr
        txt += ' not in GPG key %s\nuids: ' % key['fingerprint']
        for uid in key['uids']:
            txt += ' %s' %uid
        raise RemailGPGKeyException(txt)

    def _check_key_expiry(self, key, addr):
        exp = key['expires']
        if not exp:
            return
        now = time.time()
        if now < float(exp):
            return

        txt = 'Account %s' % addr
        txt += ' GPG key %s expired' % key['fingerprint']
        raise RemailGPGKeyException(txt)

    def _check_key(self, keys, account, private=False):
        for key in keys:
            if account.fingerprint != key['fingerprint']:
                continue

            # Validate that it's the right one
            self._check_key_addr(key, account.addr)
            for alias in account.aliases:
                self._check_key_addr(key, alias)
            self._check_key_expiry(key, account.addr)
            return

        if private:
            txt = 'No private key found for %s' % account.addr
        else:
            txt = 'No public key found for %s' % account.addr
        raise RemailGPGKeyException(txt)

    def check_key(self, account):
        self._check_key(self.keys, account)

    def do_encrypt(self, payload, fingerprints):
        ''' Common encryption helper'''

        if self.config.sign:
            signit = self.account.fingerprint
        else:
            signit = None

        enc = self.gpg.encrypt(payload, fingerprints, armor=self.config.armor,
                               always_trust=self.config.always_trust,
                               sign=signit)
        if enc.ok:
            return str(enc)
        raise RemailGPGException('Encryption fail: %s' % enc.status)

    def gpg_encrypt(self, msg, account):
        ''' Encrypt a message for a subscriber. Depending on the
            subscribers preference, inline it or use the enveloped
            version.
        '''

        fingerprints = [str(account.fingerprint)]
        payload = msg.get_payload()

        # GPG inline encryption magic

        # Use plain GPG if requested by accoung and possible
        isplain = msg.get_content_type() == 'text/plain'
        if account.gpg_plain and type(payload) == str and isplain:
            encpl = self.do_encrypt(payload, fingerprints)
            msg_set_payload(msg, msg_from_string(encpl))
            return msg

        # Extract payload for encryption
        payload = msg_get_payload_as_string(msg)
        encpl = self.do_encrypt(payload, fingerprints)
        msg_set_gpg_payload(msg, encpl, account.addr)
        return msg

    def gpg_decrypt_plain(self, msg):
        '''
        Try to decrypt inline plain/text

        If gpg decrypt returns 'no data was provided' treat the
        message as unencrypted plain text.
        '''
        pl = msg.get_payload(decode=True)
        plain = self.gpg.decrypt(pl, always_trust=self.config.always_trust)
        if plain.ok:
            msg_set_payload(msg, msg_from_string(str(plain)))
            if plain.signature_id:
                msg_set_header(msg, 'Signature-Id', plain.username)
        elif plain.status != 'no data was provided':
            # Check for an empty return path which is a good indicator
            # for a mail server message.
            rp = msg.get('Return-path')
            if rp and rp != '<>':
                raise RemailGPGException('Decryption failed: %s' % plain.status)
        return msg

    def gpg_decrypt_enveloped(self, msg):
        '''
        Try to decrypt an enveloped mail
        '''
        contents = msg.get_payload()
        proto = msg.get_param('protocol')
        if proto != 'application/pgp-encrypted':
            raise RemailGPGException('PGP wrong protocol %s' % proto)
        if len(contents) != 2:
            raise RemailGPGException('PGP content length %d' % len(contents))
        ct = contents[0].get_content_type()
        if ct != 'application/pgp-encrypted':
            raise RemailGPGException('PGP wrong app type %s' % ct)
        ct = contents[1].get_content_type()
        if ct != 'application/octet-stream':
            raise RemailGPGException('PGP wrong app type %s' % ct)

        plain = self.gpg.decrypt(contents[1].get_payload(decode=True),
                                 always_trust=self.config.always_trust)
        if not plain.ok:
            raise RemailGPGException('Decryption failed: %s' % plain.status)

        if plain.signature_id:
            msg_set_header(msg, 'Signature-Id', plain.username)

        pl = msg_from_string(str(plain))
        msg_set_payload(msg, pl)
        return msg

    def decrypt(self, msg):
        '''
        Try to handle received mail with PGP. Return decoded or plain mail
        '''
        msgout = msg_from_string(msg.as_string())
        ct = msg.get_content_type()

        if ct == 'text/plain':
            return self.gpg_decrypt_plain(msgout)

        elif ct == 'multipart/encrypted':
            return self.gpg_decrypt_enveloped(msgout)

        # There might be inline PGP with no mentioning in the content type
        if not msg.is_multipart():
            return msgout

        payloads = msgout.get_payload()
        payldecr = []
        for pl in payloads:
            pl = self.decrypt(pl)
            payldecr.append(pl)
        msgout.set_payload(payldecr)
        return msgout

    def encrypt(self, msg, account):
        '''
        Encrypt a message for a recipient
        '''
        return self.gpg_encrypt(msg, account)


class Complex:
    def __init__(self, r, i):
        self.r = r
        self.i = i

    def __add__(self, other):
        ans_r = self.r + other.r
        ans_i = self.i + other.i
        return f'{ans_r} + {ans_i}j'

    def __mul__(self, other):
        ans_r = self.r * other.r - self.i * other.i
        ans_i = self.r * other.i + other.r * self.i
        return f'{ans_r} + {ans_i}j'

val1 = Complex(3, 90)
val2 = Complex(5, 2)
print(val1 + val2)
print(val1 * val2)


import poek as pk

model = pk.model()

N = 10
Rx = list(range(1, N + 1))

x = model.add_variable(index=Rx)
for i in Rx:
    if i % 2 == 1:
        x[i].value = 0
    else:
        x[i].value = -0.5 - i

model.add_objective(0.5 * sum(x[i] ** 2 for i in Rx))

Rc1 = list(range(1, N))

for k in Rc1:
    model.add_constraint(x[k + 1] - x[k] >= -0.5 + (-1) ** k * k)

model.add_constraint(x[1] - x[N] >= N - 0.5)


import re
import sys

import clr

from sunrise import pid
from sunrise.filter import MetaFilter
from sunrise.util import get_file_as_byte

sys.path.append(r"./dll/")
clr.FindAssembly("FiddlerCore4")
clr.AddReference("FiddlerCore4")

import Fiddler

def _before_response_session(session):
    for r in Filter.Rules.values():
        if re.search(r.pattern, session.fullUrl, re.I) is not None:
            session.ResponseBody = get_file_as_byte(r.respond)

def _before_request_session(session):
    if session.LocalProcessID == pid:
        session.bBufferResponse = True

class Filter(MetaFilter):
    def __init__(self):
        Fiddler.CONFIG.IgnoreServerCertErrors = True
        Fiddler.FiddlerApplication.BeforeRequest += _before_request_session
        Fiddler.FiddlerApplication.BeforeResponse += _before_response_session

        if not Fiddler.CertMaker.rootCertIsTrusted():
            Fiddler.CertMaker.createRootCert()
            Fiddler.CertMaker.trustRootCert()

    def startup(self):
        Fiddler.FiddlerApplication.Startup(self.port, Fiddler.FiddlerCoreStartupFlags.Default)

    def shutdown(self):
        Fiddler.FiddlerApplication.Shutdown()


import torch
import torch.nn as nn
import torch.nn.functional as F
from losses import cob_loss, point_loss

def getLoss(loss_name):
    if loss_name == "BCELoss":
        criterion = nn.BCELoss()
    elif loss_name == "CrossEntropy":
        criterion = nn.CrossEntropyLoss()
    elif loss_name == "point_cob":
        criterion = cob_loss.computeLoss
    elif loss_name == "point":
        criterion = point_loss.computeLoss
    elif loss_name == "hybrid":
        criterion = point_loss.computeLoss # or COB Loss
    elif loss_name == "dice":
        criterion = DiceLoss()
    else:
        raise ValueError(f"Loss {loss_name} not implemented.")

    return criterion

class DiceLoss(nn.Module):
    def __init__(self, weight=None, size_average=True):
        super(DiceLoss, self).__init__()

    def forward(self, inputs, targets, smooth=1):

        #comment out if your model contains a sigmoid or equivalent activation layer
        #inputs = F.sigmoid(inputs)

        #flatten label and prediction tensors
        inputs = inputs.reshape(-1)
        targets = targets.reshape(-1)

        intersection = (inputs * targets).sum()
        dice = (2.*intersection + smooth)/(inputs.sum() + targets.sum() + smooth)

        return 1 - dice


import inflect
p = inflect.engine()

class Farm:
    def __init__(self, name):
        self.name = name
        self.animals = {}

    def add_animal(self, animal, number=1):
        if animal in self.animals.keys():
            self.animals[animal] += number
        else:
            self.animals[animal] = number

    def get_info(self):
        output = f"{self.name}'s Farm\n"
        for animal, amount in self.animals.items():
            space = " "* (20 - len(animal))
            if amount > 1:
                animal = p.plural(animal)
            output += f"{animal} {space}: {amount}\n"
        output += " E_I_E_I_O"
        return output

farm = Farm("McDonald")
print(farm.animals)
farm.add_animal("cow", 5)
farm.add_animal("sheep", 2)
print(farm.animals)
print(farm.get_info())

#     def add_animal(animal, amount):
#         self.animals.append({animal : amount})

# macdonald = Farm("MacDonald")
# farm_animals = [("cow", 5),("goat", 12),("Sheep", 2)]

# for animal in farm_animals:
#     macdonald.add_animal()

# print(farm_animals)

# import inflect
# p = inflect.engine()
# class Farm:
#     def __init__(self, name):
#         self.name = name
#         self.animals = {}
#     def add_animal(self, animal, number=1):
#         if animal in self.animals.keys():
#             self.animals[animal] += number
#         else:
#             self.animals[animal] = number
#     def get_info(self):
#         output = f"{self.name}'s Farm\n"
#         for animal, amount in self.animals.items():
#             if amount > 1:
#                 animal = p.plural(animal)
#             space = " " * (10 - len(animal))
#             output += f"{animal} {space} :{amount}\n"
#         output += "    E-I-E-I-O    "
#         return output
#     def get_animal_types(self):
#         return sorted(list(self.animals.keys()))
#     def get_short_info(self):
#         animals = ", ".join(self.get_animal_types())
#         return f"{self.name}'s farm has {animals}."
# farm = Farm("McDonald")
# print(farm.animals)
# farm.add_animal("cow", 5)
# farm.add_animal("sheep")
# farm.add_animal("goat", 4)
# print(farm.animals)
# print(farm.get_info())
# print(farm.get_animal_types())
# print(farm.get_short_info())

Assignments and Daily Challeneges/Week 5/Day 1/Daily Challenge/daily_challenge.py


import random

n = random.randint(1, 100)
range_min = 1
range_max = 100
current_range = (range_min, range_max)

print(f"I'm thinking of a number between {range_min} and {range_max}. Can you guess what it is?")

attempts = 0
while True:
    # Ask the user to guess the numbe
    guess = int(input("Enter your guess: "))
    attempts += 1
    # Check if the guess is correct
    if guess == n:
        print(f"Congratulations, you guessed the number in {attempts} attempts!")
        break
    elif guess < n:
        print("Too low! Try again.")
        # Update the range of possible guesses
        current_range = (guess + 1, current_range[1])
    else:
        print("Too high! Try again.")
        # Update the range of possible guesses
        current_range = (current_range[0], guess - 1)

    # If the user has made 3 consecutive incorrect guesses, give a hint
    if attempts % 3 == 0:
        midpoint = (current_range[0] + current_range[1]) // 2
        print(f"Hint: the number is between {midpoint - midpoint//2} and {midpoint + midpoint//2}")


from strategy_finder.obj_func import *
from opt.hash_reuse import *
from opt.hash_xor import *
from opt.salu_reuse import *
from opt.salu_merge import *
from lib.timeout import TimeoutManager

from lib.inst_list import *
from lib.logging import *

################ O2B tse ################

def o2b_partition(o2a_inst_list, number_list):
    if len(number_list) == 1:
        yield [ number_list ]
        return

    first = number_list[0]

    for smaller in o2b_partition(o2a_inst_list, number_list[1:]):
        for n, subset in enumerate(smaller):
            check_o2a_inst_list = get_list_using_number_list(o2a_inst_list, [ first ] + subset)
            if SALU_merge_check(check_o2a_inst_list):
                yield smaller[:n] + [[ first ] + subset]  + smaller[n+1:]

        yield [[first]] + smaller

# input: list of o2a_inst with the same flowkey and the same index type
# output: list of o2a_inst and o2b_inst
# constraint: find out the best partition to minimize obj func
def greedy_o2b_tse(o2a_inst_list):
    o2b_objective_function_cache = {}

    o2a_inst_list = sort_o2a_inst_by_r(o2a_inst_list)

    o2b_inst_list = []

    number_list = list(range(len(o2a_inst_list)))
    min = float('inf')
    answer_partition = []
    for i, p in enumerate(o2b_partition(o2a_inst_list, number_list), 1):
        partition_candidate = sorted(p)

        sum = 0
        for subset in partition_candidate:
            key = tuple(subset)
            if key in o2b_objective_function_cache:
                val = o2b_objective_function_cache[key]
            else:
                if len(subset) == 1:
                    val = o2a_inst_list[subset[0]].objective_function()['overall']
                else:
                    temp_o2a_inst_list = get_list_using_number_list(o2a_inst_list, subset)
                    o2b_inst = SALUMergeSketchInstance(temp_o2a_inst_list)
                    val =  o2b_inst.objective_function()['overall']

                o2b_objective_function_cache[key] = val
            sum += val

        if min > sum:
            min = sum
            answer_partition = partition_candidate

    for subset in answer_partition:
        if len(subset) == 1:
            o2b_inst_list.append(o2a_inst_list[subset[0]])
        else:
            new_o2a_inst_list = get_list_using_number_list(o2a_inst_list, subset)
            o2b_inst = SALUMergeSketchInstance(new_o2a_inst_list)
            o2b_inst_list.append(o2b_inst)

    return o2b_inst_list

################ O2B greedy ################

def pull_out_o2b_inst(o2a_inst_list):
    if len(o2a_inst_list) == 1:
        return None, o2a_inst_list

    for_o2b_inst_list = []
    remaining_o2a_inst_list = []

    big_o2a_inst = o2a_inst_list[0]

    for_o2b_inst_list.append(big_o2a_inst)

    if big_o2a_inst.get_counter_dp_type() == True:
        for o2a_inst in o2a_inst_list[1:]:
            if SALU_merge_check(for_o2b_inst_list + [o2a_inst]):
                for_o2b_inst_list += [o2a_inst]
            else:
                remaining_o2a_inst_list.append(o2a_inst)
    else:
        for o2a_inst in o2a_inst_list[1:]:
            if o2a_inst.get_counter_dp_type() == True:
                if SALU_merge_check(for_o2b_inst_list + [o2a_inst]):
                    for_o2b_inst_list += [o2a_inst]
                else:
                    remaining_o2a_inst_list.append(o2a_inst)
        for o2a_inst in o2a_inst_list[1:]:
            if o2a_inst.get_counter_dp_type() == False:
                if SALU_merge_check(for_o2b_inst_list + [o2a_inst]):
                    for_o2b_inst_list += [o2a_inst]
                else:
                    remaining_o2a_inst_list.append(o2a_inst)

    if len(for_o2b_inst_list) == 1:
        return None, o2a_inst_list

    o2b_inst = SALUMergeSketchInstance(for_o2b_inst_list)

    return o2b_inst, remaining_o2a_inst_list

def greedy_o2b_greedy(o2a_inst_list):
    o2a_inst_list = sort_o2a_inst_by_r(o2a_inst_list, False)

    o2b_inst_list = []
    new_o2a_inst_list = []

    count = 0
    while True:
        count += 1
        # print(count)
        if len(o2a_inst_list) == 0:
            break
        if len(o2a_inst_list) == 1:
            new_o2a_inst_list.append(o2a_inst_list[0])
            break
        o2b_inst, o2a_inst_list = pull_out_o2b_inst(o2a_inst_list)
        if o2b_inst == None:
            new_o2a_inst_list.append(o2a_inst_list[0])
            o2a_inst_list = o2a_inst_list[1:]
        else:
            o2b_inst_list.append(o2b_inst)

    return o2b_inst_list + new_o2a_inst_list

################ main ################

# input: list of o2a_inst
# output: list of o2a_inst or o2b_inst

def run_O2B_tse(clusters, return_dict):
    print_msg(f"        [greedy O2B tse start]", 6, "cyan")

    o2inst_list = []
    for cluster in clusters:
        result = greedy_o2b_tse(cluster)
        o2inst_list += result

    return_dict["o2inst_list"] = o2inst_list
    print_msg(f"        [greedy O2B tse end]", 6, "cyan")

def run_O2B_greedy(clusters):
    print_msg(f"        [greedy O2B greedy start]", 6, "cyan")
    o2inst_list = []
    for cluster in clusters:
        result = greedy_o2b_greedy(cluster)
        o2inst_list += result
    print_msg(f"        [greedy O2B greedy end]", 6, "cyan")
    return o2inst_list

def greedy_O2B(o2inst_list, timeout):
    print_msg(f"    [greedy O2B start]", 6, "cyan")

    clusters = get_O2B_cluster(o2inst_list)
    timeout_manager = TimeoutManager(timeout, run_O2B_tse, [clusters])
    if timeout > 0 and timeout_manager.start():
        return_dict = timeout_manager.get_return_dict()
        o2inst_list = return_dict["o2inst_list"]
    else:
        o2inst_list = run_O2B_greedy(clusters)

    print_msg(f"    [greedy O2B end]", 6, "cyan")
    return o2inst_list

def o2_greedy_main(inst_list, timeout_O2):
    print_msg("[o2_greedy_main start]", 4, "cyan")

    print_inst_list_msg(inst_list, 8)
    print_msg(f"[before greedy O2A] {len(inst_list)}", 5, "cyan")

    o2a_inst_list = apply_salu_reuse(inst_list)

    print_msg(f"[after greedy O2A] {len(o2a_inst_list)}", 5, "cyan")
    print_answer_list_msg(o2a_inst_list, 8)

    o2inst_list = greedy_O2B(o2a_inst_list, timeout_O2)

    print_msg(f"[after greedy O2B] {len(o2inst_list)}", 5, "cyan")
    print_answer_list_msg(o2inst_list, 8)

    print_msg("[o2_greedy_main end]", 4, "cyan")
    return o2inst_list


import asyncio
import os
import time

import dask
from dask.distributed import Client
from distributed.utils_test import inc
import pytest
import skein

from dask_yarn import YarnCluster
from dask_yarn.core import (
    _make_specification,
    _make_submit_specification,
    _make_scheduler_kwargs,
)
from .conftest import check_is_shutdown

try:
    import bokeh  # noqa

    bokeh_installed = True
except Exception:
    bokeh_installed = False

@pytest.mark.parametrize("deploy_mode", ["remote", "local"])
def test_basic(deploy_mode, skein_client, conda_env):
    with YarnCluster(
        environment=conda_env,
        deploy_mode=deploy_mode,
        worker_memory="512 MiB",
        scheduler_memory="512 MiB",
        name="test-basic",
        skein_client=skein_client,
        dashboard_address=":8787",
        port=8786,
        worker_options={"resources": {"FOO": "BAZ"}},
        worker_class="dask.distributed.Nanny",
    ) as cluster:
        # Smoketest repr

        if bokeh_installed:
            assert cluster.dashboard_link is not None
            if deploy_mode == "local":
                assert ":8787" in cluster.dashboard_link

        if deploy_mode == "local":
            assert ":8786" in cluster.scheduler_address

        # Scale up
        cluster.scale(2)

        with Client(cluster) as client:
            future = client.submit(inc, 10)
            assert future.result() == 11
            client.get_versions(check=True)
            resource_tags = client.run(lambda dask_worker: dask_worker.total_resources)
            assert {"FOO": "BAZ"} in list(resource_tags.values())

        # Check that 2 workers exist
        start = time.time()
        while len(cluster.workers()) != 2:
            time.sleep(0.1)
            assert time.time() < start + 30, "timeout cluster.scale(2)"

        # Scale down
        cluster.scale(1)

        start = time.time()
        while len(cluster.workers()) != 1:
            time.sleep(0.1)
            assert time.time() < start + 30, "timeout cluster.scale(1)"

    check_is_shutdown(skein_client, cluster.app_id)

def test_adapt(skein_client, conda_env):
    with YarnCluster(
        environment=conda_env,
        deploy_mode="local",
        worker_memory="256 MiB",
        name="test-adapt",
        skein_client=skein_client,
    ) as cluster:

        cluster.adapt()

        with Client(cluster) as client:
            future = client.submit(lambda x: x + 1, 10)
            result = future.result()
            assert result == 11

        start = time.time()
        while cluster.workers():
            time.sleep(0.1)
            assert time.time() < start + 30, "auto-scaledown timeout"

    check_is_shutdown(skein_client, cluster.app_id)

def test_from_specification(skein_client, conda_env, tmpdir):
    spec = _make_specification(
        environment=conda_env,
        worker_memory="512 MiB",
        scheduler_memory="512 MiB",
        name="dask-yarn-test-from-specification",
    )
    fn = os.path.join(str(tmpdir), "spec.yaml")
    with open(fn, "w") as f:
        f.write(spec.to_yaml())

    with YarnCluster.from_specification(fn, skein_client=skein_client) as cluster:
        with Client(cluster):
            pass

    check_is_shutdown(skein_client, cluster.app_id)

def test_from_specification_errors():
    bad_spec = skein.ApplicationSpec.from_yaml(
        """
        name: bad_spec
        services:
          bad:
            resources:
              memory: 1 GiB
              vcores: 1
            script: exit 1
        """
    )
    with pytest.raises(ValueError):
        YarnCluster.from_specification(bad_spec)

    with pytest.raises(TypeError):
        YarnCluster.from_specification(object())

def test_from_application_id(skein_client, conda_env):
    with YarnCluster(
        environment=conda_env,
        worker_memory="512 MiB",
        scheduler_memory="512 MiB",
        name="test-from-application-id",
        skein_client=skein_client,
    ) as cluster:

        # Connect to the application with the application id
        cluster2 = YarnCluster.from_application_id(cluster.app_id, skein_client)

        cluster2.scale(1)

        start = time.time()
        while len(cluster2.workers()) != 1:
            time.sleep(0.1)
            assert time.time() < start + 30, "timeout cluster.scale(1)"

        del cluster2

        # Cluster is still running, finalizer not run in cluster2
        assert len(cluster.workers()) == 1

    check_is_shutdown(skein_client, cluster.app_id)

def test_from_current(skein_client, conda_env, monkeypatch, tmpdir):
    # Not running in a container
    with pytest.raises(ValueError) as exc:
        YarnCluster.from_current()
    assert str(exc.value) == "Not running inside a container"

    with YarnCluster(
        environment=conda_env,
        worker_memory="512 MiB",
        scheduler_memory="512 MiB",
        name="test-from-current",
        skein_client=skein_client,
    ) as cluster:

        # Patch environment so it looks like a container
        container_id = "container_1526134340424_0012_01_000005"
        cont_dir = tmpdir.mkdir(container_id)
        with open(str(cont_dir.join(".skein.crt")), "wb") as fil:
            fil.write(skein_client.security._get_bytes("cert"))
        with open(str(cont_dir.join(".skein.pem")), "wb") as fil:
            fil.write(skein_client.security._get_bytes("key"))

        for key, val in [
            ("SKEIN_APPLICATION_ID", cluster.app_id),
            ("CONTAINER_ID", container_id),
            ("SKEIN_APPMASTER_ADDRESS", cluster.application_client.address),
            ("LOCAL_DIRS", str(tmpdir)),
        ]:
            monkeypatch.setenv(key, val)

        import skein.core

        monkeypatch.setattr(skein.core, "properties", skein.core.Properties())

        cluster2 = YarnCluster.from_current()
        assert cluster2.app_id == cluster.app_id
        assert cluster2.scheduler_address == cluster.scheduler_address

        # Smoketest method
        cluster2.scale(1)

        start = time.time()
        while len(cluster2.workers()) != 1:
            time.sleep(0.1)
            assert time.time() < start + 30, "timeout cluster.scale(1)"

        del cluster2

        # Cluster is still running, finalizer not run in cluster2
        assert len(cluster.workers()) == 1

    check_is_shutdown(skein_client, cluster.app_id)

@pytest.mark.parametrize("deploy_mode", ["remote", "local"])
def test_configuration(deploy_mode):
    config = {
        "yarn": {
            "environment": "myenv.tar.gz",
            "queue": "myqueue",
            "name": "dask-yarn-tests",
            "user": "alice",
            "tags": ["a", "b", "c"],
            "specification": None,
            "deploy-mode": deploy_mode,
            "worker": {
                "memory": "1234 MiB",
                "count": 1,
                "vcores": 1,
                "restarts": -1,
                "gpus": 2,
                "env": {"foo": "bar"},
                "worker_class": "abc",
                "worker_options": {"FOO": "BAZ"},
            },
            "scheduler": {"memory": "1234 MiB", "vcores": 1, "gpus": 0},
            "host": "0.0.0.0",
            "port": 8786,
            "dashboard_address": ":8787",
        }
    }

    with dask.config.set(config):
        spec = _make_specification()
        assert spec.name == "dask-yarn-tests"
        assert spec.user == "alice"
        assert spec.queue == "myqueue"
        assert spec.tags == {"a", "b", "c"}
        assert spec.services["dask.worker"].resources.memory == 1234
        assert spec.services["dask.worker"].resources.gpus == 2
        assert spec.services["dask.worker"].env == {
            "foo": "bar",
            "worker_class": "abc",
            "worker_options": "eyJGT08iOiAiQkFaIn0=",
        }

        if deploy_mode == "remote":
            assert spec.services["dask.scheduler"].resources.memory == 1234
            assert spec.services["dask.scheduler"].resources.gpus == 0
        else:
            assert "dask.scheduler" not in spec.services

        kwargs = _make_scheduler_kwargs()
        assert kwargs["host"] == "0.0.0.0"
        assert kwargs["port"] == 8786
        assert kwargs["dashboard_address"] == ":8787"

def test_configuration_full_specification(conda_env, tmpdir):
    spec = _make_specification(
        environment=conda_env,
        worker_memory="512 MiB",
        scheduler_memory="512 MiB",
        name="test-configuration-full-specification",
    )
    fn = os.path.join(str(tmpdir), "spec.yaml")
    with open(fn, "w") as f:
        f.write(spec.to_yaml())

    # path to full specification
    with dask.config.set({"yarn": {"specification": fn}}):
        spec = _make_specification()
        assert spec == spec

    # full specification inlined
    with dask.config.set({"yarn": {"specification": spec.to_dict()}}):
        spec = _make_specification()
        assert spec == spec

    # when overrides specified, ignores full specification
    override = {"specification": spec.to_dict()}
    override.update(dask.config.get("yarn"))
    override["name"] = "config-name"
    with dask.config.set({"yarn": override}):
        spec = _make_specification(environment="foo", name="test-name", n_workers=4)
        assert spec.services["dask.worker"].instances == 4
        assert spec.name == "test-name"

def test_make_specification_errors():
    with dask.config.set({"yarn.environment": None}):
        with pytest.raises(ValueError) as info:
            _make_specification()
        assert "conda-pack" in str(info.value)

    with pytest.raises(ValueError) as info:
        _make_specification(environment="foo.tar.gz", deploy_mode="unknown")
    assert "deploy_mode" in str(info.value)

    with pytest.raises(ValueError) as info:
        _make_specification(environment="foo.tar.gz", worker_memory=1234)
    assert "1234 MiB" in str(info.value)

    with pytest.raises(ValueError) as info:
        _make_specification(environment="foo.tar.gz", scheduler_memory=1234)
    assert "1234 MiB" in str(info.value)

@pytest.mark.parametrize(
    "env,path", [("conda://env-name", "env-name"), ("conda:///env/path", "/env/path")]
)
def test_environment_conda(env, path):
    spec = _make_submit_specification(
        "script.py", args=("foo", "bar"), deploy_mode="remote", environment=env
    )
    scheduler = spec.services["dask.scheduler"]
    assert not scheduler.files
    assert (
        scheduler.script == ("conda activate %s\ndask-yarn services scheduler") % path
    )
    worker = spec.services["dask.worker"]
    assert not worker.files
    assert worker.script == ("conda activate %s\ndask-yarn services worker") % path
    client = spec.services["dask.client"]
    assert set(client.files) == {"script.py"}
    assert (
        client.script
        == ("conda activate %s\ndask-yarn services client script.py foo bar") % path
    )

def test_environment_venv():
    env = "venv:///path/to/environment"
    path = "/path/to/environment"
    spec = _make_submit_specification(
        "script.py", args=("foo", "bar"), deploy_mode="remote", environment=env
    )
    scheduler = spec.services["dask.scheduler"]
    assert not scheduler.files
    assert (
        scheduler.script
        == ("source %s/bin/activate\ndask-yarn services scheduler") % path
    )
    worker = spec.services["dask.worker"]
    assert not worker.files
    assert worker.script == ("source %s/bin/activate\ndask-yarn services worker") % path
    client = spec.services["dask.client"]
    assert set(client.files) == {"script.py"}
    assert (
        client.script
        == ("source %s/bin/activate\ndask-yarn services client script.py foo bar")
        % path
    )

def test_environment_python_path():
    env = "python:///path/to/python"
    cmd = "/path/to/python -m dask_yarn.cli"
    spec = _make_submit_specification(
        "script.py", args=("foo", "bar"), deploy_mode="remote", environment=env
    )
    scheduler = spec.services["dask.scheduler"]
    assert not scheduler.files
    assert scheduler.script == "%s services scheduler" % cmd
    worker = spec.services["dask.worker"]
    assert not worker.files
    assert worker.script == "%s services worker" % cmd
    client = spec.services["dask.client"]
    assert set(client.files) == {"script.py"}
    assert client.script == "%s services client script.py foo bar" % cmd

def test_environment_archive():
    env = "env.tar.gz"
    spec = _make_submit_specification(
        "script.py", args=("foo", "bar"), deploy_mode="remote", environment=env
    )
    scheduler = spec.services["dask.scheduler"]
    assert set(scheduler.files) == {"environment"}
    assert scheduler.script == (
        "source environment/bin/activate\n"
        "environment/bin/python -m dask_yarn.cli services scheduler"
    )
    worker = spec.services["dask.worker"]
    assert set(worker.files) == {"environment"}
    assert worker.script == (
        "source environment/bin/activate\n"
        "environment/bin/python -m dask_yarn.cli services worker"
    )
    client = spec.services["dask.client"]
    assert set(client.files) == {"environment", "script.py"}
    assert client.script == (
        "source environment/bin/activate\n"
        "environment/bin/python -m dask_yarn.cli services client script.py foo bar"
    )

@pytest.mark.parametrize("deploy_mode", ["local", "remote"])
def test_make_submit_specification(deploy_mode):
    spec = _make_submit_specification(
        "../script.py",
        deploy_mode=deploy_mode,
        environment="myenv.tar.gz",
        name="test-name",
        user="alice",
        client_vcores=2,
        client_memory="2 GiB",
    )

    assert spec.name == "test-name"
    assert spec.user == "alice"
    if deploy_mode == "local":
        assert set(spec.services) == {"dask.worker"}
        assert "environment" in spec.services["dask.worker"].files
    else:
        client = spec.services["dask.client"]
        scheduler = spec.services["dask.scheduler"]
        assert client.files["environment"] == scheduler.files["environment"]
        assert client.files["script.py"].source.startswith("file:///")
        assert client.resources.memory == 2048
        assert client.resources.vcores == 2

    config = {
        "yarn.name": "dask-yarn-tests",
        "yarn.queue": "myqueue",
        "yarn.environment": "myenv.tar.gz",
        "yarn.client.memory": "1234 MiB",
        "yarn.client.vcores": 2,
        "yarn.client.env": {"foo": "bar"},
    }

    with dask.config.set(config):
        spec = _make_submit_specification("script.py", deploy_mode=deploy_mode)
        assert spec.name == "dask-yarn-tests"
        assert spec.queue == "myqueue"
        if deploy_mode == "remote":
            assert spec.services["dask.client"].resources.memory == 1234
            assert spec.services["dask.client"].resources.vcores == 2
            assert spec.services["dask.client"].env == {"foo": "bar"}

def test_environment_relative_paths():
    relpath = "path/to/foo.tar.gz"
    abspath = os.path.abspath(relpath)
    a = _make_specification(environment=relpath).to_dict()
    b = _make_specification(environment=abspath).to_dict()
    assert a == b

@pytest.mark.asyncio
@pytest.mark.parametrize("deploy_mode", ["remote", "local"])
async def test_basic_async(deploy_mode, skein_client, conda_env):
    async with YarnCluster(
        environment=conda_env,
        deploy_mode=deploy_mode,
        worker_memory="512 MiB",
        scheduler_memory="512 MiB",
        name="test-basic-async",
        skein_client=skein_client,
        asynchronous=True,
    ) as cluster:
        # Smoketest repr
        repr(cluster)

        if bokeh_installed:
            assert cluster.dashboard_link is not None

        # Scale up
        await cluster.scale(2)

        async with Client(cluster, asynchronous=True) as client:
            result = await client.submit(inc, 10)
            assert result == 11
            await client.get_versions(check=True)

        # Check that 2 workers exist
        start = time.time()
        while len(await cluster.workers()) != 2:
            await asyncio.sleep(0.1)
            assert time.time() < start + 30, "timeout cluster.scale(2)"

        # Scale down
        await cluster.scale(1)

        start = time.time()
        while len(await cluster.workers()) != 1:
            await asyncio.sleep(0.1)
            assert time.time() < start + 30, "timeout cluster.scale(1)"

    check_is_shutdown(skein_client, cluster.app_id)

def test_widget_and_html_reprs(skein_client, conda_env):
    pytest.importorskip("ipywidgets")

    with YarnCluster(
        environment=conda_env,
        deploy_mode="local",
        worker_memory="256 MiB",
        name="test-widget",
        skein_client=skein_client,
    ) as cluster:
        # Smoke test widget
        cluster._widget()

        # Test non-widget html repr
        assert cluster.app_id in cluster._repr_html_()

        assert "0" in cluster._widget_status()

        # Scale up and wait
        cluster.scale(1)
        start = time.time()
        while len(cluster._observed) != 1:
            time.sleep(0.1)
            assert time.time() < start + 30, "timeout cluster.scale(1)"

        assert "1" in cluster._widget_status()

        # Scale down
        cluster.scale(1)

    check_is_shutdown(skein_client, cluster.app_id)

def test_logs(conda_env, skein_client):
    with YarnCluster(
        environment=conda_env,
        deploy_mode="local",
        worker_memory="256 MiB",
        name="test-widget",
        skein_client=skein_client,
    ) as cluster:
        cluster.scale(2)
        start = time.time()
        while len(cluster._observed) != 2:
            time.sleep(0.1)
            assert time.time() < start + 30, "timeout cluster.scale(2)"

        logs = cluster.logs()
        assert len(logs) == 3

        logs = cluster.logs(scheduler=True, workers=False)
        assert len(logs) == 1

        logs = cluster.logs(scheduler=False, workers=False)
        assert len(logs) == 0

        logs = cluster.logs(scheduler=False, workers=True)
        assert len(logs) == 2

    check_is_shutdown(skein_client, cluster.app_id)


#!/usr/bin/env python

import os

os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"

class Test:
    def __init__(self, data_path, process_path, input_shape, label_shape, channels, n_classes, save_name, save_path):
        self.data_path = data_path
        self.process_path = process_path
        self.input_shape = input_shape
        self.label_shape = label_shape
        self.channels = channels
        self.n_classes = n_classes
        self.save_name = save_name
        self.save_path = save_path

    def test(self):
        # # [0] configure
        # data_path = 'lvmvm/'
        # process_path = 'processed/lvmvm/'
        # input_shape = (50, 128, 128, 4)
        # label_shape = (50, 128, 128, 1)
        # channels = 4
        # n_classes = 2
        # save_name = 'ear3d_1000e_2b.hdf5'
        # save_path = 'processed/lvmvm/pred/'

        # # [1] process data
        # from mriutils.datasets.lvmvm import LoadH5
        # lh5 = LoadH5(self.data_path, self.process_path)
        # data = lh5.read()
        # lh5.save()

        # [2] load data
        from mriutils.utils.load_data import LoadData
        ld = LoadData(self.process_path)
        ld.load_data_dict()
        train, test, mode_list = ld.data_split()

        from mriutils.utils.resize import Resize
        res = Resize(train, 'data', self.input_shape)
        train = res.run()
        res = Resize(test, 'label', self.label_shape)
        test = res.run()

        # [3] build the model
        from mriutils.models.ear3d import EAR3D
        model = EAR3D(self.input_shape, model_level = 2, n_classes = 2, loss = 'dice', metric = 'dice')

        # [4] train the model
        from mriutils.train import Train
        training = Train(model, train, self.channels, self.n_classes)
        training.train(mode_list[0], mode_list[1], epochs = 1000, batch_size = 2, save_name = self.save_name)

        # [5] test the model
        from mriutils.test import Test
        testing = Test(model, self.save_name, self.channels, self.n_classes, test)
        testing.test(mode_list[0], mode_list[1], self.save_path)


from sell.models import Sell
from django import forms

class SellForm(forms.ModelForm):
    class Meta:
        model = Sell
        fields = "__all__"
        widgets = {
            'sell_date': forms.DateInput(attrs={'type': 'date'}),
            'sell_note': forms.Textarea(attrs={'rows': '3'})
        }