import numpy as np

from gpaw import debug

class Symmetry:

    def __init__(self, id_a, cell_cv, pbc_c=np.ones(3, bool), tolerance=1e-11):
        """Symmetry object.

        Two atoms can only be identical if they have the same atomic numbers,
        setup types and magnetic moments. If it is an LCAO type of calculation,
        they must have the same atomic basis set also.

        """

        self.id_a = id_a
        self.cell_cv = np.array(cell_cv, float)
        assert self.cell_cv.shape == (3, 3)
        self.pbc_c = np.array(pbc_c, bool)
        self.tol = tolerance

        self.op_scc = np.identity(3, int).reshape((1, 3, 3))
        self.a_sa = np.arange(len(id_a)).reshape((1, -1))

    def analyze(self, spos_ac):
        """Determine list of symmetry operations.

        First determine all symmetry operations of the cell. Then call
        ``prune_symmetries`` to remove those symmetries that are not satisfied
        by the atoms.

        """

        # Symmetry operations as matrices in 123 basis
        self.op_scc = []

        # Metric tensor
        metric_cc = np.dot(self.cell_cv, self.cell_cv.T)

        # Generate all possible 3x3 symmetry matrices using base-3 integers
        power = (6561, 2187, 729, 243, 81, 27, 9, 3, 1)

        # operation is a 3x3 matrix, with possible elements -1, 0, 1, thus
        # there are 3**9 = 19683 possible matrices
        for base3id in xrange(19683):
            op_cc = np.empty((3, 3), dtype=int)
            m = base3id
            for ip, p in enumerate(power):
                d, m = divmod(m, p)
                op_cc[ip // 3, ip % 3] = 1 - d

            # The metric of the cell should be conserved after applying
            # the operation
            opmetric_cc = np.dot(np.dot(op_cc, metric_cc), op_cc.T)

            if np.abs(metric_cc - opmetric_cc).sum() > self.tol:
                continue

            # Operation must not swap axes that are not both periodic
            pbc_cc = np.logical_and.outer(self.pbc_c, self.pbc_c)
            if op_cc[~(pbc_cc | np.identity(3, bool))].any():
                continue

            # Operation must not invert axes that are not periodic
            pbc_cc = np.logical_and.outer(self.pbc_c, self.pbc_c)
            if not (op_cc[np.diag(~self.pbc_c)] == 1).all():
                continue

            # operation is a valid symmetry of the unit cell
            self.op_scc.append(op_cc)

        self.op_scc = np.array(self.op_scc)

        # Check if symmetry operations are also valid when taking account
        # of atomic positions
        self.prune_symmetries(spos_ac)

    def prune_symmetries(self, spos_ac):
        """Remove symmetries that are not satisfied by the atoms."""

        # Build lists of (atom number, scaled position) tuples.  One
        # list for each combination of atomic number, setup type,
        # magnetic moment and basis set
        species = {}
        for a, id in enumerate(self.id_a):
            spos_c = spos_ac[a]
            if id in species:
                species[id].append((a, spos_c))
            else:
                species[id] = [(a, spos_c)]

        opok = []
        a_sa = []
        # Reduce point group using operation matrices
        for op_cc in self.op_scc:
            map = np.zeros(len(spos_ac), int)
            ok = True
            for specie in species.values():
                for a1, spos1_c in specie:
                    spos1_c = np.dot(spos1_c, op_cc)
                    ok = False
                    for a2, spos2_c in specie:
                        sdiff = spos1_c - spos2_c
                        sdiff -= np.floor(sdiff + 0.5)
                        if np.dot(sdiff, sdiff) < self.tol:
                            ok = True
                            map[a1] = a2
                            break
                    if not ok:
                        break
                if not ok:
                    break
            if ok:
                opok.append(op_cc)
                a_sa.append(map)

        if debug:
            for op_cc, map_a in zip(opok, a_sa):
                for a1, id1 in enumerate(self.id_a):
                    a2 = map_a[a1]
                    assert id1 == self.id_a[a2]
                    spos1_c = spos_ac[a1]
                    spos2_c = spos_ac[a2]
                    sdiff = np.dot(spos1_c, op_cc) - spos2_c
                    sdiff -= np.floor(sdiff + 0.5)
                    assert np.dot(sdiff, sdiff) < self.tol

        self.a_sa = a_sa
        self.op_scc = np.array(opok)

    def check(self, spos_ac):
        """Check if positions satisfy symmetry operations."""

        nsymold = len(self.op_scc)
        self.prune_symmetries(spos_ac)
        if len(self.op_scc) < nsymold:
            raise RuntimeError('Broken symmetry!')

    def reduce(self, bzk_kc):
        """Reduce k-points to irreducible part of the BZ.

        Returns the irreducible k-points and the weights.

        """
        nbzkpts = len(bzk_kc)
        ibzk0_kc = np.empty((nbzkpts, 3))
        ibzk_kc = ibzk0_kc[:0]
        weight_k = np.ones(nbzkpts)
        nibzkpts = 0

        # Mapping between k and symmetry related point in the irreducible BZ
        kibz_k = np.empty(nbzkpts, int)
        # Symmetry operation mapping the k-point in the irreducible BZ to k
        sym_k = np.empty(nbzkpts, int)
        # Time-reversal symmetry used on top of the point group operation
        time_reversal_k = np.zeros(nbzkpts, dtype=bool)

        for i, op_cc in enumerate(self.op_scc):
            if (np.abs(op_cc - np.eye(3,dtype=int)) < 1e-8).all():
                opI = i
                break

        kbz = nbzkpts
        for k_c in bzk_kc[::-1]:
            kbz -= 1
            found = False

            for s, op_cc in enumerate(self.op_scc):
                if len(ibzk_kc) == 0:
                    break
                diff_kc = np.dot(ibzk_kc, op_cc.T) - k_c
                b_k = ((diff_kc - diff_kc.round())**2).sum(1) < self.tol
                if not b_k.any():
                    # Try with time reversal:
                    diff_kc += 2 * k_c
                    b_k = ((diff_kc - diff_kc.round())**2).sum(1) < self.tol
                    if not b_k.any():
                        continue
                    else:
                        time_reversal_k[kbz] = True

                found = True
                kibz = np.where(b_k)[0][0]
                weight_k[kibz] += 1.0
                kibz_k[kbz] = kibz
                sym_k[kbz] = s
                break

            if not found:
                kibz_k[kbz] = nibzkpts
                sym_k[kbz] = opI
                nibzkpts += 1
                ibzk_kc = ibzk0_kc[:nibzkpts]
                ibzk_kc[-1] = k_c

        # Reverse order (looks more natural)
        return (ibzk_kc[::-1].copy(),
                weight_k[:nibzkpts][::-1] / nbzkpts,
                sym_k, time_reversal_k, nibzkpts - 1 - kibz_k)

    def prune_symmetries_grid(self, N_c):
        """Remove symmetries that are not satisfied by the grid."""

        U_scc = []
        a_sa = []
        for U_cc, a_a in zip(self.op_scc, self.a_sa):
            if not (U_cc * N_c - (U_cc.T * N_c).T).any():
                U_scc.append(U_cc)
                a_sa.append(a_a)

        self.a_sa = np.array(a_sa)
        self.op_scc = np.array(U_scc)

    def symmetrize(self, a, gd):
        """Symmetrize array."""

        gd.symmetrize(a, self.op_scc)

    def symmetrize_wavefunction(self, a_g, kibz_c, kbz_c, op_cc,
                                time_reversal):
        """Generate Bloch function from symmetry related function in the IBZ.

        a_g: ndarray
            Array with Bloch function from the irreducible BZ.
        kibz_c: ndarray
            Corresponing k-point coordinates.
        kbz_c: ndarray
            K-point coordinates of the symmetry related k-point.
        op_cc: ndarray
            Point group operation connecting the two k-points.
        time-reversal: bool
            Time-reversal symmetry required in addition to the point group
            symmetry to connect the two k-points.

        """

        # Identity
        if (np.abs(op_cc - np.eye(3, dtype=int)) < 1e-10).all():
            if time_reversal:
                return a_g.conj()
            else:
                return a_g
        # Inversion symmetry
        elif (np.abs(op_cc + np.eye(3, dtype=int)) < 1e-10).all():
            return a_g.conj()
        # General point group symmetry
        else:
            import _gpaw
            b_g = np.zeros_like(a_g)
            if time_reversal:
                # assert abs(np.dot(op_cc, kibz_c) - -kbz_c) < tol
                _gpaw.symmetrize_wavefunction(a_g, b_g, op_cc.T.copy(),
                                              kibz_c, -kbz_c)
                return b_g.conj()
            else:
                # assert abs(np.dot(op_cc, kibz_c) - kbz_c) < tol
                _gpaw.symmetrize_wavefunction(a_g, b_g, op_cc.T.copy(),
                                              kibz_c, kbz_c)
                return b_g

    def symmetrize_forces(self, F0_av):
        """Symmetrice forces."""

        F_ac = np.zeros_like(F0_av)
        for map_a, op_cc in zip(self.a_sa, self.op_scc):
            op_vv = np.dot(np.linalg.inv(self.cell_cv),
                           np.dot(op_cc, self.cell_cv))
            for a1, a2 in enumerate(map_a):
                F_ac[a2] += np.dot(F0_av[a1], op_vv)
        return F_ac / len(self.op_scc)

    def print_symmetries(self, text):

        n = len(self.op_scc)
        text('Symmetries present: %s' % n)


import pickle as pic
import networkx as nx

from cassiopeia.TreeSolver.utilities import convert_network_to_newick_format
from cassiopeia.TreeSolver.post_process_tree import post_process_tree
from cassiopeia.TreeSolver.simulation_tools.simulation_utils import get_leaves_of_tree
import random
import numpy as np

import copy

from tqdm import tqdm
import hashlib

class Cassiopeia_Tree:
	"""
	An abstract class for trees used in Cassiopeia. Basically a wrapper for a networkx file and a newick string that can be used to interface
	with other applications.

	Attributes:
		- method: the algorithm used to reconstruct this tree.
		- name: name of the tree
		- network: a networkx object representing the tree
		- newick: the newick string corresponding to the tree.
		- cm: the character matrix used as input to the tree solver.
		- alternative_solutions: a list of networkx objects, each of which corresponds to an alternative solution

	Methods:
		- dump_network: write out networkx object to a .pkl file
		- dump_newick: write out the newick file as text
		- get_network: retrieve the networkx object
		- get_newick: retrieve the newick text as a string
		- get_targets: get target nodes in tree
		- post_process: post-process the tree by adding cells with same allele onto tree; also make depth uniform and remove non-sample leaves.
		- score_parsimony: get the parsimony of the tree.
		- generate_triplet: obtain a random triplet from the tree.
		- find_triplet_structure: find the mrca of the triplet (used to score triplets)
		- get_leaves: get the leaves of the tree.

	"""

	def __init__(self, method, name = None, network = None, newick = None, character_matrix = None, alternative_solutions = None, base_network = None):
		"""
		Initialize the Cassiopeia_Tree object.

		:param method:
			Algorithm used to reconstruct the tree
		:param name:
			Name of the tree.
		:param network:
			Networkx object representing the tree.
		:param newick:
			Newick string for tree.
		:param character_matrix:
			character matrix used as input for reconstructing the tree.
		:param alternative_solutions:
			a list of alternative solutions (each of which is a networkx object)

		:return:
			None
		"""

		assert network is not None or newick is not None

		assert method in ['greedy', 'hybrid', 'ilp', 'cassiopeia', 'camin-sokal', 'neighbor-joining', 'simulated', 'from_newick']

		self.name = name
		self.method = method
		self.network = network
		self.newick = newick
		self.cm = character_matrix
		self.alternative_solutions = alternative_solutions
		self.base_network = base_network

	def dump_network(self, output_name):
		"""
		Write network as a pickle file.

		:param output_name:
			File path to write to.
		:return:
			None
		"""

		if not self.network:
			self.network = newick_to_network(self.newick)

		pic.dump(self.network, open(output_name, "wb"))

	def dump_newick(self, output_name):
		"""
		Write newick string to text.

		:param output_name:
			File path to write to.
		:return:
			None
		"""

		if not self.newick:
			self.newick = convert_network_to_newick_format(self.network)

		with open(output_name, "w") as f:
			f.write(self.newick)

	def get_network(self):
		"""
		Get networkx object.

		:return:
			Networkx object.
		"""

		if not self.network:
			self.network = newick_to_network(self.newick)

		return self.network

	def get_newick(self):
		"""
		Get newick string.

		:return:
			Newick string.
		"""

		if not self.newick:
			self.newick = convert_network_to_newick_format(self.network)

		return self.newick

	def get_targets(self):
		"""
		Get targets of tree (as determined by Node.is_target boolean).

		:return:
			List of target Nodes.
		"""

		if not self.network:
			self.network = newick_to_network(self.newick)

		return [n for n in self.network if n.is_target]

	def collapse_edges(self):

		def _collapse(graph, edges_to_collapse):

			new_network = nx.DiGraph()
			root = [n for n in graph if graph.in_degree(n) == 0][0]
			for n in nx.dfs_postorder_nodes(graph, source=root):
				if n == root:
					continue
				edge = (list(graph.predecessors(n))[0], n)
				if edge[0].get_character_vec() == edge[1].get_character_vec():
					for p in graph.predecessors(edge[0]):
						new_network.add_edge(p, edge[1])
				else:
					new_network.add_edge(edge[0], edge[1])

			return new_network

		def find_edges_to_collapse(graph):
			edges = []
			source = [n for n in graph if graph.in_degree(n) == 0][0]

			for e in nx.dfs_edges(graph, source=source):
				if e[0].get_character_vec() == e[1].get_character_vec():
					edges.append(e)
			return edges[::-1]

		net = self.network
		root = [n for n in net if net.in_degree(n) == 0][0]

		edges_to_collapse = find_edges_to_collapse(net)
		while len(edges_to_collapse) > 0:
			net = _collapse(net)
			edges_to_collapse = find_edges_to_collapse(net)

		to_remove = []
		for n in net.nodes:
			if net.in_degree(n) == 0 and n != root:
				to_remove.append(n)

		net.remove_nodes_from(to_remove)

		self.network = net
		self.newick = convert_network_to_newick_format(self.network)

	def post_process(self, cm = None):
		"""
		Post process the tree by:
			- adding in non-unique samples as leaves
			- pruning off leaves that are not targets
			- Adding 'dummy' edges to make the depth of the tree uniform.

		:return:
			A Cassiopeia_Tree that is post-processed.
		"""

		if cm is not None:
			self.cm = cm

		assert self.cm is not None

		net = self.get_network().copy()
		copy_dict = {}
		for n in net:
			copy_dict[n] = copy.copy(n)

		net = nx.relabel_nodes(net, copy_dict)

		G = post_process_tree(net, self.cm.copy(), self.method)

		return Cassiopeia_Tree(self.method, network=G)

	def score_parsimony(self, cm = None):
		"""
		Score the parsimony of the tree.

		:param cm:
			Character matrix, if the Cassiopeia_Tree object does not already have one stored.

		:return:
			An integer representing the number of mutations in the tree.
		"""

		if cm is not None:
			self.cm = cm

		assert self.cm is not None

		net = self.get_network().copy()
		copy_dict = {}
		for n in net:
			copy_dict[n] = copy.copy(n)

		net = nx.relabel_nodes(net, copy_dict)

		#net = fill_in_tree(net, cm)
		#net = tree_collapse(net)

		root = [n for n in net if net.in_degree(n) == 0][0]

		score = 0
		for e in nx.dfs_edges(net, source=root):
			score += e[0].get_mut_length(e[1])

		return score

	def score_likelihood(self, priors):

		net = self.network
		root = [n for n in net if net.in_degree(n) == 0][0]

		score = 0
		for e in nx.dfs_edges(net, source=root):
			score += e[0].get_mut_length(e[1], priors=priors)

		return score

	def generate_triplet(self, targets = None):
		"""
		Generate a random triplet of targets in the tree.

		:param targets:
			Targets to choose from. If this is None, select randomly from the leaves of the tree (these should be
			targets themselves).

		:return:
			A list of Nodes corresponding to the triplet.
		"""

		if targets is None:

			targets = get_leaves_of_tree(self.network)

		a = random.choice(targets)
		target_nodes_original_network_copy = list(targets)
		target_nodes_original_network_copy.remove(a)
		b = random.choice(target_nodes_original_network_copy)
		target_nodes_original_network_copy.remove(b)
		c = random.choice(target_nodes_original_network_copy)

		return [a, b, c]

	def find_triplet_structure(self, triplet):
		"""
		Find the structure of the triplet -- i.e. are A and B more closely related to one another or B and C?

		:param triplet:
			A list of Nodes (length = 3)

		:return:
			The structure of the triplet and the minimum number of ancestors that overlap between each Node in the triplet.
		"""

		a, b, c = None, None, None

		for n in self.network.nodes:
			if n.char_string == triplet[0].char_string:
				a = n
			if n.char_string == triplet[1].char_string:
				b = n
			if n.char_string == triplet[2].char_string:
				c = n

		a_ancestors = [node for node in nx.ancestors(self.network, a)]
		b_ancestors = [node for node in nx.ancestors(self.network, b)]
		c_ancestors = [node for node in nx.ancestors(self.network, c)]
		ab_common = len(set(a_ancestors) & set(b_ancestors))
		ac_common = len(set(a_ancestors) & set(c_ancestors))
		bc_common = len(set(b_ancestors) & set(c_ancestors))
		index = min(ab_common, bc_common, ac_common)

		true_common = '-'
		if ab_common > bc_common and ab_common > ac_common:
			true_common = 'ab'
		elif ac_common > bc_common and ac_common > ab_common:
			true_common = 'ac'
		elif bc_common > ab_common and bc_common > ac_common:
			true_common = 'bc'

		return true_common, index

	def get_leaves(self):
		"""
		Given a tree, returns all leaf nodes with/without their identifiers

		:param tree:
			networkx tree generated from the simulation process
		:param clip_identifier:
			Boolean flag to remove _identifier from node names
		:return:
			List of leaves of the corresponding tree in string format
		"""

		tree = self.network

		return [n for n in tree if tree.out_degree(n) == 0 and tree.in_degree(n) == 1]
		# source = [x for x in tree.nodes() if tree.in_degree(x)==0][0]

		# max_depth = max(nx.shortest_path_length(tree,source,node) for node in tree.nodes())
		# shortest_paths = nx.shortest_path_length(tree,source)

		# return [x for x in tree.nodes() if tree.out_degree(x)==0 and tree.in_degree(x) == 1 and shortest_paths[x] == max_depth]

	def sample_alternative_solutions(self, maximum_alt_solutions = 100):

		alt_solutions = []

		target_charstrings = [(n.char_string, n.pid) for n in self.network if n.is_target]

		num_solutions = np.prod([len(self.alternative_solutions[r]) for r in self.alternative_solutions.keys()])

		num_considered_solutions = 0
		sol_identifiers = []  # keep track of solutions already sampled

		# we'll sample maximum_alt_solutions from the set of possible solutions
		pbar = tqdm(
			total=maximum_alt_solutions, desc="Enumerating alternative solutions"
		)

		while num_considered_solutions < min(num_solutions, maximum_alt_solutions):

			current_sol = []
			for r in self.alternative_solutions.keys():
				res_list = self.alternative_solutions[r]
				current_sol.append(np.random.choice(len(res_list)))

			if tuple(current_sol) not in sol_identifiers:

				new_network = self.base_network.copy()

				for i, r in zip(current_sol, self.alternative_solutions.keys()):

					sub_net = self.alternative_solutions[r][i]

					# reattach alternative solution
					r_parent = None
					r_parents = [p for p in self.base_network.predecessors(r)]
					if len(r_parents) > 0:
						r_parent = r_parents[0]

					rn = [n for n in sub_net if sub_net.in_degree(n) == 0][0]

					new_network.add_edges_from(sub_net.edges(data=True))
					new_network.add_nodes_from(sub_net.nodes(data=True))

					if r_parent != None:
						new_network.add_edge(r_parent, rn)

				# assign targets
				seen_charstrings = [n.char_string for n in self.base_network if n.is_target]
				for n in new_network:

					if (n.char_string, n.pid) in target_charstrings and n.char_string not in seen_charstrings:
						n.is_target = True
						seen_charstrings.append(n.char_string)

				alt_solutions.append(new_network)

				sol_identifiers.append(tuple(current_sol))
				num_considered_solutions += 1

				pbar.update(1)  # update progress bar

		return alt_solutions


from matplotlib.ticker import PercentFormatter
import matplotlib.pyplot as plt
import statsmodels . api as sm
import statsmodels.stats.diagnostic as smd
import statsmodels.stats.stattools as smt
import pandas as pd
import os
import numpy as np
import scipy as sp

def folder_definer(folder):
    cwd = os.getcwd()
    PATH = cwd + "/"+folder+"/"

    if not os.path.exists(PATH):
        os.mkdir(PATH)

    return PATH

def plotbar(P,SavePath, one_value = 0.01, five_value = 0.05,
            ten_value = 0.1):

    """/3_p_value_plots/"""
    variable = P.name
    P = pd.DataFrame(data = P, columns = [variable])
    mean = P.loc['Mean', variable]
    P['stock_names'] = P.index

    def bar_highlight(value, one_value,
                      five_value,
                      ten_value,
                      mean):
        if value <= one_value:
            return 'red'
        elif value <= five_value:
            return 'orange'
        elif value <= ten_value:
            return 'gold'
        if value == mean:
            return 'black'
        else:
            return 'grey'
    fig, ax = plt.subplots()

    P['colors'] = P[variable].apply(bar_highlight, args = (one_value,
                          five_value,
                          ten_value,
                          mean))

    bars = plt.bar(P['stock_names'], P[variable], color=P['colors'])
    x_pos = range(P['stock_names'].shape[0])
    plt.xticks(x_pos, P['stock_names'], rotation=90)

    variable = variable.replace(":","_")
    plt.savefig(folder_definer(SavePath)+"/"+variable+".png")
    plt.show()

    plt.close()

def plotCAPM(rStock,Market,stock_names,OLSRes,SavePath):
    """/2_testCAPM/"""
    for e in stock_names:

        plt.figure()
        plt.plot(Market, OLSRes.loc[e, 'beta: Market']*Market+OLSRes.loc[e, 'Alpha'])
        plt.scatter(Market,rStock[e])
        plt . xlabel ('Eurostoxx')
        plt . ylabel (e)
        plt.savefig(folder_definer(SavePath[0])+"/"+SavePath[1]+"-"+e+SavePath[2]+".png")
        plt.close()

"""
- setx = market return
- set y = stock returns
- title = title of the graph
- xlabel = name of the x-axis
- ylabel = name of the y-axis
- sigla = unique name (either ER for euribor or ERB for bunds) to differentiate
          between plots of excess returns of stocks computed with Euribor
          or with the bund yield.
- Subset = dataframe with the data of total index returns
- string_to_save = name of the folder in which the graph will be saved in the
                   working directory
"""

def plotscatter(setx,sety,title,xlabel,ylabel,sigla,SavePath):

    l = pd.DataFrame(index = sety.columns, columns= ['Plot'])

    for e in sety.columns:

        ax1 = plt.figure().add_subplot()

        ax1.scatter(setx,sety[e])
        plt.title(title)
        plt . xlabel (xlabel)
        plt . ylabel (e+ylabel)
        plt.savefig(folder_definer(SavePath[0])+"/"+SavePath[1]+"-"+e+"_"+sigla+".png")
        l.loc[e,'Plot'] = plt.figure()
        plt.close()

    return l

def f_test_retrieval(l):

    df = pd.DataFrame(columns = ['F-Test_Value','F-Test_p-value'])

    for i in range(len(l)):

        name = l[i].model.endog_names
        df.loc[name, 'F-Test_Value'] = l[i].fvalue
        df.loc[name, 'F-Test_p-value'] = l[i].f_pvalue

    return df

def f_test_retrieval_2(l):

    critical_alpha = l[l.iloc[:,1] < 0.05].iloc[:,1]

    l = CAPM_list

    df = pd.DataFrame(columns = [critical_alpha.name, 'F-Test_p-value'],
                      index = critical_alpha.index)

    df.loc[:,critical_alpha.name] = critical_alpha
    r = list(critical_alpha.index)

    for i in range(len(l)):

        name = l[i].model.endog_names

        if name in r:

            df.loc[name, 'F-Test_p-value'] = l[i].f_pvalue

    return df

def OLS(y, *x, hac =False, conf_int = False):

    intercept = pd.DataFrame(data = np.ones(y.shape[0] ),
                              columns = ["intercept"],
                              index = y.index)

    X = pd.concat([intercept,*x],axis = 1)

    exog_names = list(X.columns)

    l = ['Alpha', 'p-value_alpha']

    for i in range(1, len(exog_names)):

        l.append("beta: " + exog_names[i])
        l.append("p-value_beta: "+ exog_names[i])

    l.append("R-Squared")
    l.append('bic')
    l.append('aic')

    if conf_int:

        l.append('LBound')
        l.append('UBound')

    endog_names = list(y.columns)
    result = pd.DataFrame(index = endog_names, columns = l)

    reg = []

    for i in endog_names:

        Res1 = sm . OLS ( y[i] ,X). fit ()
        Res1.summary()

        if hac == True:

            #Checking for heteroskedasticity
            residuals = Res1.resid
            exogen = Res1.model.exog
            het = smd.het_white(residuals, exogen)
            ind = smd.acorr_breusch_godfrey(Res1, nlags = 1)

            if (het[3] < 0.05) and (ind[3] <0.05):
                Res1 = sm . OLS ( y[i] ,X). fit (cov_type ='HAC',cov_kwds= {'maxlags':1})
                #print('HAAAAAAAAAAAC: {}'.format(i))

            elif het[3] < 0.05:

                Res1 = sm . OLS ( y[i] ,X). fit (cov_type ='HC3')
                #print('HETEROOOOOOO: {}'.format(i) )

            elif ind[3] < 0.05:
                Res1 = sm . OLS ( y[i] ,X). fit (cov_type ='HAC',cov_kwds= {'maxlags':1})
                #print('SERIAL CORRELAAAAATION: {}'.format(i))

        r2 = Res1.rsquared
        bic = Res1.bic
        aic = Res1.aic
        param = Res1.params
        pval = Res1.pvalues
        reg.append(Res1)

        if conf_int:

            intervals = Res1.conf_int(alpha = 0.05)
            intervals = intervals.loc['Market', :]

        l_val = []

        for j in range(len(param)):

            l_val.extend([param[j],pval[j]])

        l_val.append(r2)
        l_val.append(bic)
        l_val.append(aic)

        if conf_int:

            l_val.append(intervals[0])
            l_val.append(intervals[1])

        result.loc[i] = l_val

    return result, reg

def comparison_scatter(df_stocks, df_portfolios, market,
                       title,xlabel,ylabel,
                       SavePath,CAPM_Port=None):
    ax1 = plt.figure().add_subplot()

    for i in df_stocks.columns:
        ax1.scatter(market, df_stocks.loc[:,i],
                    c = 'silver', alpha = 0.5)
    plt.title(title)
    plt . xlabel (xlabel)
    plt . ylabel (ylabel)

    ax1.scatter(market, df_portfolios, c = 'black')

    if CAPM_Port is not None:
        ax1.plot(market, CAPM_Port.loc['Portfolio - EW','beta: Market']*
                market+CAPM_Port.loc['Portfolio - EW','Alpha'])

    plt.savefig(folder_definer(SavePath[0])+"/"+SavePath[1]+"-"+i+".png")
    plt.show()

def m_scatter(CAPM_summary, df_factors, df_stocks,
              SavePath):
    """
    /2_scatter_comparison/
    """

    x = list(CAPM_summary.index)
    y = x[:3] + x[-1:-4:-1]

    figure, axis = plt.subplots(2, 3)

    for i in range(3):

        axis[0,i].scatter(df_factors['Market'],
                                df_stocks.loc[:,y[i]])

    for i in range(3,6):
        axis[1,i-3].scatter(df_factors['Market'],
                                df_stocks.loc[:,y[i]])

    plt.savefig(folder_definer(SavePath[0])+"/"+SavePath[1]+".png")

    plt.show()

def RESET_test(l):
    df = pd.DataFrame(columns= ['F-Value', 'p-value'])
    results = []
    for i in range(len(l)):
        l_val = []
        x = l[i]
        x.fittedvalues = np.array(x.fittedvalues)
        f = smd.linear_reset(res = x , power = 3, test_type = "fitted", use_f = True)
        l_val.append(f.fvalue)
        l_val.append(f.pvalue)
        df.loc[l[i].model.endog_names,:] = l_val
        results.append(f)
    return df

def h_test(l):

    df = pd.DataFrame(columns= ['F-Value', 'p-value'])

    for i in range(len(l)):

        l_val = []
        residuals = l[i].resid
        exogen = l[i].model.exog
        f = smd.het_white(residuals, exogen)
        l_val.append(f[2])
        l_val.append(f[3])
        df.loc[l[i].model.endog_names,:] = l_val

    return df

def Durbin_Watson_test(l):

    df = pd.DataFrame(columns= ["Test-statistic"])

    for i in range(len(l)):

        l_val = []

        residuals = l[i].resid.copy()
        residuals = np.array(residuals)

        f = smt.durbin_watson(residuals)
        l_val.append(f)

        df.loc[l[i].model.endog_names,:] = l_val

    return df

def Breusch_Godfrey_test(l):

    df = pd.DataFrame(columns= ['F-Value', 'p-value'])

    for i in range(len(l)):

        l_val = []

        f = smd.acorr_breusch_godfrey(l[i], nlags = 3)

        l_val.append(f[2])
        l_val.append(f[3])

        df.loc[l[i].model.endog_names,:] = l_val

    return df

def comparison_barplot(FF_summary, CAPM_summary):

    name = 'empty'

    print('The possible comparisons are: {}\n'.format(list(FF_summary.columns)))
    print("(Insert 0 to stop)\n")
    name = input('Which one would you like to compare?\n')

    index = np.arange(FF_summary.index.shape[0])
    bar_width = 0.35

    fig, ax = plt.subplots()
    summer = ax.bar(index, FF_summary.loc[:,name], bar_width,
                    label="Fama_French")

    winter = ax.bar(index + bar_width, CAPM_summary.loc[:,name], bar_width,
                    label="CAPM")

    ax.set_xlabel('Company')
    ax.set_ylabel('Value')
    ax.set_title('Comparison between CAPM and Fama-French model: {}'.format(name))

    x_pos = range(FF_summary.index.shape[0])
    plt.xticks(x_pos, FF_summary.index, rotation=90)

    ax.set_xticklabels(FF_summary.index)
    ax.legend()

    plt.show()

def GETS_BIC(FF_summary, df_factors, df_stocks):

    elim = []

    summary = FF_summary.copy()
    df_fac = df_factors.copy()

    bic = summary.loc['Mean', 'bic']
    bic_list = [bic]

    results = []

    while True:

        p_v = [i for i in list(summary.columns) if ('p-value' in i)]

        del p_v[0:2]
        names = [j[-3:] for j in p_v ]

        temp_df = pd.DataFrame(index = names, columns = ['p-values'])

        p_values = []

        for i in p_v:

            p_values.append(summary.loc['Mean', i])

        if p_values:

            temp_df.iloc[:,0] = p_values
            temp_df = temp_df.sort_values('p-values', ascending = False)

            if temp_df.iloc[0,0] > 0.05:

                elim.append(temp_df.index[0])
                df_fac = df_fac.drop(elim[-1], axis = 1)

                summary, FF_list_2 = OLS(df_stocks,df_fac, hac = True)
                summary.loc['Mean'] = summary.mean()
                results.append(summary)

                if summary.loc['Mean', 'bic'] < bic:
                    bic = summary.loc['Mean', 'bic']
                    bic_list.append(bic)

                else:
                    bic = summary.loc['Mean', 'bic']
                    bic_list.append(bic)
                    results.pop()
                    break

            else:
                break

        else:
            break

    return results[-1],bic_list

def GETS_AIC(FF_summary, df_factors, df_stocks):

    elim = []

    summary = FF_summary.copy()
    df_fac = df_factors.copy()

    aic = summary.loc['Mean', 'aic']
    aic_list = [aic]

    results = []

    while True:

        p_v = [i for i in list(summary.columns) if ('p-value' in i)]

        del p_v[0:2]
        names = [j[-3:] for j in p_v ]

        temp_df = pd.DataFrame(index = names, columns = ['p-values'])

        p_values = []

        for i in p_v:

            p_values.append(summary.loc['Mean', i])

        if p_values:

            temp_df.iloc[:,0] = p_values
            temp_df = temp_df.sort_values('p-values', ascending = False)

            if temp_df.iloc[0,0] > 0.05:

                elim.append(temp_df.index[0])
                df_fac = df_fac.drop(elim[-1], axis = 1)

                summary, FF_list_2 = OLS(df_stocks,df_fac, hac = True)
                summary.loc['Mean'] = summary.mean()
                results.append(summary)

                if summary.loc['Mean', 'aic'] < aic:
                    aic = summary.loc['Mean', 'aic']
                    aic_list.append(aic)

                else:
                    aic = summary.loc['Mean', 'aic']
                    aic_list.append(aic)
                    results.pop()
                    break

            else:
                break

        else:
            break

    return results[-1],aic_list

def GETS_BIC_worst(FF_summary, df_factors, df_stocks):

    elim = []

    summary = FF_summary.copy()
    df_fac = df_factors.copy()

    bic = max(summary.loc[:, 'bic'])
    bic_list = [bic]

    results = []

    while True:

        p_v = [i for i in list(summary.columns) if ('p-value' in i)]

        del p_v[0:2]
        names = [j[-3:] for j in p_v ]

        temp_df = pd.DataFrame(index = names, columns = ['p-values'])

        p_values = []

        for i in p_v:

            p_values.append(summary.loc['Mean', i])

        if p_values:

            temp_df.iloc[:,0] = p_values
            temp_df = temp_df.sort_values('p-values', ascending = False)

            if temp_df.iloc[0,0] > 0.05:

                elim.append(temp_df.index[0])
                df_fac = df_fac.drop(elim[-1], axis = 1)

                summary, FF_list_2 = OLS(df_stocks,df_fac, hac = True)
                summary.loc['Mean'] = summary.mean()
                results.append(summary)

                if max(summary.loc[:, 'bic']) < bic:
                    bic = max(summary.loc[:, 'bic'])
                    bic_list.append(bic)

                else:
                    bic = max(summary.loc[:, 'bic'])
                    bic_list.append(bic)
                    results.pop()
                    break

            else:
                break

        else:
            break

    return results[-1],bic_list

def GETS_BIC_p(FF_summary, df_factors, df_stocks):

    elim = []

    summary = FF_summary.copy()
    df_fac = df_factors.copy()

    bic = summary.loc[summary.index[0], 'bic']
    bic_list = [bic]

    results = []

    while True:

        p_v = [i for i in list(summary.columns) if ('p-value' in i)]

        del p_v[0]
        names = [j[-3:] for j in p_v[1:] ]
        """
        ----------------------------------------- HARD-CODED
        """
        names.insert(0, 'Market')

        temp_df = pd.DataFrame(index = names, columns = ['p-values'])

        p_values = []

        for i in p_v:

            p_values.append(summary.loc[summary.index[0], i])

        if p_values:

            temp_df.iloc[:,0] = p_values
            temp_df = temp_df.sort_values('p-values', ascending = False)

            if temp_df.iloc[0,0] > 0.05:

                elim.append(temp_df.index[0])
                df_fac = df_fac.drop(elim[-1], axis = 1)

                summary, FF_list_2 = OLS(df_stocks,df_fac, hac = True)

                results.append(summary)

                if summary.loc[summary.index[0], 'bic'] < bic:
                    bic = summary.loc[summary.index[0], 'bic']
                    bic_list.append(bic)

                else:
                    bic = summary.loc[summary.index[0], 'bic']
                    bic_list.append(bic)
                    results.pop()
                    break

            else:
                break

        else:
            break

    return results[-1],bic_list

def ad_hoc_GETS(FF_summary, df_factors, df_stocks):

    df = pd.DataFrame(index = FF_summary.index, columns = FF_summary.columns)
    final_models = []

    for i in df.index:

        print(i)

        if i == 'Mean':

            continue

        #Create an object to be compatible with the function GETS_BIC_p
        df_2 = pd.DataFrame( columns = FF_summary.columns)
        df_2.loc[len(df_2)] = FF_summary.loc[i,:].values

        df_3 = pd.DataFrame(index = df_stocks.index)
        df_3[i] = df_stocks.loc[:,i].values

        res, bic = GETS_BIC_p(df_2, df_factors,df_3)
        final_models.append(res)

    return final_models

def CHOW_TEST(df_stocks, df_factors):
    sub = 0.2
    prop = int(sub*df_stocks.shape[0])

    if prop <= 20:
        prop = 21

    end = df_stocks.shape[0] - prop

    index = df_stocks.index[(prop-1):]
    index = index[:-(prop)]

    p_val_df = pd.DataFrame(columns = df_stocks.columns, index = index)

    hac_check = True

    while prop <= end:

        prop_df = df_stocks.iloc[:prop,:]
        compl_df = df_stocks.iloc[prop:,:]

        if df_factors.ndim == 1:

            df_factors = df_factors.to_frame()
            hac_check = False

        prop_factors = df_factors.iloc[:prop, :]
        prop_factors = prop_factors.loc[:,'Market']

        compl_factors = df_factors.iloc[prop:, :]
        compl_factors = compl_factors.loc[:,'Market']

        prop_summary, prop_reg = OLS(prop_df, prop_factors, hac = hac_check)

        compl_summary, compl_reg = OLS(compl_df, compl_factors, hac = hac_check)

        total_summary, total_reg = OLS(df_stocks, df_factors.loc[:,'Market'], hac = hac_check)

        for i in range(len(prop_reg)):

            if (prop_reg[i].model.endog_names ==
                compl_reg[i].model.endog_names) and (prop_reg[i].model.endog_names ==
                                                     total_reg[i].model.endog_names):

                RSS_prop = prop_reg[i].ssr
                RSS_compl = compl_reg[i].ssr
                RSS_tot = total_reg[i].ssr

                F = ((RSS_tot - RSS_prop - RSS_compl)/2)/ ((RSS_prop + RSS_compl)/(df_stocks.shape[0] - 2*2))

                p_value = 1 - sp.stats.f.cdf(F, 2, df_stocks.shape[0] - 2*2)

                p_val_df.loc[prop_df.index[-1], prop_reg[i].model.endog_names] = p_value

            else:
                print("PROBLEM")

        prop += 1

    return p_val_df

def CHOW_TEST_FF(df_stocks, df_factors):
    sub = 0.2
    prop = int(sub*df_stocks.shape[0])

    if prop <= 20:
        prop = 21

    end = df_stocks.shape[0] - prop

    index = df_stocks.index[(prop-1):]
    index = index[:-(prop)]

    p_val_df = pd.DataFrame(columns = df_stocks.columns, index = index)

    while prop <= end:

        prop_df = df_stocks.iloc[:prop,:]

        prop_factors = df_factors.iloc[:prop, :]

        compl_df = df_stocks.iloc[prop:,:]

        compl_factors = df_factors.iloc[prop:, :]

        prop_summary, prop_reg = OLS(prop_df, prop_factors, hac = True)

        compl_summary, compl_reg = OLS(compl_df, compl_factors, hac = True)

        total_summary, total_reg = OLS(df_stocks, df_factors, hac = True)

        k = len(prop_factors.columns) + 1

        for i in range(len(prop_reg)):

            if (prop_reg[i].model.endog_names ==
                compl_reg[i].model.endog_names) and (prop_reg[i].model.endog_names ==
                                                     total_reg[i].model.endog_names):

                RSS_prop = prop_reg[i].ssr
                RSS_compl = compl_reg[i].ssr
                RSS_tot = total_reg[i].ssr

                F = ((RSS_tot - RSS_prop - RSS_compl)/k)/ ((RSS_prop + RSS_compl)/(df_stocks.shape[0] - 2*k))

                p_value = 1 - sp.stats.f.cdf(F, 2, df_stocks.shape[0] - 2*2)

                p_val_df.loc[prop_df.index[-1], prop_reg[i].model.endog_names] = p_value

            else:
                print("PROBLEM")

        prop += 1

    return p_val_df

def CAPM_break_dates(p_val_df, CAPM_summary,
                     df_stocks, df_factors):
    min_pval = []
    index_min = []

    break_dates_df = pd.DataFrame(index = p_val_df.columns, columns = ['min_pval', 'date'])

    for i in p_val_df.columns:

        min_pval = min(p_val_df.loc[:,i])
        index_min = p_val_df.loc[:,i].idxmin()
        break_dates_df.loc[i,:] = [min_pval, index_min]

    import Official_Experiment_Function_10

    d2 = {}

    l_col = []
    l_col = l_col + list(CAPM_summary.columns) + ['beg_date', 'end_date']

    for i in df_stocks.columns:

        d2[i] = pd.DataFrame(columns = l_col)

    no_break_stocks = []

    no_break_stocks = break_dates_df[break_dates_df['min_pval'] > 0.05].index.to_list()

    break_dates_df = break_dates_df[break_dates_df['min_pval'] < 0.05]

    for i in no_break_stocks:

        d2[i] = pd.concat([d2[i],CAPM_summary.loc[i].to_frame().T],
                            ignore_index = True)
        d2[i]['beg_date'] = df_stocks.index[0]
        d2[i]['end_date'] = df_stocks.index[-1]

    for name in break_dates_df.index:
        break_date = 1

        d2[name] = pd.DataFrame(columns = l_col)
        start_date = df_stocks.index[0]

        break_date = break_dates_df.loc[name, 'date']

        i=0

        while break_date != 0:
            reg_summary, reg_list = OLS(df_stocks.loc[start_date:break_date, name].to_frame(),
                                           df_factors.loc[start_date:break_date, 'Market'],
                                           hac = True)
            """
            Storing the model before the break
            """

            final_res = OLS(df_stocks.loc[start_date:break_date, name].to_frame(),
                                       df_factors.loc[start_date:break_date, 'Market'])

            d2[name] = pd.concat([d2[name], final_res[0]], ignore_index = True)

            d2[name].iloc[i,-2] = df_stocks.loc[start_date:break_date].index[0]
            d2[name].iloc[i,-1] = df_stocks.loc[start_date:break_date].index[-1]

            """
            DO WE HAVE TO CHECK FOR BREAKS EVEN BEFORE THE BREAK???????????????????????????
            """
            #p_val_df_2 = CHOW_TEST(df_stocks.loc[:break_date, name].to_frame(),
                                         #df_factors.loc[:break_date, 'Market'])

            """
            ----------------------------------------------------------------------------------------
            ----------------------------------------------------------------------------------------
            """

            """
            Checking for further breaks
            """

            p_val_df_2 = CHOW_TEST(df_stocks.loc[break_date:, name].to_frame(),
                                         df_factors.loc[break_date:, 'Market'])

            if p_val_df_2.empty:

                i = i + 1
                reg_summary, reg_list = OLS(df_stocks.loc[break_date:, name].to_frame(),
                                               df_factors.loc[break_date:, 'Market'])

                d2[name] = pd.concat([d2[name], reg_summary], ignore_index = True)

                d2[name].iloc[i,-2] = df_stocks.loc[break_date:].index[0]
                d2[name].iloc[i, -1] = df_stocks.loc[break_date:].index[-1]
                break_date = 0

            elif (min(p_val_df_2[name]) > 0.05) :

                i = i + 1
                reg_summary, reg_list = OLS(df_stocks.loc[break_date:, name].to_frame(),
                                               df_factors.loc[break_date:, 'Market'])

                d2[name] = pd.concat([d2[name], reg_summary], ignore_index = True)

                d2[name].iloc[i,-2] = df_stocks.loc[break_date:].index[0]
                d2[name].iloc[i, -1] = df_stocks.loc[break_date:].index[-1]
                break_date = 0

            else:
                start_date = break_date
                break_date = p_val_df_2[name].idxmin()

            i = i+1

    return d2

def break_dates_optimization(p_val_df_FF, FF_summary, df_stocks, df_factors):
    """
    PROCEDURE TO FIND BREAK DATES
    """

    """
    First we create a dataframe that, for each stock, assign the min_pval of the
    Chow test and the corresponding date that corresponds to that minimum value
    """

    min_pval = []
    index_min = []

    break_dates_df_FF = pd.DataFrame(index = p_val_df_FF.columns, columns = ['min_pval', 'date'])

    for i in p_val_df_FF.columns:

        min_pval = min(p_val_df_FF.loc[:,i])
        index_min = p_val_df_FF.loc[:,i].idxmin()
        break_dates_df_FF.loc[i,:] = [min_pval, index_min]

    """
    -------------------------------------------------------------------------------
    Estimating optimized models for which no structural break was detected
    -------------------------------------------------------------------------------
    """

    """
    We create a dictionary in which we will store the results of the models
    estimated and optimized in each interval according to their break dates.
    These results will be stored in dataframes.
    If a stock doesn't have any break there will be a dataframe composed of a single
    row.
    If a stock show the presence of one or more breaks, it will have a number of rows
    equal to the number of breaks detected plus one.
    """
    d = {}

    l_col = []
    l_col = l_col + list(FF_summary.columns) + ['beg_date', 'end_date']

    for i in df_stocks.columns:

        d[i] = pd.DataFrame(columns = l_col)

    #Finding the stocks for which the FF model with all the factors didn't show breaks
    list_to_GETS = break_dates_df_FF[break_dates_df_FF['min_pval'] > 0.05].index.to_list()

    """
    Removing those stocks from the dataframe in which we stored the date and p-value
    of critical dates
    """
    break_dates_df_FF = break_dates_df_FF[break_dates_df_FF['min_pval'] < 0.05]

    """
    Optimizing by removing irrelevant variables for the models of the stocks that
    didn't show any breaks
    """
    final_res = ad_hoc_GETS(FF_summary.loc[list_to_GETS],
                               df_factors, df_stocks[list_to_GETS])

    """
    Storing the results of these models in a dataframe
    """

    final_res_df= pd.DataFrame( columns = l_col)

    for i in range(len(final_res)):
        final_res_df = pd.concat([final_res_df, final_res[i]], axis = 0)

    final_res_df['beg_date'] = df_stocks.index[0]
    final_res_df['end_date'] = df_stocks.index[-1]

    """
    Storing these dataframes in the dictionary
    """

    for i in range(len(final_res)):

        name = final_res[i].index[0]
        d[name] = pd.concat([d[name],final_res_df.loc[name].to_frame().T],
                            ignore_index = True)

    """
    Stocks that have breaks
    """

    for name in break_dates_df_FF.index:
        break_date = 1

        d[name] = pd.DataFrame(columns = l_col)
        start_date = df_stocks.index[0]

        break_date = break_dates_df_FF.loc[name, 'date']

        i=0

        while break_date != 0:
            reg_summary, reg_list = OLS(df_stocks.loc[start_date:break_date, name].to_frame(),
                                           df_factors[start_date:break_date], hac = True)
            """
            Storing the model before the break after removing irrelevant variables
            """

            final_res = ad_hoc_GETS(reg_summary,
                                       df_factors[start_date:break_date],
                                       df_stocks.loc[start_date:break_date, name].to_frame())
            d[name] = pd.concat([d[name], final_res[0]], ignore_index = True)

            d[name].iloc[i,-2] = df_stocks.loc[start_date:break_date].index[0]
            d[name].iloc[i,-1] = df_stocks.loc[start_date:break_date].index[-1]

            """
            DO WE HAVE TO CHECK FOR BREAKS EVEN BEFORE THE BREAK???????????????????????????
            """
            p_val_df_2 = CHOW_TEST_FF(df_stocks.loc[:break_date, name].to_frame(),
                                         df_factors[:break_date])

            """
            Checking for further breaks
            """

            p_val_df_2 = CHOW_TEST_FF(df_stocks.loc[break_date:, name].to_frame(),
                                         df_factors[break_date:])

            if p_val_df_2.empty:

                i = i + 1
                reg_summary, reg_list = OLS(df_stocks.loc[break_date:, name].to_frame(),
                                               df_factors[break_date:], hac = True)

                final_res = ad_hoc_GETS(reg_summary,
                                           df_factors[break_date:],
                                           df_stocks.loc[break_date:, name].to_frame())

                d[name] = pd.concat([d[name], final_res[0]], ignore_index = True)

                d[name].iloc[i,-2] = df_stocks.loc[break_date:].index[0]
                d[name].iloc[i, -1] = df_stocks.loc[break_date:].index[-1]
                break_date = 0

            elif (min(p_val_df_2[name]) > 0.05) :

                i = i + 1
                reg_summary, reg_list = OLS(df_stocks.loc[break_date:, name].to_frame(),
                                               df_factors[break_date:], hac = True)

                final_res = ad_hoc_GETS(reg_summary,
                                           df_factors[break_date:],
                                           df_stocks.loc[break_date:, name].to_frame())

                d[name] = pd.concat([d[name], final_res[0]], ignore_index = True)

                d[name].iloc[i,-2] = df_stocks.loc[break_date:].index[0]
                d[name].iloc[i, -1] = df_stocks.loc[break_date:].index[-1]
                break_date = 0

            else:
                start_date = break_date
                break_date = p_val_df_2[name].idxmin()

            i = i+1

    return d


import sys
import os.path

HERE0 = os.path.dirname(__file__) or os.curdir
os.chdir(HERE0)
HERE = os.curdir
sys.path.insert(0, HERE)

from setuptools import setup, find_packages

# -----------------------------------------------------------------
# CONFIGURATIONS:
# -----------------------------------------------------------------
python_version = float("%s.%s" % sys.version_info[:2])
CONCURRENTSPEC = os.path.join(HERE, "src")
README = os.path.join(HERE, "README.md")
long_description = "".join(open(README).readlines())

# -----------------------------------------------------------------
# UTILITIES:
# -----------------------------------------------------------------
def find_packages_by_root_package(where):
    """
    Better than excluding everything that is not needed,
    collect only what is needed.
    """
    root_package = os.path.basename(where)
    packages = [ "%s.%s" % (root_package, sub_package)
                 for sub_package in find_packages(where)]
    packages.insert(0, root_package)
    return packages

setup(
    name = 'cSpec',
    version = '1.0.0.alpha',
    description = 'A behavior-driven development tool using Python',
    long_description=long_description,
    author = 'Kun-Che Li, Bing-Yun Wang, Che-Hung Tsai',
    author_email = 'milkkuan@gmail.com',
    url = 'https://github.com/benny870704/concurrentSpec_research',
    packages=find_packages_by_root_package(CONCURRENTSPEC),
    entry_points = {
        "console_scripts": [
            "cspec = src.main:main"
        ]
    }
)


"""
SettingsFrame Module
"""
import tkinter as tk
import tkinter.messagebox as messagebox
import inspect
from openaiclient.view.frame.baseframe import BaseFrame

class SettingsFrame(BaseFrame):
    """
    Class SettingsFrame:

    This class is for a frame that holds settings for
    a user to change. Created to work with the Completion
    and Edit requests to OpenAI.
    """

    def __init__(self, main, controller):
        super().__init__(main, controller)

        self._PADX = 10
        self._PADY = 5

        # variable to hold the settings we will be using
        # has a default value for testing.
        self.settings = {
            "testKey": "testVal",
            "testIntKey": 1,
            "testBoolKey": True,
            "testFloatKey": 1.0,
            "testNotInOptions": None
        }
        # hold the tk variables for returning inputs from the frame
        self.outputs = dict()

        # row count
        self.row = 0

    # creates all of the widgets for
    # the frame. This default method
    # can be overriden for different
    # types of input in other settings
    # classes.
    def create(self):
        """
        Creates the settings input frames by looping through the settings
        dictionary (self.settings) and creating a SettingsInputFrame object
        for each setting. It then attaches each frame in the next row on
        the frame with the grid() method.

        Finally, it creates the save and exit buttons and attaches them to
        the settings frame (self).
        """
        # key is the label, value is the value in the
        # dictionary of settings
        for key, value in self.settings.items():
            # get the name of the input type
            # and make the first letter lowercase
            typeOfValue = type(value).__name__
            typeOfValue = typeOfValue.replace(
                typeOfValue[0],
                typeOfValue[0].lower(),
                1
            )

            try:
                # get the correct setting function
                # based on value type
                func = getattr(self, typeOfValue+"Setting")
            except AttributeError:
                # skip the current attribute.
                # later, this will be logged into a logger
                print(f"No widget of type {typeOfValue} defined.")
                continue

            # get label and widget from function
            setting = func(key, value)

            # attach label
            setting.label.grid(
                column=0,
                row=self.row,
                padx=10,
                pady=5,
                sticky=tk.E
            )

            # attach widget
            setting.widget.grid(
                column=1,
                row=self.row,
                padx=self._PADX,
                pady=self._PADY,
                sticky=tk.W
            )

            # move down to the next row
            self.row += 1

        self.saveAndExitButtons()

        return self

    def saveSettings(self):
        """saves the settings inputted in the window"""
        for key in self.settings:
            try:
                val = self.outputs[key].get()
                self.setAttr(key, val)
            except KeyError:
                pass
            except tk.TclError as error:
                messagebox.showerror(
                    f"Incorrect input in {key}", f"Key \"{key}\" {str(error)}")
            except RuntimeError as error:
                messagebox.showerror(
                    f"Incorrect input in {key}", f"Key {str(error)}")

        self.exitSettings()

    def setAttr(self, key, val):
        """This should be overridden. Returns setter for saveSettings() method"""
        raise NotImplementedError

    def exitSettings(self):
        """exits the window without savings the changes"""
        self.master.destroy()

    def saveAndExitButtons(self):
        """Creates the save and exit button for the frame"""

        frame = tk.Frame(self)

        # button for savings settings
        tk.Button(
            master=frame,
            text="Save",
            command=self.saveSettings
        ).grid(
            column=0,
            row=0,
            padx=self._PADX,
            pady=self._PADY,
            ipadx=40
        )

        # button to close the window
        tk.Button(
            master=frame,
            text="Exit",
            command=self.exitSettings,
        ).grid(
            column=1,
            row=0,
            padx=self._PADX,
            pady=self._PADY,
            ipadx=40
        )

        frame.grid(
            column=0,
            row=self.row,
            columnspan=2,
            padx=self._PADX,
            pady=self._PADY
        )

    # string setting input
    def strSetting(self, key, value):
        """
        Creates a string setting input with the Entry object.
        """
        setting = self.Setting()

        setting.label = tk.Label(
            master=self,
            text=key
        )

        tkFunc = tk.Entry

        kwargs = self._kwargs(tkFunc, tk.StringVar, "textvariable", key)

        setting.widget = tkFunc(
            self,
            **kwargs
        )

        return setting

    # integer setting input
    def intSetting(self, key, value):
        """
        Creates an integer setting input with the Entry object.
        """
        setting = self.Setting()

        setting.label = tk.Label(
            master=self,
            text=key
        )

        tkFunc = tk.Entry

        kwargs = self._kwargs(tkFunc, tk.IntVar, "textvariable", key)

        setting.widget = tkFunc(
            self,
            **kwargs
        )

        return setting

    # float setting input
    def floatSetting(self, key, value):
        """
        Creates an integer setting input with the Entry object.
        """
        setting = self.Setting()

        setting.label = tk.Label(
            master=self,
            text=key
        )

        tkFunc = tk.Entry

        kwargs = self._kwargs(tkFunc, tk.DoubleVar, "textvariable", key)

        setting.widget = tkFunc(
            self,
            **kwargs
        )

        return setting

    # boolean setting input
    def boolSetting(self, key, value):
        """
        Creates a boolean setting input with on and off values as 1 and 0 and
        the input as a checkbutton.
        """
        setting = self.Setting()
        kwargs = dict()

        kwargs["onvalue"] = 1
        kwargs["offvalue"] = 0

        setting.label = tk.Label(
            master=self,
            text=key
        )

        tkFunc = tk.Checkbutton

        kwargs = self._kwargs(tkFunc, tk.IntVar, "variable", key, kwargs)

        setting.widget = tkFunc(
            self,
            **kwargs
        )

        return setting

    def _kwargs(self, tkFunc, tkVar, varKey, key, kwargs={}):
        """Sets up the kwargs for a setting input"""
        # add varKey: tkVar() to the kwargs
        kwargs = kwargs | {varKey: tkVar()}

        # point the output dictionary value key to
        # the tkVar() we just set up
        self.outputs[key] = kwargs[varKey]

        # set the value of the tkVar
        kwargs[varKey].set(self.settings[key])

        # return kwargs
        return kwargs

    class Setting:
        """Holds a setting with label and widget"""

        def __init__(self, label=None, widget=None):
            self._label = label
            self._widget = widget

        @property
        def label(self):
            """return label widget"""
            return self._label

        @label.setter
        def label(self, val):
            """set label widget"""
            self._label = val

        @property
        def widget(self):
            """return input widget"""
            return self._widget

        @widget.setter
        def widget(self, val):
            self._widget = val

if __name__ == "__main__":
    settingsWindow = tk.Tk()

    settingsFrame = SettingsFrame(
        main=settingsWindow,
        controller=None
    )

    settingsFrame.create()

    settingsFrame.pack()

    settingsWindow.mainloop()


import numpy as np
import matplotlib.pyplot as plt
import matplotlib.animation as animation
import matplotlib.patches as patches
TWOPI = 2*np.pi

fig, ax = plt.subplots()
#xy1 = np.array([0.2, 0.2])
xy2 = np.array([1.2, -0.1])
#circle = patches.Circle(xy1, 0.1, color = "r")
rec=patches.Rectangle(xy2,0.2,0.2, color = "r")
ax.add_patch(rec)
ax = plt.axis([-1,1,-1,1])

Y = np.loadtxt('time_position')
x=Y.T
"""
fig, ax = plt.subplots()
ax = plt.axis([-1,1,-1,1])
#l, =plt.plot(x[0][0:1],x[1][0:1])
xy2 = np.array([0.5, 0.5])
#rec =patches.Rectangle(xy2,1,1, color = "r")
circle = patches.Circle(xy2, 0.1, color = "r")
ax.add_patch(circle)
#def animate(i):
    #rec.set_x(x[1][i]-0.5)
#return rec,
"""
def animate(i):
    rec.set_x(x[1][i]-0.1)
    return rec,
plt.title("Forced damped Oscillator")
plt.xlabel('Position (m)')
plt.ylabel('')
myAnimation = animation.FuncAnimation(fig, animate, frames=np.arange(0, len(x[0]), 1), \
                                      interval=10, blit=True, repeat=True)
plt.show()


import numpy as np

from .control import model_setup
from .cp_confocal import twod
from .cp_triplet import trip

# 2D + TT Gauß
# Model 6003
def CF_Gxy_gauss_2DTT(parms, tau):
    u""" Two-dimensional free diffusion
        with a Gaussian laser profile, including two triplet components.
        The triplet factor takes into account a blinking term.
        Set *T* or *τ_trip* to 0, if no triplet component is wanted.

        particle = 1/(1+τ/τ_diff)
        triplet1 = 1 + T₁/(1-T₁)*exp(-τ/τ_trip₁)
        triplet2 = 1 + T₂/(1-T₂)*exp(-τ/τ_trip₂)
        G = 1/n*particle*triplet1*triplet2 + offset

        *parms* - a list of parameters.
        Parameters (parms[i]):
        [0]  n        Effective number of particles in confocal volume
        [1]  τ_diff   Diffusion time of particle
        [2]  τ_trip₁  Characteristic residence time in triplet state
        [3]  T₁       Fraction of particles in triplet (non-fluorescent) state
                      0 <= T < 1
        [4]  τ_trip₂  Characteristic residence time in triplet state
        [5]  T₂       Fraction of particles in triplet (non-fluorescent) state
                      0 <= T < 1
        [6]  offset
        *tau* - lag time
    """
    n = parms[0]
    taud = parms[1]
    tautrip1 = parms[2]
    T1 = parms[3]
    tautrip2 = parms[4]
    T2 = parms[5]
    off = parms[6]

    g = twod(tau=tau, taudiff=taud)

    tr1 = trip(tau=tau, T=T1, tautrip=tautrip1)
    tr2 = trip(tau=tau, T=T2, tautrip=tautrip2)

    G = off + 1/n * g * tr1 * tr2

    return G

def supplements(parms, countrate=None):
    # We can only give you the effective particle number
    n = parms[0]
    Info = list()
    if countrate is not None:
        # CPP
        cpp = countrate/n
        Info.append(["cpp [kHz]", cpp])
    return Info

parms = [
    4,       # n
    .4,      # taud
    0.001,   # tautrip1
    0.01,    # T1
    0.002,   # tautrip2
    0.01,    # T2
    0.0      # offset
]

# Boundaries
# strictly positive
boundaries = [[0, np.inf]]*len(parms)
# T
boundaries[3] = [0, .9999999999999]
boundaries[5] = [0, .9999999999999]
# offset
boundaries[-1] = [-np.inf, np.inf]

model_setup(
    modelid=6003,
    name="2D diffusion with double triplet (confocal)",
    comp="T+T+2D",
    mtype="Confocal (Gaussian) with double triplet",
    fctn=CF_Gxy_gauss_2DTT,
    par_labels=[
        u"n",
        u"τ_diff [ms]",
        u"τ_trip₁ [ms]",
        u"T₁",
        u"τ_trip₂ [ms]",
        u"T₂",
        u"offset"
    ],
    par_values=parms,
    par_vary=[True, True, False, False, False, False, False],
    par_boundaries=boundaries,
    par_constraints=[[2, "<", 1], [4, ">", 2]],
    par_hr_labels=[
        u"n",
        u"τ_diff [ms]",
        u"τ_trip₁ [µs]",
        u"T₁",
        u"τ_trip₂ [µs]",
        u"T₂",
        u"offset"
    ],
    par_hr_factors=[
        1.,     # n
        1.,     # taudiff
        1000.,  # tautrip1 [µs]
        1.,     # T1
        1000.,  # tautrip2 [µs]
        1.,     # T2
        1.      # offset
    ],
    supplementary_method=supplements
)


#!/usr/bin/env python3

from app.comparison.scenarios.scenario_2 import ComparisonScenarioTwo
from app.simulation.graph.synoptic_panel_video import SynopticPanelVideo
from controller.random_action.controller import RandomActionController

def main():
    scenario_two = ComparisonScenarioTwo(
        simulation_options={
            'max_steps': 1e5,
            'max_steps_without_train_movement': 100,
            'max_cost': 5e9,
        },
        solutions_size=5,
        max_iterations=20,
        controllers=[
            RandomActionController,
        ]
    )
    scenario_two.run(export_results=False)

    for controller in scenario_two.controllers:
        for solution in controller.solutions:
            synoptic = SynopticPanelVideo(solution.results)
            synoptic.export_video()

if __name__ == '__main__':
    main()


#! /usr/bin/env python
# -*- coding: utf-8 -*-

from collections import defaultdict, deque
from Start_multi_datetime import app
import threading
#import inspect
#import ctypes
import select
import socket
import heapq  # 实现最小堆模块
import time
import ssl

'''
select.epoll(sizehint=-1, flags=0)    创建epoll对象
epoll.close()                         关闭epoll对象的文件描述符
epoll.closed                          检测epoll对象是否关闭
epoll.fileno()                        返回epoll对象的文件描述符数量
epoll.fromfd(fd)                      根据指定的文件描述符创建epoll对象
epoll.register(fd[, eventmask])       向epoll对象中注册文件描述符和对应的事件
epoll.modify(fd, eventmask)           修改注册
epoll.unregister(fd)                  取消注册
epoll.poll(timeout=-1, maxevents=-1)  阻塞，直到注册的文件描述符所绑定的事件发生,会返回一个dict，格式为：{(f

select 事件常量       意义   pycurl 事件数
1  EPOLLIN        读就绪    1
4  EPOLLOUT       写就绪    2
2  EPOLLPRI       有数据紧急读取
8  EPOLLERR       assoc. fd有错误情况发生
16 EPOLLHUP       assoc. fd发生挂起
2147483648 EPOLLET        设置边缘触发(ET)（默认的是水平触发）
1073741824 EPOLLONESHOT   设置为 one-short 行为，一个事件(event)被拉出后，对应的fd在内部被禁用
64 EPOLLRDNORM    和 EPOLLIN 相等
128 EPOLLRDBAND    优先读取的数据带(data band)
256 EPOLLWRNORM    和 EPOLLOUT 相等
512 EPOLLWRBAND    优先写的数据带(data band)
1024 EPOLLMSG       忽视
'''

class Loop(object):
    __slots__ = ['epoll', 'free_sockets', 'fileno_socket', 'active_sockets', 'epoll_delay', 'gen', 'gen_list', 'gen_loop', 'next_gen_loop', 'next_gen_periodic', 'next_gen_socket_closed', 'timer', 'timer_thread', 'max_active_sockets', 'timeout', 'loop_host_dict', 'timer_loop_others', 'loop_delay']

    def __init__(self, connect_timeout=20, epoll_delay=0.0001, loop_host_dict={}, loop_delay=0.1015):
        self.epoll = select.epoll()
        self.free_sockets = defaultdict(list)
        self.active_sockets = defaultdict(set)
        self.fileno_socket = {}
        self.epoll_delay = epoll_delay
        self.gen = []
        self.gen_list = set()
        self.gen_loop = None
        self.next_gen_loop = deque()
        self.next_gen_periodic = set()
        self.next_gen_socket_closed = set()
        self.max_active_sockets = 21
        self.timeout = 10
        self.loop_host_dict = loop_host_dict
        self.timer_loop_others = set()
        self.loop_delay = loop_delay

        for host, value in self.loop_host_dict.items():
            for i in range(value['create_free_sockets']):
                self.create_socket(host)

    def add_future(self, gen):
        try:
            self.gen_list.add(gen)
            self.gen.append(gen)
            gen.send(None)
        except StopIteration:
            self.gen_list.remove(gen)
            self.gen.pop()

    def send_gen(self, gen, value=None):
        try:
            self.gen.append(gen)
            gen.send(value)
        except StopIteration:
            self.gen_list.remove(gen)
            self.gen.pop()
        except UnicodeDecodeError:
            self.send_gen(gen, '')

    def add_future_loop(self, gen, delay=None):
        if self.gen_loop:
            self.timer_loop_others.add((_Timer_loop_other, gen, delay))
        else:
            self.gen_loop = gen

    def request(self, host=None, url=None, headers=[], data=None, access_token='', timeout=20, after_idle_sec=1, interval_sec=3, max_fails=5, is_http=False):
        if self.free_sockets[host]:
            ssock = self.free_sockets[host].pop(0)
            ssock.gen = self.gen.pop()
            if self._send_data(ssock, url, host, headers, data, access_token):
                self.epoll.register(ssock.fileno(), select.EPOLLIN |select.EPOLLERR | select.EPOLLET)
        else:
            ssock = self.create_socket(host, is_http=is_http)
            if ssock:
                ssock.gen = self.gen.pop()
                ssock.dict_post = {'url':url, 'headers':headers, 'data':data, 'access_token':access_token}
            else:
                self.next_gen_socket_closed.add(self.gen.pop())

    def create_socket(self, host, after_idle_sec=1, interval_sec=3, max_fails=5, is_http=False):
        if is_http:
            port = 80
            ssock = MySocket(socket.AF_INET,socket.SOCK_STREAM)
            ssock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)  # 禁用 Nagle 算法，减少延迟
            ssock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) # 在客户端开启心跳维护
            ssock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, after_idle_sec)  # 设置连接上如果没有数据发送的话，多久后发送keepalive探测分组，单位是秒
            ssock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, interval_sec)  # 前后两次探测之间的时间间隔，单位是秒
            ssock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, max_fails)  # 关闭一个非活跃连接之前的最大重试次数
        else:
            port = 443
            sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
            sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)  # 禁用 Nagle 算法，减少延迟
            sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) # 在客户端开启心跳维护
            sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, after_idle_sec)  # 设置连接上如果没有数据发送的话，多久后发送keepalive探测分组，单位是秒
            sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, interval_sec)  # 前后两次探测之间的时间间隔，单位是秒
            sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, max_fails)  # 关闭一个非活跃连接之前的最大重试次数
            context = ssl.create_default_context()
            ssock = context.wrap_socket(sock,server_hostname=host)
        try:
            ssock.connect((host, port))
        except (socket.timeout, TimeoutError):
            print('连接超时错误')
            ssock.close()
            return None
        else:
            ssock.setblocking(False)
            ssock.host = host
            ssock.cookie = None
            ssock.startime = time.time()
            self.active_sockets[host].add(ssock)
            fileno = ssock.fileno()
            self.fileno_socket[fileno] = ssock
            self.epoll.register(fileno, select.EPOLLOUT | select.EPOLLET)
            return ssock

    def _send_data(self, ssock, url, host, headers, data, access_token):
        fileno = ssock.fileno()
        headers = ''.join(headers)
        data = f"POST {url} HTTP/1.1\r\nHost:{host}\r\nContent-Length:{len(data)}\r\n{access_token}{headers+ssock.cookie if 'Cookie' not in headers and ssock.cookie else headers}\r\n{data}".encode()
        try:
            ssock.sendall(data)  # BrokenPipeError
            return True
        except (OSError, BrokenPipeError):
            print('发送管道错误', ssock.gen)
            self.next_gen_socket_closed.add(ssock.gen)
            ssock.close()
            return False
        except (socket.timeout, TimeoutError):
            print('发送超时错误', ssock)
            self.next_gen_socket_closed.add(ssock.gen)
            ssock.close()
            return False

    def await_sleep(self, delay):
        self.timer.add(self.gen.pop(), time.time()+delay)

    def _result_worker(self, fileno, sock, gen, first_data, value=''):
        if b'keep-alive' in first_data:
            self.free_sockets[sock.host].append(sock)
            self.epoll.unregister(fileno)
            self.send_gen(gen, value=value)
        else:
            #print('服务器端关闭')
            self.send_gen(gen, value=value)
            self.epoll.unregister(fileno)
            del self.fileno_socket[fileno]
            sock.close()

    def _loop(self):
        events = self.epoll.poll(self.epoll_delay)
        for fileno, event in events:
            sock = self.fileno_socket[fileno]
            #print('事件', fileno, event)
            if event & select.EPOLLIN:
                gen = sock.gen
                try:
                    first_data = sock.recv(530)
                    #print('读取', first_data, sock)
                except (OSError, socket.timeout, TimeoutError):
                    print('预读取超时错误', fileno)
                    self._result_worker(fileno, sock, gen, b'')
                else:
                    if first_data:
                        if first_data[9:12] == b'200':
                            if b'Set-Cookie' in first_data:
                                sock.cookie = f"Cookie:{first_data.split(b'Set-Cookie: ', 1)[1].split(b'; Path', 1)[0].decode()}\r\n"
                            buffer = first_data
                            #print('读取', sock, first_data.decode())
                            if b'chunked' in first_data:
                                SSLWantReadError_times = 0
                                while 1:
                                    try:
                                        data = sock.recv(2048)  # 阻塞模式下，从可读文件描述符读取到空数据，会引发 BlockIOError 错误，而非阻塞模式下，会引发 ssl.SSLWantReadError 读取错误
                                        #print('读取',data)
                                    except ssl.SSLWantReadError:
                                        #print('块读取SSLWantReadError错误')
                                        SSLWantReadError_times += 1
                                        if SSLWantReadError_times > 100000:
                                            #print('块读取SSLWantReadError错误达到上限', buffer)
                                            self._result_worker(fileno, sock, gen, b'')
                                            break
                                        else:
                                            continue
                                    except BlockingIOError:
                                        if first_data[-5:] == b'0\r\n\r\n':
                                            buffer = first_data[:-5]
                                            body = buffer.split(b'\r\n\r\n' , 1)
                                            self._result_worker(fileno, sock, gen, first_data, value=b''.join(body[1].split(b'\r\n')[1::2]).decode(errors='replace'))
                                        else:
                                            self._result_worker(fileno, sock, gen, b'')
                                        break
                                    except (socket.timeout, TimeoutError):
                                        print('chunked 超时错误', sock)
                                        self._result_worker(fileno, sock, gen, b'')
                                        break
                                    else:
                                        if data[-5:] == b'0\r\n\r\n':
                                            buffer += data[:-5]
                                            body = buffer.split(b'\r\n\r\n' , 1)
                                            self._result_worker(fileno, sock, gen, first_data, value=b''.join(body[1].split(b'\r\n')[1::2]).decode(errors='replace'))
                                            break
                                        else:
                                            buffer += data
                            else:
                                size = int(first_data.split(b'Content-Length: ', 1)[1].split(b'\r\n', 1)[0])
                                body = buffer.split(b'\r\n\r\n', 1)[1]
                                if size > len(body):
                                    size -= len(body)
                                    while size:
                                        try:
                                            data = sock.recv(size)
                                            #print('读取', sock, data.decode())
                                        except ssl.SSLWantReadError:
                                            pass
                                        except (OSError, socket.timeout, TimeoutError):
                                            print('Content-Length 超时错误', fileno)
                                            self._result_worker(fileno, sock, gen, b'')
                                            break
                                        else:
                                            body += data
                                            size -= len(data)
                                    else:
                                        self._result_worker(fileno, sock, gen, first_data, value=body.decode(errors='replace'))
                                else:
                                    self._result_worker(fileno, sock, gen, first_data, value=body.decode(errors='replace'))
                        elif first_data[9:12] == b'502':
                            #print('502错误', fileno, first_data)
                            if b'keep-alive' in first_data:
                                try:
                                    sock.recv(1024)
                                except (ssl.SSLWantReadError, OSError):
                                    pass
                                self.free_sockets[sock.host].append(sock)
                            else:
                                del self.fileno_socket[fileno]
                                sock.close()
                            self.epoll.unregister(fileno)
                            self.send_gen(gen, value='')
                        else:
                            #print('其他错误', fileno, first_data)
                            self.send_gen(gen, value='')
                            del self.fileno_socket[fileno]
                            self.epoll.unregister(fileno)
                            sock.close()
                    else:
                        print('接收空', fileno)
                        self.epoll.unregister(fileno)
                        self.send_gen(gen, '')
                        del self.fileno_socket[fileno]
                        sock.close()
            elif event & select.EPOLLOUT:
                self.active_sockets[sock.host].remove(sock)
                if getattr(sock, 'dict_post', None):
                    if self._send_data(sock, sock.dict_post['url'], sock.host, sock.dict_post['headers'], sock.dict_post['data'], sock.dict_post['access_token']):
                        del sock.dict_post
                        self.epoll.modify(fileno, select.EPOLLIN | select.EPOLLET)
                    #self.epoll.modify(fileno, select.EPOLLIN |select.EPOLLERR | select.EPOLLET)  # select.EPOLLIN：水平模式，select.EPOLLIN | select.EPOLLET：边缘模式
                else:
                    self.epoll.unregister(fileno)
                    self.free_sockets[sock.host].append(sock)
            '''elif event & select.EPOLLERR:
                print('文件描述符错误', fileno)
                self.epoll.unregister(fileno)
                self.send_gen(gen, '')
                del self.fileno_socket[fileno]
                sock.close()'''

    def run_until_complete(self):
        self._run_timer()
        while 1:
            self._loop()
            if self.next_gen_periodic:
                self.send_gen(self.next_gen_periodic.pop())
            if not self.timer.periodic_handlers and not self.gen_list:
                self.epoll.close()
                for socks in self.active_sockets.values():
                    for sock in socks:
                        sock.close()
                for socks in self.free_sockets.values():
                    for sock in socks:
                        sock.close()
                break

    def run_forever(self):
        self._run_timer()
        while 1:
            for i in range(100):
                self._loop()
                while self.next_gen_loop:
                    next(self.next_gen_loop.popleft())
                for host, value in self.loop_host_dict.items():
                    if value['min_free_sockets'] > len(self.free_sockets[host]):
                        #print('空闲套接字太少', len(self.free_sockets[host]))
                        for i in range(value['create_free_sockets'] - len(self.free_sockets[host])):
                            self.create_socket(host)
                    elif len(self.free_sockets[host]) > value['max_free_sockets']:
                        #print('空闲套接字太多', len(self.free_sockets[host]))
                        for i in range(len(self.free_sockets[host]) - value['max_free_sockets']):
                            sock = self.free_sockets[host].pop(0)
                            sock.close()
            for host, sockets in self.active_sockets.items():
                if len(sockets) > self.max_active_sockets:
                    socks = sockets.copy()
                    now = time.time()
                    #print(host, '连接数过大')
                    for sock in socks:
                        if now - sock.startime > self.timeout:
                            self.epoll.unregister(sock.fileno())
                            sock.close()
                            if getattr(sock, 'gen', None):
                                self.next_gen_socket_closed.add(sock.gen)
                            self.active_sockets[host].remove(sock)
            if self.next_gen_periodic:
                self.send_gen(self.next_gen_periodic.pop())
            if self.next_gen_socket_closed:
                self.send_gen(self.next_gen_socket_closed.pop(), value='')

    def _run_timer(self):
        self.timer = _Timer(self)
        self.timer_thread = threading.Thread(target=self.timer.run,args=())
        self.timer_thread.setDaemon(True)
        self.timer_thread.start()
        for timer_loop_other, gen, delay in self.timer_loop_others:
            t = threading.Thread(target=timer_loop_other(self).run,args=(gen, delay))
            t.setDaemon(True)
            t.start()

'''def _async_raise(tid, exctype):
    """raises the exception, performs cleanup if needed"""
    tid = ctypes.c_long(tid)
    if not inspect.isclass(exctype):
        exctype = type(exctype)
    res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype))
    if res == 0:
        raise ValueError("无效的线程ID")
    elif res != 1:
        # """if it returns a number greater than one, you're in trouble,
        # and you should call it again with exc=NULL to revert the effect"""
        ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None)
        raise SystemError("PyThreadState_SetAsyncExc 失败")

def _stop_thread(thread):
    _async_raise(thread.ident, SystemExit)
    #print('成功销毁线程')'''

class _Timer(object):
    __slots__ = ['periodic_handlers', 'loop']

    def __init__(self, loop):
        self.periodic_handlers = []
        self.loop = loop

    def add(self, gen, deadline):
        heapq.heappush(self.periodic_handlers, _TimeHandler(gen, deadline))

    def run(self):
        if self.loop.gen_loop:
            while 1:
                for i in range(10):
                    self.loop.next_gen_loop.append(self.loop.gen_loop)
                    time.sleep(self.loop.loop_delay)
                if self.periodic_handlers and self.periodic_handlers[0].deadline <= time.time():
                    self.loop.next_gen_periodic.add(heapq.heappop(self.periodic_handlers).gen)
        else:
            while 1:
                time.sleep(1)
                if self.periodic_handlers:
                    if self.periodic_handlers[0].deadline <= time.time():
                        self.loop.send_gen(heapq.heappop(self.periodic_handlers).gen)
                elif not self.loop.fileno_socket:break

class _Timer_loop_other(object):
    __slots__ = ['loop']

    def __init__(self, loop):
        self.loop = loop

    def run(self, gen, delay):
        while 1:
            self.loop.next_gen_loop.append(gen)
            time.sleep(delay)

class _TimeHandler(object):
    __slots__ = ['gen', 'deadline']

    def __init__(self, gen, deadline):
        self.gen = gen
        self.deadline = deadline

    def __lt__(self, other):
        return self.deadline < other.deadline

class MySocket(socket.socket):

    def __init__(self, family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0, fileno=None):
        super().__init__(family, type, proto, fileno)