#Complete functions
def revComp(inputSeq):
  """
  This function takes an input sequence and returns the reverse complement.

  Input: inputSeq in str format
  Output: revComp in str format

  """
  complement = {'A': 'T', 'C': 'G', 'G': 'C', 'T': 'A'}

  revComp = ""
  for base in inputSeq[::-1]:
    revComp += complement[(base.upper())]

  return revComp

def make_homology_arm_fragments(tfsDF):
    """
    Takes in the dataframe of information about start/stop codon regions,
    and appends with columns for the 225 bp upstream and downstream of
    this site (the homlogy arm fragments).

    input: tfsDF - as defined in previous function and on GitHub README, with Reference_Seq 1600bp either side of the start/stop.
    output: tfsDF appended with upstreamHA and downstreamHA

    """

    tfsDF["upstreamHA"] = tfsDF.Reference_Seq.str[1375:1600]
    tfsDF["downstreamHA"] = tfsDF.Reference_Seq.str[1604:1829]

    return tfsDF

def refSeq():
    """
    Creates a Bio SeqIO element from the Drosophila melanogaster reference genome.

    returns: refSeqPerChromosome - a dictionary of seq ID: sequence (in SeqIO fasta format) for the full reference genome.
    """
    from Bio import SeqIO

    refSeqPerChromosome = {}
    for seq in SeqIO.parse(open("inputfiles/dmel-all-chromosome-r6.48.fasta"), 'fasta'): #This is the FASTA file of the reference genome sequence
        refSeqPerChromosome[seq.id] = seq.seq

    return refSeqPerChromosome

def make_dataframe_from_TFs_list(TF_list, refSeqPerChromosome, annotation = "inputfiles/dmel-all-r6.48.gtf"):
    '''
    Creating a dataframe from sequence information and genes of interest. Depends on functions revComp() and make_homology_arm_fragments().

    params:
        TF_list: xlsx file listing the genes of interest
        refSeqPerChromosome: reference chromosome stored in SeqIO sequence format, indexed by refSeqPerChromosome[seq.id] = seq.seq
        annotation: gtf file containing the gene annotation information for the reference genome

    returns: dataframe of information about each start/stop site, in format:
        'Gene_ID': 'FBgn0004652',
        'Transcript_ID': 'FBtr0083651',
        'Chromosome': '3R',
        'Gene_Region': 'stop_codon',
        'Start': 18426145,
        'Stop': 18426147,
        'Strand': '-',
        'Reference_Seq': 'TTGATCGTAGGACAC',
        'upstreamHA': 'ATGCCTG',
        'downstreamHA': 'CTGGATC'

    '''
    import pandas as pd

    #This is the input file containing the TFs we want to query
    queryTFsdf = pd.read_excel(TF_list)

    #This is the .gtf file with annotations for each gene on the reference genome
    refAnnotationsHeaders = ["Chromosome", "Source", "Gene_Region", "Start", "Stop", "Score", "Strand", "Frame", "Attribute"]
    refGenomeAnnotation = pd.read_csv(annotation, sep = "\t", header = None, index_col = False, names = refAnnotationsHeaders)

    #This is to reformat the "Attribute" category in refGenomeAnnotation, to extract Gene_ID, Gene_Symbol, and Transcript ID
    index = 0

    #Add new categories to the dataframe
    refGenomeAnnotation = refGenomeAnnotation.assign(Gene_ID = "", Gene_Symbol = "", Transcript_ID = "")

    #For each attribute value, extract the gene ID and symbol and add this to the new categories
    for attribute in refGenomeAnnotation['Attribute']:
        fullatt = (refGenomeAnnotation.loc[index]["Attribute"]).replace(";", "")
        fullatt = fullatt.replace('"', "")
        fullattsplit = fullatt.split(" ")
        refGenomeAnnotation.at[index,"Gene_ID"] = fullattsplit[1]
        refGenomeAnnotation.at[index,"Gene_Symbol"] = fullattsplit[3]
        if len(fullattsplit) > 4:
            refGenomeAnnotation.at[index,"Transcript_ID"] = fullattsplit[5]
            refGenomeAnnotation.at[index,"Transcript_Symbol"] = fullattsplit[7]
        index+=1

    #Delete Attributes category
    del refGenomeAnnotation["Attribute"]

    #Select only rows that TFs are in, and keep only the start and stop codon gene regions

    refGenomeAnnotation = refGenomeAnnotation.loc[refGenomeAnnotation["Gene_Region"].isin(["start_codon", "stop_codon"])]

    TFsdf = refGenomeAnnotation[["Gene_ID", "Transcript_ID", "Chromosome", "Gene_Region", "Start", "Stop", "Strand"]].loc[refGenomeAnnotation["Gene_ID"].isin(queryTFsdf["Flybase_ID"])]

    #Add reference genome sequence per gene region
    #This will correspond to 1.6kb upstream and downstream of ATG/stop codon
    TFsdf = TFsdf.assign(Reference_Seq = "")

    for index, rowcontents in TFsdf.iterrows():
            #Define 3.2kb gene region
            regionStart = rowcontents["Start"] - 1601
            regionStop = rowcontents["Stop"] + 1600

            #Add reference sequence
            refPosStrandSeq = str(refSeqPerChromosome[rowcontents["Chromosome"]][regionStart:regionStop]) #This is the + strand seq, so goes from end to beginning
            TFsdf.at[index,"Reference_Seq"] = revComp(refPosStrandSeq)

    #Create fragments for the HDR-arm
    TFsdf = make_homology_arm_fragments(TFsdf)

    #re-index
    TFsdf = TFsdf.reset_index()
    del TFsdf["index"]

    return TFsdf

def find_synonymous_codons(query_codon, base_to_change, codon_table_excel = "inputfiles/codon_table.xlsx"):

    '''
    Uses the amino acids table to select codons that encode for the same amino acid as the query codon. Will ensure the specified base has been mutated.

    Params:
        query_codon: string, codon to select synonymous codons for
        base_to_change: the base within the codon that needs to change. Numeric value from 1-3.
        codon_table_excel: string, path to an excel file that lists per codon which amino acid that codon encodes.

    Returns:
        synonymous_codons: list of strings, each string is a codon that encodes for the same amino acid as the query codon.

    '''

    import pandas as pd

    codon_table = pd.read_excel(codon_table_excel)

    #Extract amino acid given codon
    amino_acid_query = codon_table[codon_table['codon'] == query_codon].iloc[0]["amino_acid"]

    #Subset df for other rows corresponding to this amino acid
    same_aa_df=codon_table[codon_table["amino_acid"] == amino_acid_query]

    #Convert the codons to a list
    codon_list = same_aa_df["codon"].values.tolist()

    #Keep only codons where the indicated base has changed (this will also remove original codon)
    changedCodons = []
    for codon in codon_list:
        if codon[base_to_change-1] != query_codon[base_to_change-1]:
            changedCodons.append(codon)

    return changedCodons

def filter_gRNA(gRNA_file, tfSingleRow, refSeqPerChromosome):
    """
    Selects gRNAs within 20pb of the start/stop site and returns these in a dataframe with information about their coordinates and the start/stop site.
    params:
        gRNA_file: gff file
        tfSingleRow: a pandas series for one row of the original tfsDF with the following format
            Gene_ID                                                    FBgn0000022
            Transcript_ID                                              FBtr0070072
            Chromosome                                                           X
            Gene_Region                                                start_codon
            Start                                                           370094
            Stop                                                            370096
            Strand                                                               +
            Reference_Seq        CTAATGAATAGATTGGTGTGTGATGTAGTGATCTAATATGGTGAAG...
            upstreamHA           TTTGCTCAGTTTTTTATTGGCGCCGGGACCAATTCCCCGGCGACCA...
            downstreamHA         TTAAGAGATAGTATAACGTTATTGTGTGACGATGCTCCTTGCTTCA...

    returns: filtered_gRNAs - a dataframe of sgRNAs for one start/stop site of the format:
            fmin	fmax	#chr	strand	sgRNA_sequence	        Gene_ID	    Transcript_ID	Chromosome	Gene_Region	Start	...	downstreamHA
        0	370082	370104	X	    +	    CTATCTCTTAAAATGGCTTTGGG	FBgn0000022	FBtr0070072	    X	        start_codon	370094	...	TTAAGAGATAGTATAACGTTATTGTGTGACGATGCTCCTTGCTTCA...
        1	370693	370715	X	    -	    CCTGTAAAAAAACAGATCAAATC	FBgn0000022	FBtr0070072	    X	        start_codon	370694	...	TTAAGAGATAGTATAACGTTATTGTGTGACGATGCTCCTTGCTTCA...

    """
    import pandas as pd

    # create a dataframe from the gRNA file
    gRNAFileAnnotation = pd.read_csv(gRNA_file, sep = "\t", index_col = False)

    # add a new category to the dataframe that provides information about whether the sequence deviates from the transgenic strain
    gRNAFileAnnotation = gRNAFileAnnotation.assign(target_site_variation= "")

    # reformat the "Attribute" category in refGenomeAnnotation, to extract Gene_ID, Gene_Symbol, and Transcript ID
    index = 0 #TODO@Marina improve this code

    # for each attribute value, extract the gene ID and symbol and add this to the new categories
    for attribute in gRNAFileAnnotation['attributes']:

        fullatt = (attribute).split(";")
        gRNAFileAnnotation.at[index,"target_site_variation"] = fullatt[8]
        index+=1

    # shorten file to essential information
    GenomeCoordinates = gRNAFileAnnotation[["target_site_variation", "fmin", "fmax", "#chr", "strand"]]

    #Filter for the appropriate conditions:
        # check whether the sgRNAs match the transgeneic genome, whether the sgRNAs match to the same chromosome as the transcription factor
        # select sgRNA that are located maximally 20 bp upstream of the start/stop codon of the transcription factors
        # select sgRNA that are located maximally 20 bp downstream of the start/stop codon of the transcription factors
    filtered_gRNAs = GenomeCoordinates[(GenomeCoordinates['target_site_variation'] == "target_site_variation=none observed") &
                                                  (GenomeCoordinates['#chr'] == tfSingleRow["Chromosome"]) &
                                                  (GenomeCoordinates['fmin']-1 >= tfSingleRow["Start"] - 20) &
                                                  (GenomeCoordinates['fmax'] <= tfSingleRow["Stop"] + 20)]

    #add sgRNA sequence using coordinates
    filtered_gRNAs = filtered_gRNAs.assign(sgRNA_sequence= "")
    for index, row in filtered_gRNAs.iterrows():
        filtered_gRNAs.loc[index,'sgRNA_sequence'] = str(refSeqPerChromosome[row['#chr']][int(row["fmin"])-1:int(row["fmax"])])

    #Add columns from tfSingleRow containing start/stop site info
    for (columnName, columnData) in tfSingleRow.items():
        filtered_gRNAs = filtered_gRNAs.assign(**{columnName: columnData})

    #drop target site variation
    filtered_gRNAs = filtered_gRNAs.drop(columns=["target_site_variation"])

    #reset index
    filtered_gRNAs = filtered_gRNAs.reset_index(drop = True)

    return filtered_gRNAs

def gRNA_stringencyIterator(tfSingleRow, refSeqPerChromosome):
    """
    Iterates trough gRNA stringency files until at least one gRNA is returned from filter_gRNAs function.

    params:
        tfSingleRow: a pandas Series for one row of the tfsDF dataframe produced by make_dataframe_from_TFs_list
        refSeqPerChromosome: a Bio SeqIO dictionary for the D. melanogaster genome

    returns:
        filtered_gRNAs: a df of gRNAs that meet the conditions at this site
    """

    gRNAfiles = ["inputfiles/Hu.2019.8.28.sgRNA_designs/NoOffTarget_high_stringency.gff", "inputfiles/Hu.2019.8.28.sgRNA_designs/NoOffTarget_med_stringency.gff", "inputfiles/Hu.2019.8.28.sgRNA_designs/NoOffTarget_low_stringency.gff", "inputfiles/Hu.2019.8.28.sgRNA_designs/1to3NonCdsOffTarget_low_stringency.gff", "inputfiles/Hu.2019.8.28.sgRNA_designs/ManyOffTarget_low_stringency.gff"]

    #Set up sgRNA variable
    sgRNA = ""
    file_ind = 0

    #Loop through files and retain the filtered sgRNA df if at least one row is present
    while len(sgRNA) == 0:
        sgRNA = filter_gRNA(gRNAfiles[file_ind], tfSingleRow, refSeqPerChromosome)
        print(f"stringency {file_ind}")
        file_ind +=1
        if file_ind == 4: #If we've tried all files and sgRNA is still 0, break out of loop
            break

    return sgRNA

def sgRNApositionCheck(sgRNAdf):
    """
    Given filtered sgRNAs for a start/stop site in the 'filtered_gRNAs' format, will create dataframe containing positional information and condition checks
    for each sgRNA.

    params: df: a pandas df of sgRNAs for one start/stop site:
            fmin	fmax	#chr	strand	sgRNA_sequence	        Gene_ID	    Transcript_ID	Chromosome	Gene_Region	Start	... downstreamHA
        0	370082	370104	X	    +	    CTATCTCTTAAAATGGCTTTGGG	FBgn0000022	FBtr0070072	    X	        start_codon	370094	...	TTAAGAGATAGTATAACGTTATTGTGTGACGATGCTCCTTGCTTCA...
        1	370693	370715	X	    -	    CCTGTAAAAAAACAGATCAAATC	FBgn0000022	FBtr0070072	    X	        start_codon	370694	...	TTAAGAGATAGTATAACGTTATTGTGTGACGATGCTCCTTGCTTCA...

    output: the same dataframe with added columns:
            fmin	fmax	#chr	strand	sgRNA_sequence	        Gene_ID	    Transcript_ID	Chromosome	Gene_Region	Start	...	downstreamHA	                                    positionScore	PAM_in_start/stop	<15_bp3’_overhang	PAM_in_CDS	PAM_outside_CDS	CDS_boundary	lastG	cutSite
        0	370082	370104	X	    +	    CTATCTCTTAAAATGGCTTTGGG	FBgn0000022	FBtr0070072	    X	        start_codon	370094	...	TTAAGAGATAGTATAACGTTATTGTGTGACGATGCTCCTTGCTTCA...	8	            False	            True	            True	    False	        >22	            31.0	26.0
        1	370693	370715	X	    -	    CCTGTAAAAAAACAGATCAAATC	FBgn0000022	FBtr0070072	    X	        start_codon	370694	...	TTAAGAGATAGTATAACGTTATTGTGTGACGATGCTCCTTGCTTCA...	16	            False	            False	            True	    False	        <20	            16.0	21.0

    """
    import pandas as pd

    #Adding position scores (fmax - stop)
    sgRNAdf["positionScore"] = sgRNAdf["fmax"] - sgRNAdf["Stop"]

    #Dataframe containing parameter ranges to interpret the positon score, based on gene strand, sgRNA strand, and start/stop
    positionScoreParameters = pd.read_excel("inputfiles/fmaxStopScore.xlsx")

    #Per parameter, append the sgRNAdf with a TRUE/FALSE value per condition.
    #Also add positional information about the CDS boundary, position of the last G/C, and the cut site from the conditions file.

    #Start by adding columns for these to sgRNAdf
    booleanColumns = ["PAM_in_start/stop", "<15_bp3’_overhang", "PAM_in_CDS", "PAM_outside_CDS"]
    positionColumns = ["CDS_boundary", "lastG", "cutSite"]
    sgRNAdf = sgRNAdf.reindex(columns = sgRNAdf.columns.tolist() + booleanColumns + positionColumns)

    #Iterating through sgRNAdf, extract the appropriate row of the conditions file and add new information to sgRNAdf.
    for ind, sgRNA in sgRNAdf.iterrows():

        #Extract the appropriate parameter row per sgRNA
        conditions = positionScoreParameters.loc[(positionScoreParameters["start/stop"] == sgRNA['Gene_Region']) & (positionScoreParameters["strand_type"] == sgRNA['Strand']) & (positionScoreParameters["sgRNA_strand"] == sgRNA['strand'])]
        conditions = conditions.reset_index(drop = True)

        #Per column, input true/false as to whether the position score meets that condition
        for col in booleanColumns:
            colValue = conditions.at[0,col] #extract parameter range values from dataframe
            #Process the value into a range (in format [min, max])
            #If the values should be 'more than' or 'less than', 25 is used as a max or -25 as min because distances cannot be more than 20
            if ">" in colValue: #could simplify this further by just defining all as ranges in excel
                minMax = [int(colValue[1:])+1, 25]
            elif "<" in colValue:
                minMax = [-25,int(colValue[1:])]
            elif ":" in colValue:
                min, max = colValue.split(":")
                minMax = [int(min), int(max)]
            else:
                print("Incorrect format of range value. Verify inputs.")

            #Into the output dataframe, print true/false as to whether the positionScore has met the condition for that column
            sgRNAdf.at[ind, col] = bool(sgRNA["positionScore"] in range(minMax[0], minMax[1]))

        #Calculate PAM last G or C position using positionScore. Add this to the sgRNA catalogue.
        sgRNAdf.at[ind, "lastG"] = sgRNAdf.at[ind, "positionScore"] + int(conditions.at[0, "lastG"])

        #Calculate cutsite coordinate using positionScore. Add this to the sgRNA catalogue.
        sgRNAdf.at[ind, "cutSite"] = sgRNAdf.at[ind, "positionScore"] + int(conditions.at[0, "cutSite"])

        #Keep CDS boundary position in output df
        sgRNAdf.at[ind, "CDS_boundary"] = conditions.at[0, "CDS_boundary"]

    return sgRNAdf

def checkCDSCutandOrder(sgRNAdf):
    """
    Given an sgRNAdf, will calculate firstly rows where the sgRNA cuts inside CDS. If multiple, selects that which cuts closest to start/stop.
    If none, selects closest cut sgRNA that is outside CDS.

    params: sgRNAdf in format:
            fmin	fmax	#chr	strand	sgRNA_sequence	        Gene_ID	    Transcript_ID	Chromosome	Gene_Region	Start	...	downstreamHA	                                    positionScore	PAM_in_start/stop	<15_bp3’_overhang	PAM_in_CDS	PAM_outside_CDS	CDS_boundary	lastG	cutSite mutated
        0	370082	370104	X	    +	    CTATCTCTTAAAATGGCTTTGGG	FBgn0000022	FBtr0070072	    X	        start_codon	370094	...	TTAAGAGATAGTATAACGTTATTGTGTGACGATGCTCCTTGCTTCA...	8	            False	            True	            True	    False	        >22	            31.0	26.0	False
        1	370693	370715	X	    -	    CCTGTAAAAAAACAGATCAAATC	FBgn0000022	FBtr0070072	    X	        start_codon	370694	...	TTAAGAGATAGTATAACGTTATTGTGTGACGATGCTCCTTGCTTCA...	16	            False	            False	            True	    False	        <20	            16.0	21.0	False

    output: pandas series of the winning sgRNA in format:
    fmin                                                            409794
    fmax                                                            409816
    #chr                                                                 X
    strand                                                               -
    sgRNA_sequence                                 CCATGACGAGCATTTGCAGCAGC
    Gene_ID                                                    FBgn0002561
    Transcript_ID                                              FBtr0070074
    Chromosome                                                           X
    Gene_Region                                                start_codon
    Start                                                           409796
    Stop                                                            409798
    Strand                                                               +
    Reference_Seq        CGACGTCGTCCAGCTTGATGAAAAACTTTGTGGCGGTGTGCGCCAG...
    upstreamHA           CATGTTATTGTAGTTGAACTTCTTCTTCTGGGAGGACAACATGGGT...
    downstreamHA         GTAATCCTTGCGAGAGTTTTCTAAGATTTAGTTTACAGATGTTGAC...
    positionScore                                                       18
    PAM_in_start/stop                                                 True
    <15_bp3’_overhang                                                 True
    PAM_in_CDS                                                       False
    PAM_outside_CDS                                                  False
    CDS_boundary                                                       >22
    lastG                                                             18.0
    cutSite                                                           23.0

    """

    #extract CDS boundary value
    sgRNAdf.reset_index(inplace= True, drop= True)
    CDSBoundary = sgRNAdf.at[0, 'CDS_boundary']

    #Calculate sgRNA/cut site difference
    sgRNAdf["cutsite-CDSbound"] = abs(sgRNAdf["cutSite"] - 20)

    #C. Check cut site in CDS

    #Define CDS boundary range
    if ">" in CDSBoundary: #could simplify this further by just defining all as ranges in excel
        CDSBoundaryRange = [int(CDSBoundary[1:])+1, 43]
    elif "<" in CDSBoundary:
        CDSBoundaryRange = [0,int(CDSBoundary[1:])]
    else:
        print("CDS boundary value not found/invalid")

    #filter for only sgRNAs that cut in CDS
    conditionC = sgRNAdf[sgRNAdf["cutSite"].between(CDSBoundaryRange[0], CDSBoundaryRange[1])]

    #if there are sgRNAs that cut in CDS, sort by distance of cutsite and CDS start. Select the one that cuts closest.
    if len(conditionC) > 0: #cuts in CDS, closest cut (C1, C2)
        conditionCclosestCut = conditionC.sort_values('cutsite-CDSbound')
        conditionCclosestCut = conditionCclosestCut.reset_index(drop = True) #reset index
        winnersgRNA = conditionCclosestCut.at[0, "sgRNA_sequence"]
    else: #no sgRNAs cut in CDS, select closest that still met condition B (C3)
        nonCDSclosestCut = sgRNAdf.sort_values('cutsite-CDSbound')
        winnersgRNA = nonCDSclosestCut.at[0, "sgRNA_sequence"]

    return winnersgRNA

def find_best_gRNA(sgRNAdf):
    """
    Select the best guide RNA from a dataframe of guideRNAs that are ±20bp from the start/stop site.

    params:
        df: the dataframe of potential guideRNAs in format:
            fmin	fmax	#chr	strand	sgRNA_sequence	        Gene_ID	    Transcript_ID	Chromosome	Gene_Region	Start	...	downstreamHA	                                    positionScore	PAM_in_start/stop	<15_bp3’_overhang	PAM_in_CDS	PAM_outside_CDS	CDS_boundary	lastG	cutSite	mutated
        0	370082	370104	X	    +	    CTATCTCTTAAAATGGCTTTGGG	FBgn0000022	FBtr0070072	    X	        start_codon	370094	...	TTAAGAGATAGTATAACGTTATTGTGTGACGATGCTCCTTGCTTCA...	8	            False	            True	            True	    False	        >22	            31.0	26.0	False
        1	370693	370715	X	    -	    CCTGTAAAAAAACAGATCAAATC	FBgn0000022	FBtr0070072	    X	        start_codon	370694	...	TTAAGAGATAGTATAACGTTATTGTGTGACGATGCTCCTTGCTTCA...	16	            False	            False	            True	    False	        <20	            16.0	21.0	False

    output:
        winnersgRNA: the sequence of the best guideRNA (str)
        mutationNeeded: True/False of whether this sgRNA needs to be mutated.

    """
    #Set up the winning guide RNA
    winnerFound = False
    mutationNeeded = False
    winnersgRNA = ""

    #A. Ideal condition - PAM in start/stop
    conditionA = sgRNAdf[sgRNAdf["PAM_in_start/stop"] == True] #This is the subset df for which PAM is in the start/stop
    if len(conditionA) > 0: #if there is one or more sgRNAs for this condition, select the first as the winner
        conditionA = conditionA.reset_index(drop = True) #reset index
        winnersgRNA = conditionA.at[0, "sgRNA_sequence"]
        winnerFound = True

    #B. sgRNA overhang is less than 15bp
    if winnerFound == False:
        conditionB = sgRNAdf[sgRNAdf["<15_bp3’_overhang"] == True]
        if len(conditionB) == 1:
            conditionB = conditionB.reset_index(drop = True)
            winnersgRNA = conditionB.at[0, "sgRNA_sequence"]
            winnerFound = True
        elif len(conditionB) > 1: #if more than one, check condition C (select for cut site in CDS preferencially, and the sgRNA where cut site is closest to CDS)
            winnersgRNA = checkCDSCutandOrder(conditionB)
            winnerFound = True

    #D. sgRNA overhang is more than 15bp, need to mutate
    if winnerFound == False:
        if len(sgRNAdf) == 1: #if just 1 sgRNA, select this one
            winnersgRNA = sgRNAdf.at[0, "sgRNA_sequence"]
        else: #more than 1 sgRNA
            winnersgRNA = checkCDSCutandOrder(sgRNAdf) #Again, iterate through condition C to select an sgRNA
            winnerFound = True

        mutationNeeded = True

    return winnersgRNA, mutationNeeded

def codonFragmenter(winnerdf):
    """
    For the start/stop site, will create a list of codons in the appropriate range where sgRNAs might be found (start/stop ±20 bp).
    If gene is on - strand, these will be revComp codons.

    params:
        winnerdf: pandas Series for the row of the sgRNA dataframe containing information about the winner sgRNA:
            fmin                                                            396165
            fmax                                                            396187
            #chr                                                                 X
            strand                                                               -
            sgRNA_sequence                                 CCGAGTGTGTTAATGAAAAATAA
            Gene_ID                                                    FBgn0004170
            Transcript_ID                                              FBtr0070073
            Chromosome                                                           X
            Gene_Region                                                start_codon
            Start                                                           396177
            Stop                                                            396179
            Strand                                                               +
            Reference_Seq        ACCTGCGATAATTTGACATTCTTAGAAACTACCTGAAGAAATAGGA...
            upstreamHA           TCTGGTCAGTGCCATACCCCTTGGTGTATACTTGCGAGTCTTAATT...
            downstreamHA         AACACACTCGGAGCTTTCTTTAACTTTCCGGATAACGATCAACAGA...
            positionScore                                                        8
            PAM_in_start/stop                                                False
            <15_bp3’_overhang                                                 True
            PAM_in_CDS                                                       False
            PAM_outside_CDS                                                   True
            CDS_boundary                                                       >22
            lastG                                                              8.0
            cutSite                                                           13.0
            mutated                                                          False

    returns:
        codonList: codons from the ±21bp region around the start/stop site in mutable format (revComp if gene is -)
    """

    #Extract HA sequences - just the 21bp near start/stop
    HAL = winnerdf["upstreamHA"][-21:]
    HAR = winnerdf["downstreamHA"][0:21]

    #Start codon list
    codonList = []

    #Add HAL codons
    for codonBase1 in range(0, len(HAL), 3):
        codonList.append(HAL[codonBase1:codonBase1+3])

    #add start/stop - this is added as 'ATG' (even if stop) but will never be mutated. As HAL and HAR only will be replaced into tfsDF,
    #this will not affect original sequences.
    codonList.append('ATG')

    #Add HAR codons
    for codonBase1 in range(0, len(HAR), 3):
        codonList.append(HAR[codonBase1:codonBase1+3])

    #Define gene strand:
    if winnerdf["Strand"] == '-': #If on the minus strand, take revComp per codon
        for ind, codon in enumerate(codonList):
            codonList[ind] = revComp(codon)

    return codonList

def codonReverseFragmenter(codonsList, winnerdf):
    """
    Will take a fragmented list of codons of the start/stop site region and replace the mutated HAL/HAR arms in tfsDF.
    If gene is on - strand, revComp codons will be reversed back to the + strand sequences.

    params:
        codonList: codons from the ±21bp region around the start/stop site in mutable format (revComp if gene is -)
        winnerdf: pandas series of the winning df in format:

        fmin                                                            396165
        fmax                                                            396187
        #chr                                                                 X
        strand                                                               -
        sgRNA_sequence                                 CCGAGTGTGTTAATGAAAAATAA
        Gene_ID                                                    FBgn0004170
        Transcript_ID                                              FBtr0070073
        Chromosome                                                           X
        Gene_Region                                                start_codon
        Start                                                           396177
        Stop                                                            396179
        Strand                                                               +
        Reference_Seq        ACCTGCGATAATTTGACATTCTTAGAAACTACCTGAAGAAATAGGA...
        upstreamHA           TCTGGTCAGTGCCATACCCCTTGGTGTATACTTGCGAGTCTTAATT...
        downstreamHA         AACACACTCGGAGCTTTCTTTAACTTTCCGGATAACGATCAACAGA...
        positionScore                                                        8
        PAM_in_start/stop                                                False
        <15_bp3’_overhang                                                 True
        PAM_in_CDS                                                       False
        PAM_outside_CDS                                                   True
        CDS_boundary                                                       >22
        lastG                                                              8.0
        cutSite                                                           13.0
        mutated                                                          False

    output: the winningdf as previously, now with mutated upstreamHA or downstreamHA.
        fmin                                                            396165
        fmax                                                            396187
        #chr                                                                 X
        strand                                                               -
        sgRNA_sequence                                 CCGAGTGTGTTAATGAAAAATAA
        Gene_ID                                                    FBgn0004170
        Transcript_ID                                              FBtr0070073
        Chromosome                                                           X
        Gene_Region                                                start_codon
        Start                                                           396177
        Stop                                                            396179
        Strand                                                               +
        Reference_Seq        ACCTGCGATAATTTGACATTCTTAGAAACTACCTGAAGAAATAGGA...
        upstreamHA           TCTGGTCAGTGCCATACCCCTTGGTGTATACTTGCGAGTCTTAATT...
        downstreamHA         AACACACTCGGAGCTTTCTTTAACTTTCCGGATAACGATCAACAGA...
        positionScore                                                        8
        PAM_in_start/stop                                                False
        <15_bp3’_overhang                                                 True
        PAM_in_CDS                                                       False
        PAM_outside_CDS                                                   True
        CDS_boundary                                                       >22
        lastG                                                              8.0
        cutSite                                                           13.0
        mutated                                                          False

    """
    HAL = ""
    HAR = ""

    #return to + strand if the gene is on -
    if winnerdf["Strand"] == '-':
        for ind, codon in enumerate(codonsList):
            codonsList[ind] = revComp(codon)

    #replace HAL
    for codon in range(0, 7):
        HAL += codonsList[codon]

    #replace HAR
    for codon in range(8, 15):
        HAR += codonsList[codon]

    #replace mutated HAL and HAR
    winnerdf.at["upstreamHA"] = winnerdf["upstreamHA"][:-21] + HAL
    winnerdf.at["downstreamHA"] = HAR + winnerdf["downstreamHA"][21:]

    return winnerdf

def mutator(winnerdf):
    """
    In the case where a fragment needs to be mutated, will mutate in CDS (preferably PAM, if not then in the sgRNA).
    If not possible, will mutate PAM outside of CDS to NTG/CTN.

    params:
        winnerdf: the pandas series of the winning sgRNA in format:
            fmin                                                            396165
            fmax                                                            396187
            #chr                                                                 X
            strand                                                               -
            sgRNA_sequence                                 CCGAGTGTGTTAATGAAAAATAA
            Gene_ID                                                    FBgn0004170
            Transcript_ID                                              FBtr0070073
            Chromosome                                                           X
            Gene_Region                                                start_codon
            Start                                                           396177
            Stop                                                            396179
            Strand                                                               +
            Reference_Seq        ACCTGCGATAATTTGACATTCTTAGAAACTACCTGAAGAAATAGGA...
            upstreamHA           TCTGGTCAGTGCCATACCCCTTGGTGTATACTTGCGAGTCTTAATT...
            downstreamHA         AACACACTCGGAGCTTTCTTTAACTTTCCGGATAACGATCAACAGA...
            positionScore                                                        8
            PAM_in_start/stop                                                False
            <15_bp3’_overhang                                                 True
            PAM_in_CDS                                                       False
            PAM_outside_CDS                                                   True
            CDS_boundary                                                       >22
            lastG                                                              8.0
            cutSite                                                           13.0
            mutated                                                          False

    """
    import pandas as pd

    #Mutated starts as false
    mutated = False

    #Codon fragmenter - codons are now in mutable format in a list, from last 21bp of HAL, the start/stop site, then the first 21bp of HAR. RevComp if gene is -
    mutableCodons = codonFragmenter(winnerdf)

    ## Coordinate information per codon of mutableCodons. This accounts for gene strand.
    if winnerdf["Strand"] == '+':
        codonCoordinates = pd.read_excel("inputfiles/fmaxStopScore.xlsx", sheet_name="CodonCoordinatePlus", index_col= 0)
    elif winnerdf["Strand"] == '-':
        codonCoordinates = pd.read_excel("inputfiles/fmaxStopScore.xlsx", sheet_name="CodonCoordinateMinus", index_col= 0)

    #Define CDS boundary range
    if ">" in winnerdf["CDS_boundary"]:
        CDSboundary = [int(winnerdf["CDS_boundary"][1:])+1, 43]
    elif "<" in winnerdf["CDS_boundary"]:
        CDSboundary = [0,int(winnerdf["CDS_boundary"][1:])]

    #Extract PAM last G coordinate
    lastG = winnerdf['lastG'] #this is a C if on minus strand

    #Coordinates for sgRNA bases sgRNA1, sgRNA2, sgRNA3, sgRNA4, sgRNA5, sgRNA6
    #Any of these coordinates that are outside of the CDS will be removed, leaving only sgRNA coordinates in CDS.
    #We also define the second PAM base based on sgRNA +/- here - secondG (again, might be c)
    sgRNAcoordinates = []
    if winnerdf['strand'] == '+': #sgRNA on + strand
        secondG = lastG-1
        for shift in range(3, 9):
            sgRNAcoordinates.append(lastG - shift)
    else:
        secondG = lastG+1
        for shift in range(3, 9): #sgRNA on - strand
            sgRNAcoordinates.append(lastG + shift)

    sgRNAcoordinates = [x for x in sgRNAcoordinates if x in range(CDSboundary[0], CDSboundary[1])]

    #A. Mutate PAM in CDS (1 mutation)
    if winnerdf["PAM_in_CDS"] == True: #check condition
        #mutate last G if possible using relative coordinates
        codonNumber = codonCoordinates.at[lastG, 'codon'] #this is the codon number (as an index in mutableCodons)
        codon = mutableCodons[codonNumber] #This is the codon we want to mutate
        base = codonCoordinates.at[lastG, 'base'] #This is the base within that codon (1-3)
        potentialCodons = find_synonymous_codons(query_codon =codon, base_to_change= base)
        if len(potentialCodons) >0:
        #check for NGA and remove this
            for cod in potentialCodons:
                if cod[:-1] == "A":
                    potentialCodons.remove(cod)
            #If we still have replacement codons left, swap the first one in:
            if len(potentialCodons) >0:
                mutableCodons[codonNumber] = potentialCodons[0]
                mutated = True
        else: #if no mutation is found, try the second G.
            codonNumber = codonCoordinates.at[secondG, 'codon']
            codon = mutableCodons[codonNumber]
            base = codonCoordinates.at[secondG, 'base']
            potentialCodons = find_synonymous_codons(query_codon = codon, base_to_change = base)
             #If we have replacement codons, swap the first one in:
            if len(potentialCodons) >0:
                mutableCodons[codonNumber] = potentialCodons[0]
                mutated = True

    #B. Mutate sgRNA in CDS (up to 2 mutations)
    #Check if mutation was successful in A, then check if there are mutable bases of the sgRNA in CDS
    if mutated == False and len(sgRNAcoordinates) != 0:
        print("Condition B")
        sgMutatedBases = 0 #count of bases mutated
        excludeCodons = [] #this is the codon number of codons we've already mutated. This ensures that we won't re-mutate already mutated codons.
        index = 0 #this is the index of the sgRNA coordinate we are trying to mutate
        while sgMutatedBases < 2 and index <= len(sgRNAcoordinates): #while 2 mutations have not yet been made and the we've not uet used all coordinates in the sgRNA coordinate list
            print(f"mutating base {index}, mutations so far are {sgMutatedBases}")
            coordinate = sgRNAcoordinates[index] #This is the coordinate of the sgRNA base
            codonNumber = codonCoordinates.at[coordinate, 'codon']
            codon = mutableCodons[codonNumber] #this is the codon number that this coordinate is in
            base = codonCoordinates.at[coordinate, 'base'] #this is the base within that codon that the coordinate corresponds to (1-3)

            #check if we've already mutated this codon
            if codonNumber not in excludeCodons: #if not, attempt to mutate this base
                potentialCodons = find_synonymous_codons(query_codon = codon, base_to_change = base)
                if len(potentialCodons) > 0: #if mutation is possible, accept this mutation
                    mutableCodons[codonNumber] = potentialCodons[0] #replace this codon in the fragmenter list
                    excludeCodons.append(codonNumber) #add this to already mutated codons
                    sgMutatedBases +=1 #add 1 to count of mutated bases

            index += 1 #move to next sgRNA base

        if sgMutatedBases > 0: #if we've mutated at least 1 sgRNA base, set mutated to True
            mutated = True

    #C. Mutate PAM outside CDS - to NTG or CTN
    if mutated == False:
        codonNumber = codonCoordinates.at[secondG, 'codon']
        codon = mutableCodons[codonNumber]
        base = codonCoordinates.at[secondG, 'base']
        newCodon = codon[:base-1] + 'T' + codon[base:] #add a 'T' where the old base was in this codon
        mutableCodons[codonNumber] = newCodon
        mutated = True

    #Replace the potentially mutated homology arms back in the upstreamHA and downstreamHA of our winnerdf
    winnerdfmutated = codonReverseFragmenter(mutableCodons, winnerdf)

    #Indicate if mutation has occurred within winner df
    winnerdfmutated['mutated'] = mutated

    return winnerdfmutated

def sgRNArunner():
    """
    Check functions so far work with the proposed changes.
    """
    import pandas as pd
    #set up reference sequence Bio SeqIO element
    refSeqPerChromosome = refSeq()

    #Make the dataframe for all transcription factor start/stop site infp
    TFsdf = make_dataframe_from_TFs_list("inputfiles/TFs.xlsx", refSeqPerChromosome)

    #set up the output DF - this is the winning sgRNA per site in TFsdf
    TFsdfWinnersandMutated = pd.DataFrame(columns=["fmin", "fmax", "#chr", "strand", "sgRNA_sequence", "Gene_ID",
                                                   "Transcript_ID", "Chromosome", "Gene_Region", "Start", "Stop",
                                                   "Strand", "Reference_Seq", "upstreamHA", "downstreamHA", "positionScore",
                                                    "PAM_in_start/stop", "<15_bp3’_overhang", "PAM_in_CDS", "PAM_outside_CDS",
                                                    "CDS_boundary", "lastG", "cutSite", "mutated"])

    #Per row of this dataframe, select a guideRNA
    for ind, row in TFsdf.iterrows():
        print(f"Selecting guide RNA for TFsdf row {ind}")

        filtered_sgRNA = gRNA_stringencyIterator(row, refSeqPerChromosome)

        #Score the sgRNAs for this site
        sgRNAdf = sgRNApositionCheck(filtered_sgRNA)

        #Add a column to establish whether mutation has occurred. This will be set to 'True' in the mutator function and starts as False by default.
        sgRNAdf['mutated'] = False

        #If no sgRNAs were found at any stringency
        if len(sgRNAdf) == 0:
            print(f"No sgRNAs found at all stringencies for {ind}.")

            #Just input the information about this site into the final DF. The columns about the guideRNA will be filled with 'NaN', indicating no guideRNA could be found.
            for col in list(row.index):
                TFsdfWinnersandMutated.at[ind, col] = row[col]

        else: #If we have at least one sgRNA
            #Select winner
            winnersgRNA, mutationNeeded = find_best_gRNA(sgRNAdf) #winnersgRNA is a string of the winning sequence, mutationNeeded is a bool variable
            print(winnersgRNA)
            winnerdf = sgRNAdf[sgRNAdf["sgRNA_sequence"] == winnersgRNA] #This is the dataframe row for the winning sgRNA from the original sgRNA dataframe
            winnerdf.reset_index(inplace= True, drop= True)
            winnerdf = winnerdf.loc[0] #This is the pandas series for the same information
            if mutationNeeded == True: #run mutator if indicated
                winnerdf = mutator(winnerdf) #this will mutate HAL or HAR as needed and return the original DF with mutated sequences, and the mutated column set to True

            #Add the winning sgRNA into the output df for this TF start/stop site
            TFsdfWinnersandMutated.loc[ind] = winnerdf

        #Return dataframe as an excel file (This should be unindented one, but while testing I'd like to see the output file updated after each row of the TFsdf)
        TFsdfWinnersandMutated.to_excel("outputFiles/winningsgRNAs.xlsx")

    return TFsdfWinnersandMutated

sgRNArunner()


# Code for generating annotations

class PlanTree:
    def __init__(self, query_plan):
        self.query_plan = query_plan
        self.head = PlanNode(self.query_plan)

    def get_head(self):
        return self.head

    def get_all_node_types(self):
        nodes_set = set()
        queue = [self.head]
        nodes_set.add(self.head.get_node_type())

        while len(queue) != 0:
            temp = []
            while len(queue) != 0:
                cur = queue.pop(0)
                nodes_set.add(cur.get_node_type())
                for child in cur.children:
                    nodes_set.add(child.get_node_type())
                    temp.append(child)
            queue = temp
        return nodes_set

    # For testing only
    def print_tree(self):
        queue = [self.head]
        print("HEAD: ", self.head.get_node_type())
        level = 1
        print("Printing tree by level:")
        while len(queue) != 0:
            temp = []
            # print(level, ": ", end='')
            print(level)
            while len(queue) != 0:
                cur = queue.pop(0)
                print("CUR: ", cur.get_node_type())
                print(cur.get_node_type(), end=' | ')
                # Testing explain_node_type()
                print(cur.explain_node_type())
                print()
                #print(cur.get_misc())
                print("LENGTH OF CHILDREN: ", len(cur.children))
                for child in cur.children:
                    print("CHILD: ", child.get_node_type(), "of cur: ", cur.get_node_type())
                    temp.append(child)
            queue = temp
            print()
            level += 1

    def get_more_info(self):
        queue = [self.head]
        all_info = []
        level = 1
        while len(queue) != 0:
            temp = []
            node_info = []
            print(level)
            while len(queue) != 0:
                cur = queue.pop(0)
                explain_actual_cost = cur.explain_actual_cost()
                explain_cost_diff = cur.explain_cost_diff()
                node_info.append(explain_actual_cost+explain_cost_diff)
                for info in node_info:
                    all_info.append(info)
                for child in cur.children:
                    temp.append(child)
            queue = temp
            print()
            level += 1
            return all_info

    def get_cost_info(self):
        queue = [self.head]
        node_info = []
        level = 1
        while len(queue) != 0:
            temp = []
            # print(level, ": ", end='')
            while len(queue) != 0:
                cur = queue.pop(0)
                est_cost = cur.get_estimated_cost()
                final_est_cost = est_cost.get('Total Cost')
                actual_cost = cur.get_actual_cost()
                final_actual_cost = actual_cost.get('Total Time')
                node_info.append(final_est_cost)
                node_info.append(final_actual_cost)
                for child in cur.children:
                    temp.append(child)
            queue = temp
            level += 1
        return node_info

"""
    Information in each query plan:
    'Node Type', 'Strategy', 'Partial Mode', 'Parallel Aware',
    'Async Capable', 'Startup Cost', 'Total Cost', 'Plan Rows',
    'Plan Width', 'Actual Startup Time', 'Actual Total Time',
    'Actual Rows', 'Actual Loops', 'Output', 'Plans'
    Information extracted by each node:
    Node Type,
    Startup Cost, Total Cost, Plan Rows, Plan Width

"""

class PlanNode:
    def __init__(self, query_plan):
        self.query_plan = query_plan
        self.children = []
        self.node_type = None
        self.estimated_cost = {}
        self.actual_cost = {}
        self.misc = {}
        self.hasFilter = False
        self.filter = {}
        self.parse_plan()
        self.add_children()

    def add_children(self):
        if "Plans" in self.query_plan:
            for plan in self.query_plan['Plans']:
                self.children.append(PlanNode(plan))

    def parse_plan(self):
        self.node_type = self.query_plan['Node Type']

        self.parse_cost()
        self.parse_misc()
        self.parse_filter()
        return

    """
    Parses cost information of the query plan.
    """

    def parse_cost(self):
        self.estimated_cost['Startup Cost'] = self.query_plan['Startup Cost']
        self.estimated_cost['Total Cost'] = self.query_plan['Total Cost']
        self.estimated_cost['Plan Rows'] = self.query_plan['Plan Rows']
        self.estimated_cost['Plan Width'] = self.query_plan['Plan Width']
        self.actual_cost['Startup Time'] = self.query_plan['Actual Startup Time']
        self.actual_cost['Total Time'] = self.query_plan['Actual Total Time']
        self.actual_cost['Actual Rows'] = self.query_plan['Actual Rows']
        self.actual_cost['Actual Loops'] = self.query_plan['Actual Loops']

    """
            Parses miscellaneous information of the query plan.
            Includes Parent Relationship, Plan Width, Output
    """

    def parse_misc(self):
        if 'Parent Relationship' in self.query_plan:
            self.misc['Parent Relationship'] = self.query_plan['Parent Relationship']
        if 'Plan Width' in self.query_plan:
            self.misc['Plan Width'] = self.query_plan['Plan Width']
        if 'Output' in self.query_plan:
            self.misc['Output'] = self.query_plan['Output']

    """
    Parses filter information of the query plan.
    """

    def parse_filter(self):
        if 'Filter' in self.query_plan:
            self.hasFilter = True
        else:
            return
        self.filter['Filter'] = self.query_plan['Filter']
        self.filter['Rows Removed'] = self.query_plan['Rows Removed by Filter']

    def get_node_type(self):
        return self.node_type

    """
    Explains the estimated cost. Returns a string.
    # TODO: relation name
    """

    def explain_node_type(self):
        explain = ""

        # 1. Retrieval Methods
        if self.node_type == "Seq Scan":
            explain += "Relation [{}] is read using Sequential Scan.".format(self.query_plan['Relation Name'])
            if self.hasFilter:
                explain += 'It then filter by [{}].'.format(self.filter['Filter'])
            explain += " This is because no index is created in the table."

        elif self.node_type == "Index Scan":
            explain += 'Relation [{}] is read using the Index Scan '.format(self.query_plan['Relation Name'])
            if "Index Cond" in self.query_plan:
                explain += 'with condition(s): [{}]. '.format(self.query_plan['Index Cond'])
            if self.hasFilter:
                explain += 'It then filter by [{}]. '.format(self.filter['Filter'])

        elif self.node_type == 'Index Only Scan':
            explain += 'Relation [{}] is read using the Index Only Scan '.format(self.query_plan['Relation Name'])
            if "Index Cond" in self.query_plan:
                explain += 'with condition(s): [{}]. '.format(self.query_plan['Index Cond'])
            if self.hasFilter:
                explain += 'It then filter by [{}]. '.format(self.filter['Filter'])

        elif self.node_type == "Bitmap Heap Scan":
            explain += 'Relation [{}] is read using the Bitmap Heap Scan.'.format(self.query_plan['Relation Name'])

        elif self.node_type == "Bitmap Index Scan":
            explain += 'Relation [{}] is read using the Bitmap Index Scan.'.format(self.query_plan['Relation Name'])

        elif self.node_type == "CTE_Scan":
            explain += 'Relation [{}] is read using the CTE_Scan '.format(self.query_plan['Relation Name'])
            if "Index Cond" in self.query_plan:
                explain += 'with condition(s): [{}]. '.format(self.query_plan['Index Cond'])
            if self.hasFilter:
                explain += 'It then filter by [{}]. '.format(self.filter['Filter'])

        # 2. Join Methods
        elif self.node_type == "Hash Join":
            explain += "The Hash Join will joins the previous operations with a hash [{}] join " \
                       "on the condition(s): [{}].".format(self.query_plan['Join Type'], self.query_plan['Hash Cond'])

        elif self.node_type == "Merge Join":
            explain += 'The Merge Join will joins the previous operations using on the condition(s): [{}].'.format(
                [self.query_plan['Merge Cond']])
            if "Join Type" in self.query_plan:
                explain += ' However, only the rows from the left table is returned.'

        elif self.node_type == "Nested Loop":
            explain += "The Nested Loop operation is performed."

        # 3. Sort Method
        elif self.node_type == "Sort":
            explain += 'The Sort operation [{}] sorts the rows based on the key: [{}]'.\
                format(self.query_plan["Sort Method"], self.query_plan['Sort Key'])
            if "INC" in self.query_plan['Sort Key']:
                explain += ' in increasing order.'
            if "DESC" in self.query_plan['Sort Key']:
                explain += ' in decreasing order.'

        elif self.node_type == "Incremental Sort":
            explain += 'The Incremental Sort operation sorts the rows based on the key: [{}]'\
                .format(self.query_plan['Sort Key'])
            if "INC" in self.query_plan['Sort Key']:
                explain += ' in increasing order.'
            if "DESC" in self.query_plan['Sort Key']:
                explain += ' in decreasing order.'

        # 4. Hash
        elif self.node_type == "Hash":
            explain += 'The Hash function will generate a hash table from the records in the previous operation.'
            explain += "The number of Hash Buckets is {}.".format(self.query_plan["Hash Buckets"])

        # 5. Aggregation
        elif self.node_type == "Aggregate":
            strategy = self.query_plan["Strategy"]

            if strategy == "Plain":
                explain += 'The Aggregate operation will be performed.'
            if strategy == "Sorted":
                explain += 'The Aggregate operation will sort the tuples based on the keys: [{}].'.format(
                    self.query_plan["Group Key"])
                if self.hasFilter:
                    explain += ' The result is then filtered by [{}]. '.format(self.filter['Filter'])
            if strategy == "Hashed":
                explain += 'The Aggregate operation will hash the rows based on keys: [{}]. ' \
                           'The selected rows are then returned '.format(self.query_plan["Group Key"])

        elif self.node_type == "Gather":
            explain += 'The Gather Merge operation is performed.' + '\n'

        elif self.node_type == "Limit":
            explain += "The Limit operation will limit the scanning, with a limitation of [{}] rows"\
                .format(self.query_plan['Plan Rows'])
        else:
            explain += "The {} operation will be performed.".format(self.node_type)

        return explain

    """
    Returns startup time, total cost, plan rows, plan width of the query plan.
    """

    def get_estimated_cost(self):
        return self.estimated_cost

    """
    Explains the estimated cost. Returns a string.
    """

    def explain_estimated_cost(self):
        explain = ""

        explain += "The estimated time spending before any output can be produced is " \
                   + str(self.estimated_cost['Startup Cost']) + " disk page fetch(es). "
        explain += "The estimated total cost until completion is " \
                   + str(self.estimated_cost['Total Cost']) \
                   + ", including the costs of all child nodes. "
        explain += "The estimated number of output rows is " \
                   + str(self.estimated_cost['Plan Rows']) + ", and "
        explain += "the average width of each row is " \
                   + str(self.estimated_cost['Plan Width']) + " byte(s)."
        return explain

    """
    Returns startup time, total time, actual rows, actual loops of the actual execution.
    """

    def get_actual_cost(self):
        return self.actual_cost

    """
    Explains the actual execution cost. Returns a string.
    # TODO: BUGG: can only concatenate str to str
    """

    def explain_actual_cost(self):
        explain = ""
        explain += "The actual startup time is " \
                   + str(self.actual_cost['Startup Time']) + " ms. "
        explain += "The actual total time is " \
                   + str(self.actual_cost['Total Time'] * self.actual_cost['Actual Loops']) + " ms " \
                   + "after being executed for " + str(self.actual_cost['Actual Loops']) + " loop(s), " \
                   + "inclusive of the costs of all child nodes. "
        explain += "The total number of rows emitted is " \
                   + str(self.actual_cost['Actual Rows'] * self.actual_cost['Actual Loops']) + "."
        return explain

    """
    Explain the cost difference between the query execution plan and actual execution.
    Returns a string.
    """

    def explain_cost_diff(self):
        explain = ""
        # Explain difference in time
        explain += "The estimation of cost only considers what matters for the planner. " \
                   "Specifically, the estimated cost does not consider the time for transmitting " \
                   "output rows to the client, which cannot be optimized by changing the plan. " \
                   "Therefore, the actual execution time is usually higher. "
        # Explain difference in rows emitted
        est_rows = self.estimated_cost['Plan Rows']
        act_rows = self.actual_cost['Actual Rows'] * self.actual_cost['Actual Loops']
        if est_rows == act_rows:
            # If the estimation and actual execution are the same
            explain += "In this case, the planner yields an accurate estimation of the number of rows omitted."
        elif est_rows > act_rows:
            # If the estimation > actual execution
            explain += "In estimation, the node is assumed to be run until full completion. " \
                       "However, the plan node execution can be stopped early due to LIMIT or similar effect. " \
                       "Thus, the actual number of rows emitted is less than estimated."
        else:
            # If the estimation < actual execution
            explain += "Sometimes the node is backed up and rescanned, " \
                       "and the planner counts these repeated emissions as if they were real additional rows. " \
                       "Consequently, the reported actual row counts can be higher than estimated."
        return explain

    """
        Returns miscellaneous information of the node.
        Includes Parent Relationship, Plan Width, Output
    """

    def get_misc(self):
        return self.misc

    """
    Returns filter information of the query plan.
    Includes filter conditions and number of rows removed by filter.
    """

    def get_filter(self):
        if not self.hasFilter:
            return None
        return self.filter


# -*- coding: utf-8 -*-
"""
Created on Tue Aug 23 09:43:36 2016

Functions to project cortical data onto a 2D plane via the sphere and back again;
select patches and augment data/labels
@author: ecr05
"""

import sys
import os

import numpy as np
import copy
import matplotlib.pyplot as plt
import Interpolation as intp
import pandas as pd

def sliceplane(plane, start_height, start_width, x_size, y_size):
    # slice patch from 3D array dim (x,y,channels)
    newdata = plane[int(start_height):int(start_height+x_size), int(start_width):int(start_width+y_size),:]
    return newdata

def normalize(data,norm_axis):
    """
        normalise feature maps

        Parameters
        ----------
        data : multivariate feature maps as 3d np.arrays dims (x,y,channels)

        Returns
        -------
        datanormed : normalised data
    """

    datastd = np.std(data,axis=norm_axis)
   # print('in normalise', data.shape ,norm_axis, datastd.shape )
   # print('mean',np.mean(data,axis=norm_axis).shape )
    if np.where(datastd == 0)[0].shape != (0,):
        print('isnan')
        datastd[np.where(datastd==0)]=1;

    datanormed = (data - np.mean(data,axis=norm_axis)) / datastd
    if np.where(np.isnan(datanormed))[0].shape != (0,):
         print('isnan2')

    #print('mean normalised',np.mean(datanormed,axis=norm_axis))
    #print('std normalised',np.std(datanormed,axis=norm_axis))

    return datanormed

def rodrigues_rotation(origin,newpt):
    """
            rotate sphere to move longitude 0, latitude 45 degrees to a new position centred on the newpt

            Parameters
            ----------
            origin: current projection centre
            newpt: coordinate intended as new projection centre

            Returns
            -------
            Rot: (3x3) array rotation matrix
    """
    #
    #http://electroncastle.com/wp/?p=39
    #https://en.wikipedia.org/wiki/Rodrigues%27_rotation_formula
    p1=intp.normalize(origin)
    p2=intp.normalize(newpt)

    angle = np.arccos(np.dot(p1,p2)) # get angle of rotation

    if angle> np.pi:
        raise ValueError('rodrigues rotation error: the specified rotation angle exceeds pi')
    else:
        print("rodrigues rotation: angle of rotation is %f"%angle)

    ID = np.identity(3)

    # get axis of rotation
    cross = np.cross(p1,p2)
    cross = intp.normalize(cross)
    # get rotation matrix
    if angle == 0:
        Rot = ID
    elif intp.normalize(cross) == 0:
        Rot = np.negative(ID);
    else:

       u = np.array([[0, -cross[2], cross[1]],
                  [cross[2], 0, -cross[0]],
                  [-cross[1], cross[0],0]])
       Rot = ID+np.sin(angle)*u+(1-np.cos(angle))*(u.dot(u))

    return Rot

def rotate(rotation, coords):
    """
           rotate coordinates

           Parameters
           ----------
           rotation: rotation matrix
           coords: coordinate matrix

           Returns
           -------
           Rot: (nx3) array of rotated n points
    """
    coordsrot = np.dot(rotation,np.transpose(coords));
    return np.transpose(coordsrot)

def rescale_labels(labels,labelfunc):
    # rest original label indexing
    func=copy.deepcopy(labelfunc)
    for l,label in enumerate(labels):

        x = np.where(labelfunc == l+1)#[0]
        func[(x)]=label

    return func

def zero_labels(labels,labelfunc):
    # labels must index concurrently from 0
    func=copy.deepcopy(labelfunc)
    func = func * 0
    newlabel=[]
    for l,label in enumerate(labels):

        x = np.where(labelfunc == label)#[0]

        func[(x)]=l+1
        newlabel.append(l+2)

    return func,newlabel

def invert_projection_test(data2D,coords,interp):

    DATA=np.zeros((coords.shape[0],data2D.shape[2]))

    # fill data with cortical features
    for x,index in enumerate(coords):
        flatindex=interp.invcorrespondence[x];
        latind=flatindex[0]
        lonind=flatindex[1]
    # fill data with cortical features

        DATA[x,:]=data2D[latind,lonind,:]

    return DATA

def invert_patch_categories_full(img,coords,interp,newH,newW,labels=[]):

    #plt.imshow(img[0,:,:,0])
    #plt.show()
    #zeropadded=inv_slice(croppedrescale2,newH,newW,h,w)
    SURFdata=invert_projection_test(img,coords,interp)
    #SURFdata=invert_projection_test(img[0,:,:,:],coords,interp,newH,newW,lons)
    if len(labels) > 0:
        rescaledlabels = rescale_labels(labels, SURFdata)
        print('rescaled shape', rescaledlabels.shape)
        return rescaledlabels
    else:
        return SURFdata

def remove_outliers(d,data):
    # quick hack to get rid of xtreme outliers (before there was some really high values)
    if  (np.max(data)>np.mean(data)+10*np.std(data)) or (np.min(data)<np.mean(data)-10*np.std(data)):
        print(d,  'data stats:', np.mean(data), np.std(data), np.max(data), np.min(data),np.mean(data)+10*np.std(data),np.mean(data)-10*np.std(data))

        data[np.where(data>np.mean(data)+10*np.std(data))]= np.mean(data)

def reformat_datamat_rows_to_columns(ALLDATA):
    newmat=np.zeros((ALLDATA.samples*ALLDATA.DATA.shape[0],ALLDATA.features))

    for f in range(ALLDATA.features):
        for s in range(ALLDATA.samples):
            #print(f,s,'ALLDATA.DATA[:,f*s] shape',s*ALLDATA.DATA.shape[0],(s+1)*ALLDATA.DATA.shape[0],newmat.shape,ALLDATA.DATA.shape[0],ALLDATA.DATA[:,f*s].shape,newmat[s*ALLDATA.DATA.shape[0]:(s+1)*ALLDATA.DATA.shape[0],f].shape)
            newmat[s*ALLDATA.DATA.shape[0]:(s+1)*ALLDATA.DATA.shape[0],f]= ALLDATA.DATA[:,s*ALLDATA.features+f]

    return newmat

def reformat_datamat_columns_to_rows(datamat,ALLDATA):
    newmat=np.zeros(ALLDATA.DATA.shape)

    for f in range(ALLDATA.features):
        for s in range(ALLDATA.samples):
            #print(f,s,'datamat[s*ALLDATA.DATA.shape[0]:,f]',datamat[s*ALLDATA.DATA.shape[0]:(s+1)*ALLDATA.DATA.shape[0],f].shape,newmat[:,f*s].shape)
            newmat[:,s*ALLDATA.features+f]= datamat[s*ALLDATA.DATA.shape[0]:(s+1)*ALLDATA.DATA.shape[0]:,f]

    return newmat

def group_normalise(ALLDATA):
    """
          normalise data across group; first restructures data s.t. all examples for each feature fit into one column
          i.e. (n_subjects*n_vertices,n_features)
          then it normalises and returns data to (n_vertices,n_subjects*n_features) format

          Parameters
          ----------
          ALLDATA: data for all subjects in (n_vertices,n_subjects*n_features) format

          Returns
          -------

    """
    #alldatamean=np.mean(ALLDATA.DATA,axis=0)
    newmat=reformat_datamat_rows_to_columns(ALLDATA)
    newmat=normalize(newmat,0) # normalise along each column
    newmat2=reformat_datamat_columns_to_rows(newmat,ALLDATA)
# =============================================================================
#     diff=ALLDATA.DATA-newmat2
#     print('diff',np.sum(diff))
# =============================================================================
    #print('group_normalise_mean',np.mean(newmat,axis=0))
    #ALLDATA._replace(DATA=newmat)
    return newmat2

def project(DATAset, interp, nlats, nlons, lons):
    """
          project points from sphere onto 2D grid

          Parameters
          ----------
          DATAset: labels/features to be projected
          interp: saved correspondences between the sphere and 2D grid
          nlats: number of bins in x direction
          nlons: number of bins in y direction
          lons: edges of longitude bins

          Returns
          -------
          Rot: (nx3) array of rotated n points
    """

    data = np.zeros((nlats, nlons, DATAset.shape[1]))

    # fill data with cortical features
    for index, x in np.ndenumerate(data[:, :, 0]):
        latind = index[0]
        lonind = index[1]

        data[latind, lonind, :] = DATAset[interp.correspondence[index], :]

    return data

def project_data(DATA,interp,data_paths,newH,newW,use_normalisation):
    """
         project labels, featuremaps and optionally feature-correlation maps onto 2D plan

         Parameters
         ----------
         alllabels: np.array containing label surface data for all subjects within a group
         alldata: np.array containing multivariate feature maps for all subjects in a group
         allcorr:  np.array containing correlation (of feature) maps for all subjects in a group
         interp: saved correspondences between the sphere and 2D grid
         outdir: output directory
         newH: y dimensions of 2d projection
         newW: x dimensions of 2d projection
         lons: edges of longitude bins
         abr: group type
         aug: numerical index relating to the number of projection centres
         usegroup: use group labels
         usecorrelations: if defined use feature correlation maps and project these to 2D plane

         Returns
         -------
   """
    delta_d=2.*np.pi/(newW-1); # width of longitude bin if w not in  randw2:s
    lons = (delta_d*np.indices((newW,1))[0,:,:]) #edges of longitude bins

    # project data to 2D
    alldata=DATA['data']

    twoD = project(alldata.DATA, interp, newH, newW, lons)

    if 'labels' in DATA:
        print('in project data, labels== true')
        # if working with label files then also project these to 2D
        twoDL = project(DATA['labels'],interp,newH,newW,lons)

    start=0
    for subj in range(0,alldata.samples):
        #print('subj', subj, DATA['data'].ids[subj] )
        if 'labels' in DATA :
            np.save(os.path.join(data_paths['Odirname'],data_paths['abr'] +'GrayScaleLabels-subj-'+ DATA['data'].ids[subj]), twoDL[:,:,subj])

        if use_normalisation:
            data_normed=normalize(np.reshape(twoD[:, :, start:start + alldata.features],(twoD.shape[0]*twoD.shape[1],alldata.features)),0)
           # print(data_normed.shape,np.std(data_normed,0).shape,np.std(data_normed,0),np.mean(data_normed,0))

            data=np.reshape(data_normed,twoD[:, :, start:start + alldata.features].shape)
        else:
            data=twoD[:, :, start:start + alldata.features]

        np.save(os.path.join(data_paths['Odirname'], data_paths['abr'] + 'data_-subj-' + DATA['data'].ids[subj]), data)

        start = start+alldata.features

def write_projection_paths(DATA, filename, data_paths, use_labels, use_normalisation):
    """
        write paths out to file

         Parameters
         ----------
         samples: list of subj ids
         filename: output filename
         indir: path to data files
         aug: indexing projection centres
         use_grouplabels: ue group labels
         use_correlations: save featuremap correlations

         Returns
         -------
       """

    df = pd.DataFrame()
    print('write projection paths')
    for subj in range(0,DATA.samples):
        #print('subj', subj, DATA.ids[subj],type(DATA.ids[subj]) )
        if use_labels:
                label = os.path.join(data_paths['Odirname'], data_paths['abr'] + 'GrayScaleLabels-subj-' + DATA.ids[subj]+ '.npy')

        data = os.path.join(data_paths['Odirname'], data_paths['abr'] + 'data_-subj-' +DATA.ids[subj]+  '.npy')

        row={};
        row['fileid']=  DATA.ids[subj]
        row['data']= data
        if use_labels:
            row['labels']= label

        df = df.append(row, ignore_index=True)
    print('types',df['fileid'].dtype)
   # print('df shape', df.shape)
#    df['fileid']=df['fileid'].astype(int)

    if data_paths['meta_csv'] is not None:
        meta_data=pd.read_pickle(data_paths['meta_csv'])
        print('types',type(meta_data['fileid']),type(df['fileid']))
       # print('meta shape', meta_data.shape)
        meta_data['fileid']=meta_data['fileid'].astype(object)
        output=df.merge(meta_data, on=['fileid'])
    else:
        output=df

    #output=output.drop(['fileid'], axis=1)
    output.to_pickle(os.path.join(data_paths['Odirname'],filename))

# =============================================================================
#         if use_correlations:
#             corrdata=os.path.join(indir, abr + 'featurecorrelations-subj-' +DATA.ids[subj] + '-aug-' + aug + '.npy')
#             meancorrdata=os.path.join(indir, abr + 'meanfeaturecorrelations-aug-' + aug+'.npy')
#             #target.write(data + ' ' + label + ' ' + corrdata + ' ' + meancorrdata + '\n')
#         else:
#             target.write(data + ' ' + label + '\n')
#
# =============================================================================


import argparse
import csv
import json
import numpy as np
from sklearn.metrics import accuracy_score, confusion_matrix, f1_score

from fever_utils import split_sentence_id

def predict(args):
    preds = {}

    def aggregate(query_id, scores, sent_ids):
        pred = {}

        best = np.argmax(scores[0])

        pred['id'] = query_id
        if best == 0:
            pred['predicted_label'] = 'REFUTES'
        elif best == 1:
            pred['predicted_label'] = 'NOT ENOUGH INFO'
        else:  # best == 2
            pred['predicted_label'] = 'SUPPORTS'
        pred['predicted_evidence'] = [list(split_sentence_id(sent)) for sent in sent_ids]

        return best, pred

    with open(args.id_file, 'r', encoding='utf-8') as f_id, open(args.scores_file, 'r', encoding='utf-8') as f_scores, \
            open(args.output_predictions_file, 'w', encoding='utf-8') as f_out:
        print('Reading scores file...')
        curr_query = None
        curr_sent_ids = []
        curr_scores = []
        for id_line, scores_line in zip(f_id, f_scores):
            if args.has_labels:
                query_id, sent_ids, label = id_line.strip().split('\t')
            else:
                query_id, sent_ids = id_line.strip().split('\t')
            query_id = int(query_id)
            _, false_score, nei_score, true_score = scores_line.strip().split('\t')

            if query_id != curr_query:
                if curr_query is not None:
                    best, pred = aggregate(curr_query, curr_scores, curr_sent_ids)
                    json.dump(pred, f_out)
                    f_out.write('\n')
                    preds[curr_query] = best
                curr_query = query_id
                curr_sent_ids.clear()
                curr_scores.clear()

            curr_sent_ids = sent_ids.split(' ')
            curr_scores.append((float(false_score), float(nei_score), float(true_score)))

        best, pred = aggregate(curr_query, curr_scores, curr_sent_ids)
        json.dump(pred, f_out)
        f_out.write('\n')
        preds[curr_query] = best

    # print label prediction metrics if dataset file provided
    if args.dataset_file:
        actual_labels = []
        pred_labels = []
        with open(args.dataset_file, 'r', encoding='utf-8') as f:
            print('Reading FEVER dataset file...')
            for line in f:
                line_json = json.loads(line.strip())

                label = line_json['label']
                if label == 'SUPPORTS':
                    actual_labels.append(2)
                elif label == 'REFUTES':
                    actual_labels.append(0)
                else:  # label == 'NOT ENOUGH INFO'
                    actual_labels.append(1)

                query_id = line_json['id']
                pred_labels.append(preds[query_id])

        print('****************************************')
        print(f'Number of Queries: {len(actual_labels)}')
        print(f'Label Accuracy: {accuracy_score(actual_labels, pred_labels)}')
        print(f'Label F1: {f1_score(actual_labels, pred_labels, average=None)}')
        print(confusion_matrix(actual_labels, pred_labels))
        print('****************************************')

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Predicts labels and evidence sentences for FEVER submission.')
    parser.add_argument('--id_file', required=True, help='Input query-doc pair ids file.')
    parser.add_argument('--scores_file',
                        required=True,
                        help='Prediction scores file outputted by T5 label prediction model.')
    parser.add_argument('--dataset_file', help='FEVER dataset file (only if labelled).')
    parser.add_argument('--output_predictions_file',
                        required=True,
                        help='Output predictions file in FEVER submission format.')
    parser.add_argument('--evidence_k',
                        type=int,
                        default=5,
                        help='Number of top sentences to use as evidence for FEVER submission.')
    parser.add_argument('--has_labels', action='store_true', help='Whether the id file is labelled.')
    args = parser.parse_args()

    predict(args)

    print('Done!')


from books.models import Book

# Initialize session for basket
class Basket():

    def __init__(self, request):
        self.session = request.session
        basket = self.session.get('skey')
        if 'skey' not in request.session:
            basket = self.session['skey'] = {}
        self.basket = basket

# Adding and updating the basket session data
    def add(self, book):
        book_id = book.id
        if book_id not in self.basket:
             self.basket['book_id'] = {'price': str(book.price)}

        self.session.modified = True


"""
@Author-anonymousP
@__WHEN YOU FEEL LIKE QUITTING, THINK ABOUT WHY YOU STARTED__@
"""

import numpy as np
import matplotlib.pyplot as plt
from itertools import combinations
from numpy.polynomial import Polynomial

def l(k, x):
    n = len(x)
    assert (k < len(x))

    x_k = x[k]
    x_copy = np.delete(x, k)

    denominator = np.prod(x_copy - x_k)

    coeff = []

    for i in range(n):
        coeff.append(sum([np.prod(x) for x in combinations(x_copy, i)]) * (-1) ** (i) / denominator)

    coeff.reverse()

    return Polynomial(coeff)

def h(k, x):
    # initialize with none
    l_k = None
    l_k_sqr = None
    l_k_prime = None
    coeff = None
    p = None
    # place your code here!!!!!!!!!!!!!!!!!!!!!!!

    l_k = l(k,x)
    l_k_sqr = l_k*l_k
    l_k_prime = l_k.deriv(1)
    coeff = [1 + 2*x[k]*l_k_prime(x[k]), (-2)*l_k_prime(x[k])]
    p = Polynomial(coeff)

    return p * l_k_sqr

def h_hat(k, x):
    # Initialize with none
    l_k = None
    l_k_sqr = None
    coeff = None
    p = None
    # place your code here!!!!!!!!!!!!!!!!!!!!!!!

    l_k = l(k,x)
    l_k_sqr = l_k*l_k
    coeff = [(-1)*x[k], 1]
    p = Polynomial(coeff)

    return p * l_k_sqr

def hermit(x, y, y_prime):
    assert (len(x) == len(y))
    assert (len(y) == len(y_prime))

    f = Polynomial([0.0])
    for i in range(len(x)):
        # f += ?
        pass  # pass statement does nothing
        # place your code here!!!!!!!!!!!!!!!!!!!!!!!
        f += y[i] * h(i, x) + y_prime[i] * h_hat(i, x)
    return f

pi = np.pi
x       = np.array([0.0, pi/2.0,  pi, 3.0*pi/2.0])
y       = np.array([0.0,    1.0, 0.0,       -1.0])
y_prime = np.array([1.0,    0.0, 1.0,        0.0])

n = 1
f3     = hermit(x[:(n+1)], y[:(n+1)], y_prime[:(n+1)])
data   = f3.linspace(n=50, domain=[-3, 3])
test_x = np.linspace(-3, 3, 50, endpoint=True)
test_y = np.sin(test_x)

plt.plot(data[0], data[1])
plt.plot(test_x, test_y)
plt.show()

n = 2
f5     = hermit(x[:(n+1)], y[:(n+1)], y_prime[:(n+1)])
data   = f5.linspace(n=50, domain=[-0.7, 3])
test_x = np.linspace(-2*pi, 2*pi, 50, endpoint=True)
test_y = np.sin(test_x)

plt.plot(test_x, test_y)
plt.plot(data[0], data[1])
plt.show()

n = 3
f7     = hermit(x[:(n+1)], y[:(n+1)], y_prime[:(n+1)])
data   = f7.linspace(n=50, domain=[-0.3, 3])
test_x = np.linspace(-2*pi, 2*pi, 50, endpoint=True)
test_y = np.sin(test_x)

plt.plot(data[0], data[1])
plt.plot(test_x, test_y)
plt.show()

#defining new set of given node information: x, y and y'
x       = np.array([0.0, 1.0,          2.0       ])
y       = np.array([1.0, 2.71828183,  54.59815003])
y_prime = np.array([0.0, 5.43656366, 218.39260013])

f7      = hermit( x, y, y_prime)
data    = f7.linspace(n=50, domain=[-0.5, 2.2])
test_x  = np.linspace(-0.5, 2.2, 50, endpoint=True)
test_y  = np.exp(test_x**2)

plt.plot(data[0], data[1])
plt.plot(test_x, test_y)
plt.show()

#defining new set of given node information: x, y and y'
x       = np.array([1.0, 3.0, 5.0])
y       = np.array([5.0, 1.0, 5.0])
y_prime = np.array([-4.0, 0.0, 4.0])

f7      = hermit( x, y, y_prime)
data    = f7.linspace(n=50, domain=[-10, 10])
test_x  = np.linspace(-10, 10, 50, endpoint=True)
test_y  = (test_x-3)**2 + 1

plt.plot(data[0], data[1])
plt.plot(test_x, test_y)
plt.show()


"""局面を思考するモジュール"""

import pre_think

def create_thinking_main(lst_, str_):
  """局面を思考するメイン関数

  Args:
    lst_: 現在の局面の配列
    str_: USIプロトコルの「position startpos」で始まるコマンドの文字列

  Returns:
    str: 指し手（bestmove）のコマンドの文字列

  """
  # 受け取ったコマンドを解析して格納する配列
  parsed_lst = pre_think.new_cmd_analysis_lst(str_)

  # 手数の表示
  print("手数: " + str(pre_think.make_moves(parsed_lst)) )

  # いろはが先手か後手かを格納する変数
  iroha_turn = pre_think.make_turn_of_iroha(parsed_lst)
  print("いろはの手番: " + iroha_turn)

  # 仮の戻り値
  return "bestmove 7g7f"


import logging
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy.orm import Session
import pandas as pd
from datetime import timedelta
import json

from database import get_db_session
from models import IndustryReturns, PValueData, ResTest, SE, VifValues, XDict, Shares, Returns

model_router = APIRouter()

logger = logging.getLogger(__name__)

def get_model_data_by_date(db: Session, table, start_date: str, end_date: str):
    filters = []

    if start_date != "None":
        filters.append(table.index >= start_date)
    if end_date != "None":
        filters.append(table.index <= end_date)

    if len(filters) != 0:
        return db.query(table).filter(*filters).all()

    return db.query(table).all()

@model_router.get("/returns")
def get_returns(db: Session = Depends(get_db_session), start_date="2022-01-01"):
    target_weights_dict = {
                                  'uniswap': 0.530646,
                                  'pancakeswap-token': 0.418644,
                                  'woo-network': 0.050710
                             }
    performance_frame = pd.DataFrame(0.0, index = [(pd.to_datetime(start_date) - timedelta(days = 1)).strftime('%Y-%m-%d')],
                                        columns = list(target_weights_dict.keys()))
    table = Returns
    query = db.query(table).filter(table.index >= start_date)
    returns_frame = pd.read_sql(query.statement, query.session.bind)
    returns_frame.set_index("index", inplace=True)
    for day in returns_frame.loc[start_date:].index:
        for token in performance_frame.columns:
            performance_frame.loc[day, token] = returns_frame.loc[day][token] * target_weights_dict[token]

    per_to_plot = pd.DataFrame(index = list(performance_frame.index)[1:], columns = ['performance'])
    per_to_plot['performance'] = list(performance_frame.sum(axis=1).add(1).cumprod().shift(1).iloc[1:])
    return per_to_plot

@model_router.get("/weights")
def get_weights():
    tokens_dict = {'UNI': '0xBf5140A22578168FD562DCcF235E5D43A02ce9B1',
     'CAKE': '0x0E09FaBB73Bd3Ade0a17ECC321fD13a19e81cE82',
     'WOO': '0x4691937a7508860F876c9c0a2a617E7d9E945D4B'}

    from web3 import Web3

    minABI = [
                        {
                            "constant": True,
                            "inputs": [{'name': "_owner", 'type': "address"}],
                            "name": "balanceOf",
                            "outputs": [{'name': "balance", 'type': "uint256"}],
                            "type": "function",
                        },
                        {
                            "constant": True,
                             "inputs":[],
                             "name":"decimals",
                             "outputs":[{"name":"","type":"uint8"}],
                             "payable": False,
                             "stateMutability":"view",
                             "type":"function"
                        }
                    ]

    web3 = Web3(Web3.HTTPProvider("https://bsc-mainnet.nodereal.io/v1/58416516ddbb492a8a9acd27ee7c09cd"))

    from pycoingecko import CoinGeckoAPI
    cg = CoinGeckoAPI()
    suka = cg.get_price(ids=['pancakeswap-token', 'uniswap', 'woo-network'], vs_currencies = 'usd')

    quotes = {"CAKE": suka["pancakeswap-token"]["usd"], "UNI": suka["uniswap"]["usd"], "WOO": suka["woo-network"]["usd"]}

    dollar_value_to_token = {}
    for token in tokens_dict.keys():
        try:
            contract = web3.eth.contract(address=tokens_dict[token], abi=minABI)
            token_balance = contract.functions.balanceOf('0x2b50BCd2A3f3568dCad84Efbc38f908e49a6F463').call()
            dec = contract.functions.decimals().call()
            dollar_value_to_token[token] = token_balance / 10 ** dec * quotes[token]
        except Exception as e:
            print(e)
            try:
                contract = web3.eth.contract(address=tokens_dict[token], abi=minABI)
                token_balance = contract.functions.balanceOf('0x2b50BCd2A3f3568dCad84Efbc38f908e49a6F463').call()
                dec = contract.functions.decimals().call()
                dollar_value_to_token[token] = token_balance / 10 ** dec * quotes[token]
            except:
                pass
    s = 0
    for val in dollar_value_to_token.values():
        s += val

    for key, val in dollar_value_to_token.items():
        dollar_value_to_token[key] /= s

    return dollar_value_to_token

@model_router.get("/shares_count")
def get_shares_count():
    tokens_dict = {'UNI': '0xBf5140A22578168FD562DCcF235E5D43A02ce9B1',
     'CAKE': '0x0E09FaBB73Bd3Ade0a17ECC321fD13a19e81cE82',
     'WOO': '0x4691937a7508860F876c9c0a2a617E7d9E945D4B'}

    from web3 import Web3

    with open("VaultV1.json") as f:
        vault_abi = json.load(f)
    web3 = Web3(Web3.HTTPProvider("https://bsc-mainnet.nodereal.io/v1/58416516ddbb492a8a9acd27ee7c09cd"))
    Vault = web3.eth.contract(address="0xbe9080Fe628F073633DC2dcFA9d3CC0cc38D4805", abi=vault_abi["abi"])
    total_share_supply = round(Vault.functions.totalSupply().call() / 10**18, 3)

    return total_share_supply

@model_router.get("/{table_name}")
def api_data(table_name: str, start_date: str = "None",
             end_date: str = "None", db: Session = Depends(get_db_session)):
    if table_name == "IndustryReturns":
        table = IndustryReturns
    elif table_name == "PValueData":
        table = PValueData
    elif table_name == "ResTest":
        table = ResTest
    elif table_name == "SE":
        table = SE
    elif table_name == "VifValues":
        table = VifValues
    elif table_name == "XDict":
        table = XDict
    elif table_name == "Shares":
        table = Shares
    elif table_name == "Returns":
        table = Returns
    else:
        raise HTTPException(status_code=404, detail="Table not found")
    data = get_model_data_by_date(db, table, start_date, end_date)
    return data


'''
The populate function populates a region with caves.
'''

from amulet import world_interface
from amulet.api.selection import *
from amulet.api.block import *
from amulet.operations import fill
from amulet.api.world import *
import random
import math

def cave_section(center, width):
    #Right now, just a cube
    #We also return information about the shape, to be used to accelerate edge detection.
    c_x = center[0]
    c_y = center[1]
    c_z = center[2]
    l_x = c_x-int((width/2))
    l_y = c_y-int((width/2))
    l_z = c_z-int((width/2))
    min = (l_x, l_y, l_z)
    max = (l_x+width, l_y+width, l_z+width)

    return Selection([SubSelectionBox(min, max)]), (min, (width, width, width))

def strict_sigmoid(input, clamp):
    #returns -1 if input < -clamp
    #returns 1 if input > clamp
    #else, returns 0
    if (input < -1*clamp):
        return -1
    elif (input > clamp):
        return 1
    else:
        return 0

def scale(input, input_lo, input_hi, output_lo, output_hi):
    return (((input - input_lo) * (output_hi - output_lo)) / (input_hi - input_lo)) + output_lo

def shift(input, mod):
    #If input = 1 or -1,
    #   and mod = input, return input
    #   and mod = -1*input, return 0
    #   and mod = 0, return input
    #Else, return mod
    if (abs(input) == 1):
        if input == mod:
            return input
        elif input == -1*mod:
            return 0
        else:
            return input
    else:
        return mod

def shift_vector(input, mod):
    return (shift(input[0], mod[0]), shift(input[1], mod[1]), shift(input[2], mod[2]))

def add_vectors(input1, input2):
    return (input1[0]+input2[0], input1[1]+input2[1], input1[2]+input2[2])

def make_cave(world, region, start, iterations, min_width, max_width, block):
    #Make a single cave
    #Also return information about the shape of the cave.
    for i in region.subboxes:
        MIN_X = i.min_x
        MIN_Y = i.min_y
        MIN_Z = i.min_z
        MAX_X = i.max_x
        MAX_Y = i.max_y
        MAX_Z = i.max_z

    cave_shapes = []

    position = start
    vector = (0, 0, 0)
    width = int(min_width+((max_width-min_width)/2))
    for i in range(iterations):
        px = random.choice([-1, 0, 1]) #pnoise1(i/1000, octaves = 3)
        py = random.choice([-1, 0, 1]) #pnoise1(px, octaves = 3)
        pz = random.choice([-1, 0, 1]) #pnoise1(py, octaves = 3)
        width+=random.choice([-1, 0, 1])
        if width < min_width:
            width = min_width
        elif width > max_width:
            width = max_width

        #mod = (strict_sigmoid(px-0.5, clamp), strict_sigmoid(py-0.5, clamp), strict_sigmoid(pz-0.5, clamp))
        mod = (px, py, pz)
        vector = shift_vector(vector, mod)
        position = add_vectors(position, vector)

        section, shape = cave_section(position, width)
        cave_shapes.append(shape)
        stop = False
        for i in section.subboxes:
            if i.min_x < MIN_X or i.min_y < 0 or i.min_z < MIN_Z:
                stop = True
            if i.max_x > MAX_X or i.max_y > 255 or i.max_z > MAX_Z:
                stop = True
        if stop:
            #print("OUT OF BOUNDS")
            break

        fill.fill(world, 0, section, {'fill_block': block})
        #print(F"I: {i} POS: {position} VEC: {vector} WIDTH: {width} MOD: {mod} PX: {px} PY: {py} PZ: {pz}")
    #print("END OF ITERATION")
    return cave_shapes

def clamp(number, low, high):
    if number < low:
        return low
    elif number > high:
        return high
    else:
        return number

def populate(world, region, block, fill_cube_size = 32, min_width = 2, max_width = 10, iterations = 1000):
    '''
    Populates a region with caves.
    world - The world to do this replacing in.
    region - A Selection specifying the region to place the caves in. Should be a single sub box!
    block - The block to fill the caves with. Air, usually.
    fill_cube_size - Make sure at least one cave is in each cube with this dimension in the region.
    min_width - Minumum width of a cave.
    max_width - Maximum width of a cave.
    iterations - Essentially, how long a cave can get.
    '''
    for i in region.subboxes:
        MIN_X = i.min_x
        MIN_Y = i.min_y
        MIN_Z = i.min_z
        MAX_X = i.max_x
        MAX_Y = i.max_y
        MAX_Z = i.max_z

    cave_shapes = []
    FILL_CUBE_SIZE = fill_cube_size
    for x_i in range(int(math.ceil((MAX_X-MIN_X)/FILL_CUBE_SIZE))):
        x_lo = clamp(MIN_X+x_i*FILL_CUBE_SIZE, MIN_X, MAX_X)
        x_hi = clamp(x_lo+FILL_CUBE_SIZE, MIN_X, MAX_X)
        for y_i in range(int(math.ceil((MAX_Y-MIN_Y)/FILL_CUBE_SIZE))):
            y_lo = clamp(MIN_Y+y_i*FILL_CUBE_SIZE, MIN_Y, MAX_Y)
            y_hi = clamp(y_lo+FILL_CUBE_SIZE, MIN_Y, MAX_Y)
            for z_i in range(int(math.ceil((MAX_Z-MIN_Z)/FILL_CUBE_SIZE))):
                z_lo = clamp(MIN_Z+z_i*FILL_CUBE_SIZE, MIN_Z, MAX_Z)
                z_hi = clamp(z_lo+FILL_CUBE_SIZE, MIN_Z, MAX_Z)

                start = (random.randrange(x_lo, x_hi), random.randrange(y_lo, y_hi), random.randrange(z_lo, z_hi))
                min_width_specific = random.randrange(min_width,max_width)
                max_width_specific = random.randrange(min_width_specific, max_width)
                #print(f"({x_i}, {y_i}, {z_i}) MAKING A CAVE AT {start}. WIDTH: {min_width_specific}-{max_width_specific}. MAX_LENGTH: {iterations}")
                shapes = make_cave(world,region,start,iterations, min_width_specific, max_width_specific, block)
                cave_shapes+=shapes
    return cave_shapes