import argparse
import json
import os
import sys
import pandas as pd
import copy

from helpers import multitab_excel_to_single_txt
from helpers import get_protocol_map
from helpers import fetch_fastq_path
from helpers import utils
from helpers import check_experimental_design

pd.options.mode.chained_assignment = None

def rename_technology_type(technology_type, technology_dict):

    json_file = technology_dict[technology_type]

    return json_file

def get_secondary_accessions(xlsx_dict, args):
    secondary_accessions = []
    secondary_accessions.append(args.project_uuid)
    keys = ["project.geo_series_accessions","project.insdc_project_accessions","project.insdc_study_accessions","project.biostudies_accessions"]
    for key in keys:
        items = xlsx_dict["project"][key].fillna('')
        items = [item for item in items if item != '']
        if items:
            if "||" in items:
                items = items.split("||")
            else:
                items = items
            secondary_accessions.extend(items)
    return secondary_accessions

def get_person_roles(xlsx_dict):

    person_roles = utils.reformat_value(xlsx_dict, "project_contributors", "project.contributors.project_role.text", "str")
    person_roles_submitter = utils.reformat_value(xlsx_dict, "project_contributors",
                                                  "project.contributors.corresponding_contributor", "str")

    for (i, elem) in enumerate(person_roles_submitter):
        person_roles[i] = person_roles[i].lower()
        if elem == "yes":
            person_roles[i] = "submitter"
        elif elem == "no":
            if "curator" in person_roles[i]:
                person_roles[i] = "data curator"
            else:
                person_roles[i] = ""

    return person_roles

def get_author_list(xlsx_dict):

    authors = utils.reformat_value(xlsx_dict, "project_publications", "project.publications.authors", "str")[0]
    author_list = authors.replace("||",", ")

    return author_list

def generate_idf_file(work_dir, args, dataset_protocol_map, xlsx_dict, accession, idf_file_name,
                      sdrf_file_name):

    tab = '\t'
    person_roles = get_person_roles(xlsx_dict)
    secondary_accessions = get_secondary_accessions(xlsx_dict, args)
    protocol_fields = get_protocol_map.get_idf_file_protocol_fields(dataset_protocol_map)
    author_list = get_author_list(xlsx_dict)

    related_scea_accessions = None
    related_scea_accessions = args.related_scea_accession

    if related_scea_accessions:

        idf_file_contents = f"""\
MAGE-TAB Version\t1.1
Investigation Title\t{utils.reformat_value(xlsx_dict, "project", "project.project_core.project_title", "str")[0].strip('.')}
Comment[Submitted Name]\t{utils.reformat_value(xlsx_dict, "project", "project.project_core.project_short_name", "str")[0]}
Experiment Description\t{utils.reformat_value(xlsx_dict, "project", "project.project_core.project_description", "str")[0]}
Public Release Date\t{args.public_release_date}
Person First Name\t{utils.get_tab_separated_list(xlsx_dict, "project_contributors", "project.contributors.name", lambda x: x.split(',')[0])}
Person Last Name\t{utils.get_tab_separated_list(xlsx_dict, "project_contributors", "project.contributors.name", lambda x: x.split(',')[2])}
Person Mid Initials\t{utils.get_tab_separated_list(xlsx_dict, "project_contributors", "project.contributors.name", lambda x: utils.get_first_letter(x.split(',')[1]))}
Person Email\t{utils.get_tab_separated_list(xlsx_dict, "project_contributors", "project.contributors.email")}
Person Affiliation\t{utils.get_tab_separated_list(xlsx_dict, "project_contributors", "project.contributors.institution")}
Person Address\t{utils.get_tab_separated_list(xlsx_dict, "project_contributors", "project.contributors.address")}
Person Roles\t{tab.join(person_roles)}
Protocol Type\t{tab.join([field[0] for field in protocol_fields])}
Protocol Name\t{tab.join([field[1] for field in protocol_fields])}
Protocol Description\t{tab.join([field[2] for field in protocol_fields])}
Protocol Hardware\t{tab.join([field[3] for field in protocol_fields])}
Term Source Name\tEFO\tArrayExpress
Term Source File\thttp://www.ebi.ac.uk/efo/efo.owl\thttp://www.ebi.ac.uk/arrayexpress/
Comment[AEExperimentType]\tRNA-seq of coding RNA from single cells
Experimental Factor Name\t{tab.join(args.experimental_factors)}
Experimental Factor Type\t{tab.join(args.experimental_factors)}
Comment[EAAdditionalAttributes]\t{''}
Comment[EACurator]\t{tab.join(args.curators)}
Comment[EAExpectedClusters]\t
Comment[ExpressionAtlasAccession]\t{accession}
Comment[RelatedExperiment]\t{tab.join(related_scea_accessions)}
Comment[HCALastUpdateDate]\t{args.hca_update_date}
Comment[SecondaryAccession]\t{tab.join(secondary_accessions)}
Comment[EAExperimentType]\t{args.experiment_type}
SDRF File\t{sdrf_file_name}
Publication Title\t{utils.reformat_value(xlsx_dict, "project_publications", "project.publications.title", "str")[0].strip('.')}
Publication Author List\t{author_list}
PubMed ID\t{utils.reformat_value(xlsx_dict, "project_publications", "project.publications.pmid", "str")[0]}
Publication DOI\t{utils.reformat_value(xlsx_dict, "project_publications", "project.publications.doi", "str")[0]}
"""
    else:

        idf_file_contents = f"""\
MAGE-TAB Version\t1.1
Investigation Title\t{utils.reformat_value(xlsx_dict, "project", "project.project_core.project_title", "str")[0].strip('.')}
Comment[Submitted Name]\t{utils.reformat_value(xlsx_dict, "project", "project.project_core.project_short_name", "str")[0]}
Experiment Description\t{utils.reformat_value(xlsx_dict, "project", "project.project_core.project_description", "str")[0]}
Public Release Date\t{args.public_release_date}
Person First Name\t{utils.get_tab_separated_list(xlsx_dict, "project_contributors", "project.contributors.name", lambda x: x.split(',')[0])}
Person Last Name\t{utils.get_tab_separated_list(xlsx_dict, "project_contributors", "project.contributors.name", lambda x: x.split(',')[2])}
Person Mid Initials\t{utils.get_tab_separated_list(xlsx_dict, "project_contributors", "project.contributors.name", lambda x: utils.get_first_letter(x.split(',')[1]))}
Person Email\t{utils.get_tab_separated_list(xlsx_dict, "project_contributors", "project.contributors.email")}
Person Affiliation\t{utils.get_tab_separated_list(xlsx_dict, "project_contributors", "project.contributors.institution")}
Person Address\t{utils.get_tab_separated_list(xlsx_dict, "project_contributors", "project.contributors.address")}
Person Roles\t{tab.join(person_roles)}
Protocol Type\t{tab.join([field[0] for field in protocol_fields])}
Protocol Name\t{tab.join([field[1] for field in protocol_fields])}
Protocol Description\t{tab.join([field[2] for field in protocol_fields])}
Protocol Hardware\t{tab.join([field[3] for field in protocol_fields])}
Term Source Name\tEFO\tArrayExpress
Term Source File\thttp://www.ebi.ac.uk/efo/efo.owl\thttp://www.ebi.ac.uk/arrayexpress/
Comment[AEExperimentType]\tRNA-seq of coding RNA from single cells
Experimental Factor Name\t{tab.join(args.experimental_factors)}
Experimental Factor Type\t{tab.join(args.experimental_factors)}
Comment[EAAdditionalAttributes]
Comment[EACurator]\t{tab.join(args.curators)}
Comment[EAExpectedClusters]\t
Comment[ExpressionAtlasAccession]\t{accession}
Comment[HCALastUpdateDate]\t{args.hca_update_date}
Comment[SecondaryAccession]\t{tab.join(secondary_accessions)}
Comment[EAExperimentType]\t{args.experiment_type}
SDRF File\t{sdrf_file_name}
Publication Title\t{utils.reformat_value(xlsx_dict, "project_publications", "project.publications.title", "str")[0].strip('.')}
Publication Author List\t{author_list}
PubMed ID\t{utils.reformat_value(xlsx_dict, "project_publications", "project.publications.pmid", "str")[0]}
Publication DOI\t{utils.reformat_value(xlsx_dict, "project_publications", "project.publications.doi", "str")[0]}
"""

    print(f"saving {work_dir}/{idf_file_name}")
    with open(f"{work_dir}/{idf_file_name}", "w") as idf_file:
        idf_file.write(idf_file_contents)

def reformat_age(age_list):

    updated_age_list = []

    for age in age_list:
        age = str(age)
        if ' - ' in age:
            age = age.replace('-', 'to')
        elif '-' in age and ' ' not in age:
            age = age.replace('-', ' to ')
        else:
            age = age

        updated_age_list.append(age)

    return updated_age_list

def get_sample_name_key(args, df):

    if args.name == 'cs_name':
        sample_name_key = 'cell_suspension.biomaterial_core.biomaterial_name'
    elif args.name == 'cs_id':
        sample_name_key = 'cell_suspension.biomaterial_core.biomaterial_id'
    elif args.name == 'sp_name':
        sample_name_key = 'specimen_from_organism.biomaterial_core.biomaterial_name'
    elif args.name == 'sp_id':
        sample_name_key = 'specimen_from_organism.biomaterial_core.biomaterial_id'

    return sample_name_key

def get_values_from_df(df, column):
    return df[column] if column in df.columns else ['']*df.shape[0]

def add_sequence_paths(sdrf, args):

    run_accessions = list(sdrf['Comment[ENA_RUN]'])

    try:
        sra_paths = fetch_fastq_path.get_sra_path_from_ena(args.study, run_accessions)
    except:
        sra_paths = fetch_fastq_path.get_sra_path_from_sra(args.study, run_accessions)
        if not sra_paths:
            try:
                sra_paths = fetch_fastq_path.get_sra_path_from_sra(run_accessions)
            except:
                sra_paths = fetch_fastq_path.get_sra_path_from_ena(args.study, run_accessions)

    if sra_paths:
        read1_names = []
        read2_names = []
        sra_names = []
        for key in sra_paths.keys():
            read1_names.append(key + "_1.fastq.gz")
            read2_names.append(key + "_2.fastq.gz")
            sra_names.append(sra_paths[key]['files'][0])
        sdrf['Comment[read1 file]'] = read1_names
        sdrf['Comment[read2 file]'] = read2_names
        sdrf['Comment[SRA_URI]'] = sra_names
    else:
        print("Could not find paths to SRA objects.")
        sdrf['Comment[SRA_URI]'] = 'PATH NOT FOUND'
        sdrf['Comment[read1 file]'] = 'PATH NOT FOUND'
        sdrf['Comment[read2 file]'] = 'PATH NOT FOUND'

    return sdrf

def get_new_protocol_column_names(sdrf, counter):

    new_column_names = []
    for col in sdrf.columns:
        column_name = "Protocol REF" + "_" + str(counter)
        counter += 1
        new_column_names.append(column_name)

    return new_column_names, counter

def order_protocols(protocols_sdrf_before_sequencing):
    collection_protocol_cols = []
    dissociation_protocol_cols = []
    enrichment_protocol_cols = []
    differentiation_protocol_cols = []
    library_preparation_protocol_cols = []
    for col in list(protocols_sdrf_before_sequencing.columns):
        if col.split(".protocol_core.protocol_id")[0] == "collection_protocol":
            collection_protocol_cols.append(col)
        if col.split(".protocol_core.protocol_id")[0] == "dissociation_protocol":
            dissociation_protocol_cols.append(col)
        if col.split(".protocol_core.protocol_id")[0] == "enrichment_protocol":
            enrichment_protocol_cols.append(col)
        if col.split(".protocol_core.protocol_id")[0] == "differentiation_protocol":
            differentiation_protocol_cols.append(col)
        if col.split(".protocol_core.protocol_id")[0] == "library_preparation_protocol":
            library_preparation_protocol_cols.append(col)
    new_columns = [sorted(collection_protocol_cols),sorted(dissociation_protocol_cols),sorted(enrichment_protocol_cols),sorted(differentiation_protocol_cols),sorted(library_preparation_protocol_cols)]
    new_columns = [item for sublist in new_columns for item in sublist]
    protocols_sdrf_before_sequencing = protocols_sdrf_before_sequencing[new_columns]
    return protocols_sdrf_before_sequencing

def add_protocol_columns(df, dataset_protocol_map):

    def convert_term(term, name):
        return get_protocol_map.map_proto_to_id(term, dataset_protocol_map)

    def convert_row(row):
        return row.apply(lambda x: convert_term(x, row.name))

    protocols_list_before_sequencing = ['collection_protocol', 'dissociation_protocol', 'enrichment_protocol', 'differentiation_protocol', 'library_preparation_protocol']

    protocols_sdrf_before_sequencing = df[[col for (proto_type, cols) in get_protocol_map.map_of_hca_protocol_type_id_keys.items() if proto_type in
                 protocols_list_before_sequencing for col in cols]]

    protocols_sdrf_before_sequencing.fillna("", inplace=True)
    pd.set_option('display.max_columns', None)

    protocols_sdrf_before_sequencing = protocols_sdrf_before_sequencing.apply(convert_row)
    pd.set_option('display.max_columns', None)

    protocols_sdrf_before_sequencing = order_protocols(protocols_sdrf_before_sequencing)

    counter = 1
    new_column_names, counter = get_new_protocol_column_names(protocols_sdrf_before_sequencing, counter)
    protocols_sdrf_before_sequencing.columns = new_column_names

    protocols_sdrf_before_sequencing.fillna(value='', inplace=True)

    protocols_list_from_sequencing = ['sequencing_protocol']
    protocols_sdrf_from_sequencing = df[[col for (proto_type, cols) in get_protocol_map.map_of_hca_protocol_type_id_keys.items() if proto_type in protocols_list_from_sequencing for col in cols]]
    protocols_sdrf_from_sequencing = protocols_sdrf_from_sequencing.apply(convert_row)
    new_column_names, counter = get_new_protocol_column_names(protocols_sdrf_from_sequencing, counter)
    protocols_sdrf_from_sequencing.columns = new_column_names

    return protocols_sdrf_before_sequencing, protocols_sdrf_from_sequencing

def add_scea_specimen_columns(args, df, experimental_design):

    if experimental_design == "standard":

        '''Open dictionary of mapped hca2scea key:pairs for specimen metadata.'''
        with open(f"json_files/sdrf_map.json") as sdrf_map_file:
            sdrf_map = json.load(sdrf_map_file)

    else:

        if experimental_design == "cell_line_only":
            '''Open dictionary of mapped hca2scea key:pairs for specimen metadata.'''
            with open(f"json_files/sdrf_map_cell_line.json") as sdrf_map_file:
                sdrf_map = json.load(sdrf_map_file)

        elif experimental_design == "organoid":
            '''Open dictionary of mapped hca2scea key:pairs for specimen metadata.'''
            with open(f"json_files/sdrf_map_cell_line_organoid.json") as sdrf_map_file:
                sdrf_map = json.load(sdrf_map_file)

        else:
            '''Open dictionary of mapped hca2scea key:pairs for specimen metadata.'''
            with open(f"json_files/sdrf_map_organoid.json") as sdrf_map_file:
                sdrf_map = json.load(sdrf_map_file)

    '''Get user-specified HCA sample names key.'''
    sample_name_key = get_sample_name_key(args, df)

    '''Update dictionary of mapped hca2scea key:pairs with sample name key.'''
    sdrf_map.update({
        'Source Name': sample_name_key,
        'Assay Name': sample_name_key,
        'Scan Name': sample_name_key,
        'Extract Name': sample_name_key,
        'Comment[BioSD_SAMPLE]': 'cell_suspension.biomaterial_core.biosamples_accession',
        'Comment[ENA_EXPERIMENT]': 'cell_suspension.insdc_experiment.insdc_experiment_accession',
        'Comment[ENA_RUN]': 'sequence_file.insdc_run_accessions',
        'Comment[technical replicate group]': 'cell_suspension.biomaterial_core.biosamples_accession'
        })

    '''Extract the HCA metadata values using the HCA keys in sdrf_map.'''
    sdrf = pd.DataFrame({k: get_values_from_df(df, v) for k, v in sdrf_map.items()})
    sdrf = sdrf.fillna('')

    '''Reformat the age values so they are excel-friendly.'''
    sdrf['Characteristics[age]'] = reformat_age(list(sdrf['Characteristics[age]']))

    '''Reformat living status to align with SCEA controlled vocabulary.'''
    sdrf['Characteristics[organism status]'] = sdrf['Characteristics[organism status]'].apply(lambda x: 'alive' if x.lower() in ['yes', 'y'] else 'dead')

    '''Add Material Type using SCEA controlled vocabulary.'''
    sdrf['Material Type_1'] = 'cell'

    return sdrf

def generate_sdrf_file(work_dir, args, df, xlsx_dict, dataset_protocol_map, sdrf_file_name, experimental_design, technology_dict):

    '''Generate a dataframe with SCEA specimen metadata.'''
    sdrf_1 = add_scea_specimen_columns(args, df, experimental_design)

    '''Get technology-specific SCEA metadata and add to sdrf_1 dataframe.'''
    technology_type = list(xlsx_dict["library_preparation_protocol"]["library_preparation_protocol.library_construction_method.ontology_label"])[0]
    technology_type = rename_technology_type(technology_type,technology_dict)
    try:
        with open(f"json_files/{technology_type}.json") as technology_json_file:
            technology_type_dict = json.load(technology_json_file)
    except:
        print("Technology type {} is not yet supported. Please ask Ami to add it to the technology type map.".format(technology_type))
        sys.exit()
    for key in technology_type_dict.keys():
        sdrf_1[key] = technology_type_dict[key]

    '''Edit single cell isolation method if the user-specified that FACS was used.'''
    if args.facs is True:
        sdrf_1['Comment[single_cell_isolation]'] = 'FACS'*sdrf_1.shape[0]
    # To do: add 'Comment[single_cell_isolation]' == Fluidigm C1 for Fluidigm C1 experiments.

    '''To Do. Get immunophenotype and treatment/stimulus information, if there is any.'''
    #"Characteristics[immunophenotype]":"enrichment_protocol.markers"
    sdrf_1['Characteristics[immunophenotype]'] = '' * sdrf_1.shape[0]
    #"Characteristics[stimulus]":"cell_suspension.growth_conditions.drug_treatment","differentiation_protocol.small_molecules",
    sdrf_1['Characteristics[stimulus]'] = '' * sdrf_1.shape[0]

    '''Get the SRA object file names and file paths from SRA or ENA (try both).'''
    sdrf_2 = add_sequence_paths(sdrf_1, args)

    '''Check all required column names are present and reorder columns by SCEA defined order.'''

    with open(f"json_files/expected_columns.json", "r") as expected_columns_file:
        expected_columns_dict = json.load(expected_columns_file)

    if experimental_design == 'standard':
        expected_columns_ordered = expected_columns_dict['standard']
    elif experimental_design == 'organoid_only':
        expected_columns_ordered = expected_columns_dict['organoid_only']
    else:
        expected_columns_ordered = expected_columns_dict['cell_line']

    column_check = [col for col in expected_columns_ordered if col not in sdrf_2.columns]
    if column_check:
        print("Error: one or more expected columns is missing from sdrf.")
        print(column_check)
        sys.exit()
    else:
        sdrf_3 = sdrf_2[expected_columns_ordered]

    '''Add protocol columns with protocol metadata in the pre-defined SCEA column order.'''
    protocols_sdrf_before_sequencing, protocols_sdrf_from_sequencing = add_protocol_columns(df, dataset_protocol_map)
    # Remove empty Protocol REF columns.
    nan_value = float("NaN")
    protocols_sdrf_before_sequencing.replace("", nan_value, inplace=True)
    protocols_sdrf_before_sequencing.dropna(how='all', axis=1, inplace=True)
    protocols_sdrf_before_sequencing.replace(nan_value, "", inplace=True)

    idx = 18
    for col in protocols_sdrf_before_sequencing.columns:
        sdrf_3.insert(idx, col, list(protocols_sdrf_before_sequencing[col]))
        idx += 1

    idx = list(sdrf_3.columns).index('Comment[LIBRARY_SELECTION]') + 1
    for col in protocols_sdrf_from_sequencing.columns:
        sdrf_3.insert(idx, col, list(protocols_sdrf_from_sequencing[col]))
        idx += 1

    '''Fix Material Type and Protocol REF column names to SCEA standard columns.'''
    for col_name in sdrf_3.columns:
        if 'Protocol REF' in str(col_name):
            sdrf_3 = sdrf_3.rename(columns={col_name: "Protocol REF"})
        elif 'Material Type' in str(col_name):
            sdrf_3 = sdrf_3.rename(columns={col_name: "Material Type"})

    '''Remove empty columns if columns are optional.'''
    with open(f"json_files/optional_columns.json", "r") as optional_columns_file:
        optional_columns_dict = json.load(optional_columns_file)

    if experimental_design == 'standard':
        optional_columns = optional_columns_dict['standard']
    else:
        optional_columns = optional_columns_dict['cell_line']

    for column_name in optional_columns:
        if set(list(sdrf_3[column_name])) == {''}:
            sdrf_3 = sdrf_3.drop([column_name], axis=1)

    '''Write the new sdrf file to a file.'''
    if not sdrf_3.empty:
        print(f"saving {work_dir}/{sdrf_file_name}")
        sdrf_3.to_csv(f"{work_dir}/{sdrf_file_name}", sep="\t", index=False)

def create_magetab(work_dir, xlsx_dict, dataset_protocol_map, df, args, experimental_design, accession_number, technology_dict):

    accession = f"E-HCAD-{accession_number}"

    idf_file_name = f"{accession}.idf.txt"
    sdrf_file_name = f"{accession}.sdrf.txt"

    generate_idf_file(work_dir, args, dataset_protocol_map, xlsx_dict, accession, idf_file_name,
                      sdrf_file_name)
    generate_sdrf_file(work_dir, args, df, xlsx_dict, dataset_protocol_map, sdrf_file_name, experimental_design, technology_dict)

def main():
    parser = argparse.ArgumentParser(description="run hca -> scea tool")
    parser.add_argument(
        "-s",
        "--spreadsheet",
        type=str,
        required=True,
        help="Please provide a path to the HCA project spreadsheet."
    )
    parser.add_argument(
        "-id",
        "--project_uuid",
        type=str,
        required=True,
        help="Please provide an HCA ingest project submission id."
    )
    parser.add_argument(
        "-study",
        type=str,
        required=True,
        help="Please provide the SRA or ENA study accession."
    )
    parser.add_argument(
        "-name",
        type=str,
        required=False,
        default = 'cs_id',
        choices = ['cs_name','cs_id','sp_name','sp_id','other'],
        help="Please indicate which field to use as the sample name. cs=cell suspension, sp = specimen."
    )
    parser.add_argument(
        "-ac",
        "--accession_number",
        type=int,
        required=True,
        help="Provide an E-HCAD accession number. Please find the next suitable accession number by checking the google tracker sheet."
    )
    parser.add_argument(
        "-c",
        "--curators",
        nargs='+',
        required=True,
        help="space separated names of curators"
    )
    parser.add_argument(
        "-et",
        "--experiment_type",
        type=str,
        required=True,
        choices=['baseline','differential'],
        help="Please indicate whether this is a baseline or differential experimental design"
    )
    parser.add_argument(
        "--facs",
        action="store_true",
        default=None,
        help="Please specify this argument if FACS was used to isolate single cells"
    )
    parser.add_argument(
        "-f",
        "--experimental_factors",
        nargs='+',
        required=True,
        help="space separated list of experimental factors"
    )
    parser.add_argument(
        "-pd",
        "--public_release_date",
        type=str,
        required=True,
        help="Please enter the public release date in this format: YYYY-MM-DD"
    )
    parser.add_argument(
        "-hd",
        "--hca_update_date",
        type=str,
        required=True,
        help="Please enter the last time the HCA prohect submission was updated in this format: YYYY-MM-DD"
    )
    parser.add_argument(
        "-r",
        "--related_scea_accession",
        nargs='+',
        required=False,
        help="space separated list of related scea accession(s)"
    )
    parser.add_argument(
        "-o",
        "--output_dir",
        required=False,
        help="Provide full path to preferred output dir"
    )

    args = parser.parse_args()
    if not args.output_dir:
        work_dir = f"script_spreadsheets/{os.path.splitext(os.path.basename(args.spreadsheet))[0]}"
    else:
        work_dir = args.output_dir

    xlsx_dict = multitab_excel_to_single_txt.multitab_excel_to_dict(work_dir, args.spreadsheet)

    xlsx_dict = multitab_excel_to_single_txt.remove_unused_protocols(xlsx_dict)

    xlsx_dict = multitab_excel_to_single_txt.rename_protocol_columns(xlsx_dict)

    technology_dict = {
        "Fluidigm C1-based library preparation": "smart-like",
        "10X 3' v1": "10Xv1_3",
        "10X 5' v1": "10Xv1_5",
        "10X 3' v2": "10Xv2_3",
        "10X 5' v2": "10Xv2_5",
        "10X 3' v3": "10Xv3_3",
        "10X 3' v1 sequencing": "10Xv1_3",
        "10X 5' v1 sequencing": "10Xv1_5",
        "10X 3' v2 sequencing": "10Xv2_3",
        "10X 5' v2 sequencing": "10Xv2_5",
        "10X 3' v3 sequencing": "10Xv3_3",
        "10x 3' v1": "10Xv1_3",
        "10x 5' v1": "10Xv1_5",
        "10x 3' v2": "10Xv2_3",
        "10x 5' v2": "10Xv2_5",
        "10x 3' v3": "10Xv3_3",
        "10x 3' v1 sequencing": "10Xv1_3",
        "10x 5' v1 sequencing": "10Xv1_5",
        "10x 3' v2 sequencing": "10Xv2_3",
        "10x 5' v2 sequencing": "10Xv2_5",
        "10x 3' v3 sequencing": "10Xv3_3",
        "Drop-seq": "drop-seq",
        "inDrop": "drop-seq",
        "Smart-like": "smart-like",
        "Smart-seq2": "smart-seq",
        "Smart-seq": "smart-seq"
    }

    check_experimental_design.check_biomaterial_linkings(xlsx_dict)
    check_experimental_design.check_protocol_linkings(xlsx_dict)
    experimental_design = check_experimental_design.get_experimental_design(xlsx_dict)

    if experimental_design == "cell_line_only" or experimental_design == "organoid":
        check_experimental_design.check_cell_lines_linked(xlsx_dict)
    if experimental_design == "organoid_only" or experimental_design == "organoid":
        check_experimental_design.check_organoids_linked(xlsx_dict)

    check_experimental_design.check_donor_exists(xlsx_dict)
    check_experimental_design.check_specimen_exists(xlsx_dict)
    check_experimental_design.check_input_to_cell_suspension(xlsx_dict)
    check_experimental_design.check_technology_eligibility(xlsx_dict, technology_dict)
    check_experimental_design.check_species_eligibility(xlsx_dict)
    check_experimental_design.check_for_pooled_samples(xlsx_dict)

    merged_df = multitab_excel_to_single_txt.merge_dataframes(xlsx_dict,experimental_design)
    merged_df_unique_runs = merged_df.drop_duplicates(subset=['sequence_file.insdc_run_accessions'])
    clean_merged_df = multitab_excel_to_single_txt.clean_df(merged_df_unique_runs)

    df = multitab_excel_to_single_txt.create_new_protocol_columns(clean_merged_df, xlsx_dict)

    dataset_protocol_map = get_protocol_map.prepare_protocol_map(xlsx_dict, df, args)

    create_magetab(work_dir, xlsx_dict, dataset_protocol_map, df, args, experimental_design, args.accession_number, technology_dict)

if __name__ == '__main__':
    main()


"""
可以同该脚本下载repo或文件，如果max_try设的很大仍不能成功，只能手动下载文件，然后用batch_create_soft_link.sh脚本建立软链接
"""
from huggingface_hub import hf_hub_download, snapshot_download
import os
import shutil
import subprocess
import argparse
from typing import Union
from datasets import load_dataset,DownloadConfig

def recursive_load_dataset(dataset_name,name=None,data_files=None,max_try=300):
    #for some regions and countries, `load_dataset` casually raise ConnectionError,
    # so recursively downloading the dataset would make the project more robust.
    try_turn = 0
    while True:
        try:
            config = DownloadConfig(resume_download=True, max_retries=300)
            data = load_dataset(dataset_name,name=name,data_files=data_files,download_config=config)
            return data
        except Exception as e:
            print(e)
            print("Download dataset failed, re-downloading...")
            try_turn += 1
            if try_turn > max_try:
                print("The number of retries exceeded `max_try`,maybe you are offline, please check the network.")
                raise ConnectionError
            else:
                continue

def recursive_download_file(repo_id,
                            filename=None,
                            repo_type="model",
                            local_dir_use_symlinks=False,
                            cache_dir=None,
                            local_dir=None,
                            rename_files=False,
                            max_try=300):
    """
    自动短点重下指定文件直至成功
    repo_id: 仓库地址，如“TheBloke/guanaco-65B-GGML”
    filename: 指定文件名
    repo_type: model, dataset, space
    cache_dir:指定缓存地址
    local_dir: 指定本地存在地址
    local_dir_use_symlinks: 比较复杂，简言之是是否建立`local_dir`和`cache_dir`的软链接
    rename_files: 是否将下载的文件名改为真实文件名，若True，则使用默认地址，若str,则该参数需指向真实文件所在的地址；
                下载的huggingface文件通常会被重命名为一个随机文件名，然后用软链接的方式链接真实文件名，
                此参数运行将随机文件名改为真实文件名，
                #! 有时能成功有时不能成功，可能跟shell脚本所处的环境有关
    max_try: 最大重复下载次数，超过此值认为处于离线环境，抛出网络异常
    """
    if local_dir and not os.path.exists(local_dir):
        os.makedirs(local_dir)
    if cache_dir and not os.path.exists(cache_dir):
        os.makedirs(cache_dir)
    try_turn = 0
    while True:
        try:

            if filename is not None:
                print("downloading",repo_id, filename)
                print("*"*80)
                file_dir = hf_hub_download(repo_id,
                                filename=filename,
                                repo_type=repo_type,
                                cache_dir=cache_dir,
                                local_dir=local_dir,
                                resume_download=True,
                                local_dir_use_symlinks=local_dir_use_symlinks
                                )

            else:
                print(repo_id)
                print("-"*80)
                file_dir = snapshot_download(repo_id,
                                repo_type=repo_type,
                                cache_dir=cache_dir,
                              #  local_dir=local_dir,
                                resume_download=True,
                              #  local_dir_use_symlinks=local_dir_use_symlinks
                                  )

            print("download file or repo succeed!")

            if rename_files:
                rename_files = "/".join(file_dir.split("/")[:-1]) if isinstance(rename_files,bool) else rename_files

                copy_shell_name = os.path.join(rename_files,"batch_rename_real_files.sh")
                # 将目录下的batch_rename_real_files.sh移至真实文件所在文件夹下
                shutil.copyfile("./batch_rename_real_files.sh",copy_shell_name)
                current_path = os.getcwd()
                os.chdir(rename_files)
                # 实测用python执行shell脚本，其中的readlink -f link或realpath link命名不能获取
                # 真实地址，而是只能得到软连接的地址
                # todo 查查该怎么操作以及为什么会这样
                subprocess.run(["/bin/bash",copy_shell_name])
                os.remove(copy_shell_name)
            return file_dir
        except Exception as e:
            print(e)
            print("Download model failed, re-downloading...")
            try_turn += 1
            if try_turn > max_try:
                print("The number of retries exceeded `max_try`,maybe you are offline, please check the network.")
                raise ConnectionError
            else:
                continue

if __name__ == "__main__":
    parser = argparse.ArgumentParser(prog="auto download from huggingface_hub",description="自动从huggingface hub下载文件")
    parser.add_argument("--function",default='recursive_download_file',type=str)
    parser.add_argument("--repo-id",type=str,default="bigscience/tokenizer")
    parser.add_argument("--filename",type=str,default=None)
    parser.add_argument("--cache-dir",type=str,default=None)
    parser.add_argument('--repo-type',type=str,default="model")
    parser.add_argument('--local-dir',type=str,default=None)
    parser.add_argument('--local-dir-use-symlinks',type=bool, default=False)

    parser.add_argument("--rename-files",type=Union[bool,str,None],default=False,
                        help="""whether to rename the downloaded file, if True, use huggingface_hub's default directory;
                        if str,input the directory of the soft links""")
    parser.add_argument("--max-try",type=int,default=300)

    parser.add_argument("--dataset-name",type=str,default="Dahoas/rm-static")
    parser.add_argument("--name",default=None,type=str)
    parser.add_argument("--data-files",default=None,type=str)

    load_dataset_args = ["dataset_name","name","data_files","max_try"]

    args = parser.parse_args()
    args_dict = vars(args)
    function_call_str = args_dict.pop("function")
    print(function_call_str)
    function_call = eval(function_call_str)
    if function_call_str == "recursive_load_dataset":
        args_dict_use = {key:value for key,value in args_dict.items() if key in load_dataset_args}
        print(args_dict_use)
    else:
        args_dict_use = {key:value for key, value in args_dict.items() if key not in load_dataset_args}
    function_call(**args_dict_use)
    # recursive_download_file(**args_dict)


import time
# from kflod import kfold
import numpy as np
import random
from torch.optim.adam import Adam
from torch import optim
from resnet_attn_new import *
from torchvision.models.resnet import resnet50, resnet18
from torchvision.models.densenet import densenet121
from preprocessing import get_dataset3d
import sys

def expAllAtn(data_path):
    reset_rand()
    def model_opt():
        model = AllAtn()
        optm = Adam(model.parameters())
        return model, optm

    all_targets,  all_pred=kfold(data_path,
          256,
          50,
          model_optimizer=model_opt,
          loss=nn.BCELoss(),
          name='BasicResnet',
          device='cuda:0',
          deterministic=True
          )
    return all_targets,  all_pred

def expBasicResnet(data_path):
    reset_rand()

    def model_opt():
        model = BasicResnet()
        optm = Adam(model.parameters())
        return model, optm

    all_targets,  all_pred=kfold(data_path,
          256,
          50,
          model_optimizer=model_opt,
          loss=nn.BCELoss(),
          name='BasicResnet',
          device='cuda:0',
          deterministic=True
          )
    return all_targets,  all_pred

from torchsummary import summary

def expLocalGlobal(data_path):
    reset_rand()

    def model_opt():
        model = LocalGlobalNetwork()
        # summary(model, (1, 32, 32))
        optm=optim.RMSprop(model.parameters())
        # optm = Adam(model.parameters())
        return model, optm

    all_targets,  all_pred=kfold(data_path,
          256,
          50,
          model_optimizer=model_opt,
          loss=nn.BCELoss(),
          name='LocalGlobalNetwork',
          device='cuda:0',
          deterministic=True
          )
    return all_targets,  all_pred

def expLocalGlobal_old(data_path):
    reset_rand()

    def model_opt():
        model = LocalGlobalNetwork_old()
        # summary(model, (1, 32, 32))
        optm = Adam(model.parameters())
        return model, optm

    all_targets,  all_pred=kfold(data_path,
          256,
          50,
          model_optimizer=model_opt,
          loss=nn.BCELoss(),
          name='LocalGlobalNetwork_old',
          device='cuda:0',
          deterministic=True
          )
    return all_targets,  all_pred

def expResnetTrans(data_path):
    reset_rand()

    def model_opt():
        model = resnet50(pretrained=True)
        model.avgpool = nn.AdaptiveAvgPool2d((1, 1))
        model.fc = nn.Sequential(
            nn.Linear(model.fc.in_features, 1),
            nn.Sigmoid()
        )

        optm = Adam(model.fc.parameters())
        return model, optm

    all_targets,  all_pred=kfold(data_path,
          256,
          50,
          model_optimizer=model_opt,
          loss=nn.BCELoss(),
          name='ResnetTrans',
          device='cuda:0',
          deterministic=True,
          dataset_func=get_dataset3d
          )
    return all_targets,  all_pred

def expDensenetTrans(data_path):
    reset_rand()

    def model_opt():
        model = densenet121(pretrained=True)
        model.classifier = nn.Sequential(
            nn.Linear(model.classifier.in_features, 1),
            nn.Sigmoid()
        )

        optm = Adam(model.classifier.parameters())
        return model, optm

    all_targets,  all_pred=kfold(data_path,
          256,
          50,
          model_optimizer=model_opt,
          loss=nn.BCELoss(),
          name='DensenetTrans',
          device='cuda:0',
          deterministic=True,
          #dataset_func=get_dataset3d
          )
    return all_targets,  all_pred

def expResnet18Trans(data_path):
    reset_rand()

    def model_opt():
        model = resnet18(pretrained=True)

        model.avgpool = nn.AdaptiveAvgPool2d((1, 1))
        model.fc = nn.Sequential(
            nn.Linear(model.fc.in_features, 1),
            nn.Sigmoid()
        )

        optm = Adam(model.fc.parameters())
        return model, optm

    all_targets,  all_pred=kfold(data_path,
          256,
          50,
          model_optimizer=model_opt,
          loss=nn.BCELoss(),
          name='Resnet18Trans',
          device='cuda:0',
          deterministic=True,
          dataset_func=get_dataset3d
          )
    return all_targets,  all_pred

def print_error():
    print('python <model_name> <data_path>')
    print('here is a list of experiments names:')
    for name in experiments.keys():
        print(name)

from trainer import Trainer
from preprocessing import get_dataset
from os import path
from torch.utils.data import DataLoader
import torch as T
import numpy as np
import random
from sklearn import metrics
from os import path
import matplotlib.pyplot as plt

def get_metrics(target, pred):
    prec, recall, f1, _ = metrics.precision_recall_fscore_support(target, pred>0.5, average='binary')
    accu=metrics.accuracy_score(target, pred>0.5)
    fpr, tpr, thresholds = metrics.roc_curve(target, pred)
    auc = metrics.auc(fpr, tpr)
    return prec, recall, auc, f1, accu

def calc_accuracy(x, y):
    x_th = (x > 0.5).long()
    matches = x_th == y.long()
    return matches

def reset_rand():
    seed = 1000
    T.manual_seed(seed)
    T.cuda.manual_seed(seed)
    np.random.seed(seed)
    random.seed(seed)

def kfold(src_path,
          batch_size,
          n_epochs,
          model_optimizer,
          loss,
          name,
          device,
          deterministic=False,
          parallel=False,
          dataset_func=get_dataset):

    print('------Experiment:',name,'------')
    all_pred = T.zeros(849)
    all_targets = T.zeros(849)
    i =0
    f = open(path.join('results', '{name}.txt'), 'w')
    f.write('{batch_size} {n_epochs} {model_optimizer}\n')
    for fold in range(10):

        reset_rand()

        print('------------ fold' + str(fold+1) + ' ------------')
        f.write('------------ fold {fold+1} ------------\n')
        trset, testset = dataset_func(path.join(src_path,str(fold)))
        print('Training Size: ' + str(len(trset)) + ' Validation Size: '+ str(len(testset)))
        trset = DataLoader(trset, batch_size, shuffle=True)
        testset = DataLoader(testset, batch_size, shuffle=False)
        model,optimizer = model_optimizer()
        tr = Trainer(
            trset,
            testset,
            batch_size,
            n_epochs,
            model,
            optimizer,
            loss,
            '{name}_{fold}',
            device,
            deterministic,
            parallel
        )

        tr.run()

        pred, target = tr.predict()
        all_pred[i:i+pred.shape[0]] = pred
        all_targets[i:i+target.shape[0]] = target
        i += target.shape[0]

        prec, recall, auc, f1, accu = get_metrics(target, pred)
        print('AUC: ' + str(auc) + ', '  + 'Accuracy: ' + str(accu) + ', ' +  'precession: ' + str(prec) + ', ' + 'Recall: ' +  str(recall) + ', ' + 'F1score: ' +str(f1))
        f.write('AUC: ' + str(auc) + ', '+ 'Accuracy: ' + str(accu) + ', '  +  'precession: ' + str(prec) + ', ' + 'Recall: ' +  str(recall) + ', ' + 'F1score: ' +str(f1) +'\n')

    matches = calc_accuracy(all_pred, all_targets)
    acc = matches.float().mean()
    all_pred = all_pred.numpy()
    all_targets = all_targets.numpy()

    prec, recall, auc, f1, accu = get_metrics(all_targets, all_pred)
    print('Accuracy: ' + str(accu) + ', ' + 'AUC: ' + str(auc) + ', '  +  'precession: ' + str(prec) + ',' + 'Recall: ' +  str(recall)+ ', ' + 'f1score: ' + str(f1)+ '\n')
    f.write('accuray: {acc}, accuray1: {accu}, AUC: {auc}, precession: {prec}, Recall: {recall}, f1score: {f1}')
    result = {'all_pred': all_pred, 'all_targets': all_targets}
    T.save(result, path.join('results','{name}_result'))
    f.close()
    return all_targets,  all_pred

if __name__ == '__main__':
    experiments = {
        'Resnet18Trans' : expResnet18Trans,
        'ResnetTrans' : expResnetTrans,
        'LocalGlobal_cbam' : expLocalGlobal,
        'LocalGlobal':expLocalGlobal_old,
        'BasicResnet' : expBasicResnet,
        'AllAtn' : expAllAtn,
        'DensenetTrans' : expDensenetTrans
    }

    #################################
    exp_name = 'Resnet18Trans'
    data_path = '/home/nisa.hameed/Downloads/Lung_class/lidc_img'

    st = time.time()
    all_targets,  all_pred=experiments[exp_name](data_path)
    et = time.time()
    elapsed_time = et - st
    print(' Total execution time for ',exp_name,'is', elapsed_time, 'seconds')

    #plot visualization
    fpr, tpr, _ = metrics.roc_curve(all_targets,  all_pred)
    auc = metrics.roc_auc_score(all_targets, all_pred)
    p1=plt.plot(fpr,tpr,label="AUC="+str(auc))
    # plt.ylabel('True Positive Rate')
    # plt.xlabel('False Positive Rate')
    # # plt.legend(loc=4)
    # plt.show()

# ########################################
    exp_name = 'ResnetTrans'
    data_path = '/home/nisa.hameed/Downloads/Lung_class/lidc_img'

    st = time.time()
    all_targets,  all_pred=experiments[exp_name](data_path)
    et = time.time()
    elapsed_time = et - st
    print(' Total execution time for ',exp_name,'is', elapsed_time, 'seconds')

    fpr, tpr, _ = metrics.roc_curve(all_targets,  all_pred)
    auc = metrics.roc_auc_score(all_targets, all_pred)
    p2=plt.plot(fpr,tpr,label="AUC="+str(auc))
    # plt.ylabel('True Positive Rate')
    # plt.xlabel('False Positive Rate')
#     # plt.legend(loc=4)
#     # plt.show()

#     ###########################################
    exp_name = 'AllAtn'
    data_path = '/home/nisa.hameed/Downloads/Lung_class/lidc_img'

    st = time.time()
    all_targets,  all_pred=experiments[exp_name](data_path)
    et = time.time()
    elapsed_time = et - st
    print(' Total execution time for ',exp_name,'is', elapsed_time, 'seconds')

    #added for plot
    fpr, tpr, _ = metrics.roc_curve(all_targets,  all_pred)
    auc = metrics.roc_auc_score(all_targets, all_pred)
    p3=plt.plot(fpr,tpr,label="AUC="+str(auc))
#     # plt.ylabel('True Positive Rate')
#     # plt.xlabel('False Positive Rate')
#     # plt.legend(loc=4)
#     # # plt.show()

    ###########################################
    exp_name = 'LocalGlobal'
    data_path = '/home/nisa.hameed/Downloads/Lung_class/lidc_img'

    st = time.time()
    all_targets,  all_pred=experiments[exp_name](data_path)
    et = time.time()
    elapsed_time = et - st
    print(' Total execution time for ',exp_name,'is', elapsed_time, 'seconds')

    # #added for plot
    fpr, tpr, _ = metrics.roc_curve(all_targets,  all_pred)
    auc = metrics.roc_auc_score(all_targets, all_pred)
    p4=plt.plot(fpr,tpr,label="AUC="+str(auc))

  ###################################
    exp_name = 'LocalGlobal_cbam'
    data_path = '/home/nisa.hameed/Downloads/Lung_class/lidc_img'

    st = time.time()
    all_targets,  all_pred=experiments[exp_name](data_path)
    et = time.time()
    elapsed_time = et - st
    print(' Total execution time for ',exp_name,'is', elapsed_time, 'seconds')

    fpr, tpr, _ = metrics.roc_curve(all_targets,  all_pred)
    auc = metrics.roc_auc_score(all_targets, all_pred)
    p5=plt.plot(fpr,tpr,label="AUC="+str(auc))
    #visualization
    plt.ylabel('True Positive Rate')
    plt.xlabel('False Positive Rate')
    # plt.legend((p1,p2,p3,p4,p5),('RESNET18','RESNET50','DENSENET121','LOCAL_GLOBAL','ATTENTION_LOCAL_GLOBAL'),loc='lower left')
    plt.legend((p4,p5),('LOCAL_GLOBAL','ATTENTION_LOCAL_GLOBAL'),loc=4)
    plt.show()


pasta = []
juice = []

for i in range(3):
    pasta.append(int(input()))

for i in range(2):
    juice.append(int(input()))

pasta.sort()  # sort by Ascending
juice.sort()  # sort by Ascending

sum = pasta[0] + juice[0]  # lowest price of pasta and juice

sum += sum / 10  # 10percent of sum

print('%.1f'%sum)  # .1f to show


from  rest_framework import serializers,status
from core.utils.exception import CustomValidation
from core.utils.helper_function import get_amount_by_percent
from ..models import monthly_salary_structure as models
from django.db import transaction
from career_path import models as career_path_models

class ListOfEmployeeReceivablesSerializer(serializers.Serializer):
    fixed_receivables_element= serializers.CharField()
    fixed_receivables_element_gross_percent= serializers.IntegerField()

class ListOfEmployeeOtherReceivablesSerializer(serializers.Serializer):
    other_receivables_element = serializers.CharField()
    other_receivables_element_gross_percent = serializers.IntegerField()
class ListOfEmployeeRegulatoryRecievablesSerializer(serializers.Serializer):
    regulatory_receivables= serializers.CharField()
    regulatory_receivables_gross_percent= serializers.IntegerField()
    regulatory_rates = serializers.IntegerField()

class ListOfEmployeeRegulatoryDeductablesSerializer(serializers.Serializer):
    regulatory_deductables= serializers.CharField()
    regulatory_deductables_gross_percent =serializers.IntegerField()
    regulatory_rates = serializers.IntegerField()

class ListOfEmployeeOtherDeductablesSerializer(serializers.Serializer):
    other_deductables= serializers.CharField()
    other_deductables_gross_percent =serializers.IntegerField()

class MonthSalaryStructureCreationSerializer(serializers.Serializer):
    "struture Style"
    structure_type = serializers.ChoiceField(choices=['monthly','daily','hourly'])
    rate =serializers.DecimalField(required=False,max_digits=20, decimal_places=2)
    number_of_work = serializers.IntegerField(required=False)

    grade_level = serializers.CharField()
    gross_money = serializers.DecimalField(max_digits=20, decimal_places=2)
    "Receivables"
    employee_receivables = ListOfEmployeeReceivablesSerializer(many=True)
    employee_regulatory_recievables =ListOfEmployeeRegulatoryRecievablesSerializer(many=True)
    employee_other_receivables  = ListOfEmployeeOtherReceivablesSerializer(many=True)

    "Deductables"
    employee_regulatory_deductables = ListOfEmployeeRegulatoryDeductablesSerializer(many=True)
    employee_other_deductables = ListOfEmployeeOtherDeductablesSerializer(many=True)

    def validate(self, attrs):
        structure_type = attrs.get('structure_type',None)
        rate=attrs.get('rate',None)
        number_of_work=attrs.get('number_of_work',None)

        if structure_type =='daily' or structure_type =='hourly':
            if rate is None:raise CustomValidation(
            detail=f'Rate  is required for {structure_type}',field='error',status_code=status.HTTP_400_BAD_REQUEST
            )
            if number_of_work is None:raise CustomValidation(
            detail=f'Number of work required for {structure_type}',field='error',status_code=status.HTTP_400_BAD_REQUEST
            )

        if not  self._validate_gross_percent(attrs): raise CustomValidation(
            detail='Total gross is ment to be 100%',field='error',status_code=status.HTTP_400_BAD_REQUEST
        )
        if not  career_path_models.CareerPath.objects.filter(career_path_id=attrs.get('grade_level')).exists():
            raise CustomValidation(detail='this level does not exist',field='grade_level',status_code=status.HTTP_400_BAD_REQUEST)
        else:
            'check if this career path exist in monthly payroll'
            career_path  = career_path_models.CareerPath.objects.get(career_path_id=attrs.get('grade_level'))
            if models.MonthlySalaryStructure.objects.filter(grade_level=career_path,structure_type=structure_type).exists():
                raise CustomValidation(detail='This level already has a Monthly Salary Payroll Strcuture',field='grade_level',status_code=status.HTTP_400_BAD_REQUEST)

        return super().validate(attrs)

    def _validate_gross_percent(self,attrs):

        def get_other_receivable_gross_percent(each_item):
            return each_item.get('other_receivables_element_gross_percent',0)
        def getfixed_receivables_element_gross_percent(each_item):
            return each_item.get('fixed_receivables_element_gross_percent',0)
        def getregulatory_receivables_gross_percent(each_item):
            return each_item.get('regulatory_receivables_gross_percent',0)
        def getregulatory_deductables_gross_percent(each_item):
            return each_item.get('regulatory_deductables_gross_percent',0)
        def getother_deductables_gross_percent(each_item):
            return each_item.get('other_deductables_gross_percent',0)

        "this validate if all the gross Receivables percent sum up to 100"

        list_of_gross = [
            sum(map(getfixed_receivables_element_gross_percent,attrs.get('employee_receivables'))),
            sum(map(getregulatory_receivables_gross_percent,attrs.get('employee_regulatory_recievables'))),
            sum(map(get_other_receivable_gross_percent,attrs.get('employee_other_receivables')))
            # sum(map(getregulatory_deductables_gross_percent,attrs.get('employee_regulatory_deductables'))),
            # sum(map(getother_deductables_gross_percent,attrs.get('employee_other_deductables'))),
        ]
        return sum(list_of_gross) ==100

    def create(self, validated_data):
        structure_type = validated_data.get('structure_type',None)
        rate=validated_data.get('rate',None)
        number_of_work=validated_data.get('number_of_work',None)
        grade_level = validated_data.get('grade_level')
        gross_money = validated_data.get('gross_money')
        employee_receivables = validated_data.get('employee_receivables',[])
        employee_regulatory_recievables = validated_data.get('employee_regulatory_recievables',[])
        employee_regulatory_deductables = validated_data.get('employee_regulatory_deductables',[])
        employee_other_deductables = validated_data.get('employee_other_deductables',[])
        employee_other_receivables= validated_data.get('employee_other_receivables',[])
        carer_path = career_path_models.CareerPath.objects.get(career_path_id=grade_level)
        "first we need to create MonthlySalaryStructure"
        monthlysalarystructureinstance = models.MonthlySalaryStructure.objects.create(
            grade_level=carer_path,
            gross_money=gross_money,
        )
        if structure_type =='daily' or structure_type =='hourly':
            monthlysalarystructureinstance.structure_type=structure_type
            monthlysalarystructureinstance.rate=rate
            monthlysalarystructureinstance.number_of_work=number_of_work
            monthlysalarystructureinstance.gross_money = monthlysalarystructureinstance.cal_gross_money()
            monthlysalarystructureinstance.save()
        "bulk create receivables "
        self.bulkCreateEmployeeReceivables(monthlysalarystructureinstance,employee_receivables)
        self.bulkCreateEmployeeRegulatoryRecievables(monthlysalarystructureinstance,employee_regulatory_recievables)
        self.bulkCreateEmployeeOtherReceivable(monthlysalarystructureinstance,employee_other_receivables)
        "bulk create deductables"
        self.bulkCreateEmployeeRegulatoryDeductable(monthlysalarystructureinstance,employee_regulatory_deductables)
        self.bulkCreateEmployeeOtherDeductables(monthlysalarystructureinstance,employee_other_deductables)
        return monthlysalarystructureinstance

    def bulkCreateEmployeeOtherReceivable(self,monthlysalarystructureinstance,employee_other_receivables:list):
        with transaction.atomic():
            for data in employee_other_receivables:
                instance= models.EmployeeOtherReceivables(
                    monthly_salary_structure=monthlysalarystructureinstance,
                    other_receivables_element=data.get('other_receivables_element','null'),
                    other_receivables_element_gross_percent=data.get('other_receivables_element_gross_percent',0),
                    value=get_amount_by_percent(
                        data.get('other_receivables_element_gross_percent',0),monthlysalarystructureinstance.gross_money
                    )
                )
                instance.save()
    def bulkCreateEmployeeReceivables(self,monthlysalarystructureinstance,employee_receivables:list):
        with transaction.atomic():
            for data in employee_receivables:
                instance = models.EmployeeReceivables(
                    monthly_salary_structure=monthlysalarystructureinstance,
                    fixed_receivables_element=data.get('fixed_receivables_element','null'),
                    fixed_receivables_element_gross_percent=data.get('fixed_receivables_element_gross_percent',0),
                    value = get_amount_by_percent(
                        data.get('fixed_receivables_element_gross_percent',0),monthlysalarystructureinstance.gross_money)
                )
                instance.save()

    def bulkCreateEmployeeRegulatoryRecievables(self,monthlysalarystructureinstance,employee_regulatory_recievables:list):

        with transaction.atomic():
            for data in employee_regulatory_recievables:
                instance = models.EmployeeRegulatoryRecievables(
                    monthly_salary_structure=monthlysalarystructureinstance,
                    Employee_regulatory_recievables=data.get('regulatory_receivables',"null"),
                    Employee_regulatory_recievables_gross_percent=data.get('regulatory_receivables_gross_percent',0.00),
                    regulatory_rates=data.get('regulatory_rates',0.00),
                    value = get_amount_by_percent(
                        data.get('regulatory_receivables_gross_percent',0),monthlysalarystructureinstance.gross_money)
                )
                instance.save()

    def bulkCreateEmployeeRegulatoryDeductable(self,monthlysalarystructureinstance,employee_regulatory_deductables:list):
            with transaction.atomic():
                for data in employee_regulatory_deductables:
                    instance = models.EmployeeRegulatoryDeductables(
                        monthly_salary_structure=monthlysalarystructureinstance,
                        Employee_regulatory_deductables=data.get('regulatory_deductables','null'),
                        Employee_regulatory_deductables_gross_percent=data.get('regulatory_deductables_gross_percent',0.00),
                        regulatory_rates=data.get('regulatory_rates',0.00),
                        value = get_amount_by_percent(
                        data.get('regulatory_deductables_gross_percent',0),monthlysalarystructureinstance.gross_money)
                    )
                    instance.save()

    def bulkCreateEmployeeOtherDeductables(self,monthlysalarystructureinstance,employee_other_deductables:list):
        with transaction.atomic():
            for data in employee_other_deductables:
                instance = models.EmployeeOtherDeductables(
                        monthly_salary_structure=monthlysalarystructureinstance,
                        Employee_other_deductables= data.get('other_deductables',"null"),
                        Employee_other_deductables_gross_percent=data.get('other_deductables_gross_percent',0.00),
                        value = get_amount_by_percent(
                        data.get('other_deductables_gross_percent',0),monthlysalarystructureinstance.gross_money)
                )
                instance.save()

class  MonthSalaryStructureCleanerSerializer(serializers.ModelSerializer):

    employee_receivables = serializers.SerializerMethodField()
    employee_regulatory_recievables = serializers.SerializerMethodField()
    employee_regulatory_deductables = serializers.SerializerMethodField()
    employee_other_deductables = serializers.SerializerMethodField()
    employee_other_receivables = serializers.SerializerMethodField()

    def get_employee_other_receivables(self,MonthlySalaryInstance):
        return models.EmployeeOtherReceivables.objects.filter(monthly_salary_structure=MonthlySalaryInstance).values(
            'other_receivables_element',
            'other_receivables_element_gross_percent','value')
    def get_employee_receivables(self,MonthlySalaryInstance):
        return models.EmployeeReceivables.objects.filter(monthly_salary_structure=MonthlySalaryInstance).values(
            'fixed_receivables_element','fixed_receivables_element_gross_percent','value'
        )

    def get_employee_regulatory_recievables(self,MonthlySalaryInstance):
        return models.EmployeeRegulatoryRecievables.objects.filter(monthly_salary_structure=MonthlySalaryInstance).values(
            'Employee_regulatory_recievables','Employee_regulatory_recievables_gross_percent','regulatory_rates',
            'value'
        )

    def get_employee_regulatory_deductables(self,MonthlySalaryInstance):
        return models.EmployeeRegulatoryDeductables.objects.filter(monthly_salary_structure=MonthlySalaryInstance).values(
            'Employee_regulatory_deductables','Employee_regulatory_deductables_gross_percent','regulatory_rates',
            'value'
        )

    def get_employee_other_deductables(self,MonthlySalaryInstance):
        return models.EmployeeOtherDeductables.objects.filter(monthly_salary_structure=MonthlySalaryInstance).values(
            'Employee_other_deductables','Employee_other_deductables_gross_percent','value',
        )
    class Meta:
        fields = "__all__"
        model = models.MonthlySalaryStructure


from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from models import (Base, Pet, Owner, Handler, Job)
# from faker import Faker

if __name__ == "__main__":
  # fake = Faker()
  print("Seeding ðŸŒ±...")
  print("Connecting to DB....")
  engine = create_engine('sqlite:///pet_app.db')
  Base.metadata.create_all(engine)

  Session = sessionmaker(bind=engine)
  session = Session()
  print("Session Created...")

  session.query(Pet).delete()
  session.query(Owner).delete()
  session.query(Handler).delete()
  session.query(Job).delete()
  session.commit()

  print("CREATING OWNERS....")
  holden = Owner(name="Holden", email="h@h.com", phone="1234", address="13 Elm Street")
  connor = Owner(name="Connor", email="c@c.com", phone="5678", address="14 Elm Street")

  session.add(holden)
  session.add(connor)
  session.commit()

  print("CREATING PETS....")
  p1 = Pet(name="fake.name()", species="Dog", breed="Mutt", temperament="bad", owner_id=holden.id)
  p2 = Pet(name="chauncy", species="Dog", breed="Golden", temperament="good", owner_id=holden.id)
  p3 = Pet(name="Hank", species="Cat", breed="Orange", temperament="really bad", owner_id=holden.id)
  p4 = Pet(name="Derby", species="Aligator", breed="Croc", temperament="snuggly", owner_id=connor.id)

  # seeding_pets = []
  # species_types = ["dog", 'cat', 'aligator']

  # for pet in range(0,100):
  #     seeding_pets.append(Pet(name=fake.name(), species="Dog", breed="Mutt", temperament="bad", owner_id=holden.id))
  # session.add_all(seeding_pets)

  session.add_all([p1,p2,p3,p4])
  session.commit()

  print("CREATING HANDLERS...")
  h1 = Handler(name="wally", email="w@w.com", phone=4321, hourly_rate=1.0)
  h2 = Handler(name="shelbs", email="s@s.com", phone=5432, hourly_rate=4.5)
  h3 = Handler(name="bill", email="b@b.com", phone=4231, hourly_rate=5.7)

  session.add_all([h1,h2, h3])
  session.commit()

  print("CREATING JOBS...")

  j1 = Job(date="Feb 1st", request="Walks", fee=55.0, pet_id=p1.id, handler_id=h1.id)
  j2 = Job(date="Feb 2nd", request="Runs", fee=23.50, pet_id=p1.id, handler_id=h2.id)
  j3 = Job(date="Feb 3rd", request="Sleep", fee=5.6, pet_id=p1.id, handler_id=h3.id)
  j4 = Job(date="April 1st", request="Meds", fee=1.0, pet_id=p3.id, handler_id=h1.id)

  session.add_all([j1, j2, j3, j4])
  session.commit()

  print("DONE!")


import csv
import logging

import psycopg2.extras

from . import DBManager
from . import dimRole
from . import dimPerson

LOGGING = logging.getLogger(__name__)

def stage_csv(conn, file_path):
    LOGGING.debug("StagingFile '{file}'".format(file=file_path))
    with open(file_path, 'r') as csv_file:
        reader = csv.DictReader(csv_file)
        # ToDo: Validation of column names and types
        with conn.cursor() as cur:
            psycopg2.extras.execute_values(
                cur,
                """
                INSERT INTO
                  stg.dimPersonRole(
                    source_file_name,
                    source_file_creation,
                    externalReference_Person,
                    externalReference_Role,
                    effective_from
                  )
                VALUES
                %s
                """,
                reader,
                template="""
                  (
                    %(source_file_name)s,
                    %(create_date)s,
                    %(person_id)s,
                    %(role)s,
                    %(profile_modify_date)s
                  )
                """,
                page_size=1000
            )
            # ToDo: Logging of rows upload, time taken, etc
            conn.commit()

def cascadeActivations(conn, source):
    # to all children first
    pass
    # any necessary actions here
    dimRole.registerInitialisations(
        conn,
        """
          (
            SELECT DISTINCT externalReference_Role FROM stg.dimPersonRole WHERE externalReference_Role IS NOT NULL
          )
            {alias}
        """,
        {'externalReference_Role': 'externalReference_Role'}
    )
    dimPerson.registerInitialisations(
        conn,
        """
          (
            SELECT DISTINCT externalReference_Person FROM stg.dimPersonRole
          )
            {alias}
        """,
        {'externalReference_Person': 'externalReference_Person'}
    )

    # to all foreign key dependancies
    dimRole.cascadeActivations(conn, 'dimPersonRole')
    if (source != 'dimPerson'):
        dimPerson.cascadeActivations(conn, 'dimPersonRole')

def cascadeRetirements(conn, source):
    # to all foreign key dependancies first
    dimRole.cascadeRetirements(conn, 'dimPersonRole')
    if (source != 'dimPerson'):
        dimPerson.cascadeRetirements(conn, 'dimPersonRole')

    # any necessary actions here
    applyOrderOfPrecedence(conn)
    stageDeactivations(conn)
    resolveStagingFKs(conn)
    stageInitsTruncsAndUpdates(conn)

    # to all children
    pass

def applyOrderOfPrecedence(conn):
    LOGGING.debug("applyOrderOfPrecedence()")
    with conn.cursor() as cur:
        cur.execute(
            """
            WITH
              collapsed
            AS
            (
              SELECT DISTINCT
                externalreference_person,
                source_file_creation,
                source_file_name,
                effective_from
              FROM
                stg.dimpersonrole
            ),
              running_lowest
            AS
            (
              SELECT
                *,
                MIN(effective_from)
                  OVER (PARTITION BY externalreference_person
                            ORDER BY source_file_creation DESC,    -- Newer files have precedence (as if applied over the top of older files)
                                    source_file_name DESC,        -- Tie breaker : Same Person, Multiple Files, Same Creation Date - Behave as if applied in name order
                                    effective_from DESC           -- Where one file contains a sequence of changes, apply the changes in chronological order
                      )
                        AS lowest_effective_from
              FROM
                collapsed
            ),
              lagged
            AS
            (
              SELECT
                *,
                LAG(effective_from, 1, 2147483647)
                  OVER (PARTITION BY externalreference_person
                            ORDER BY source_file_creation DESC,    -- Newer files have precedence (as if applied over the top of older files)
                                    source_file_name DESC,        -- Tie breaker : Same Person, Multiple Files, Same Creation Date - Behave as if applied in name order
                                    effective_from DESC           -- Where one file contains a sequence of changes, apply the changes in chronological order
                      )
                        AS next_effective_from
              FROM
                running_lowest
              WHERE
                effective_from = lowest_effective_from             -- If the effective_from is higher than the lowest encountered so far, it's from an older file so can be ignored
            )
            UPDATE
              stg.dimpersonrole
            SET
              effective_stop = lagged.next_effective_from
            FROM
              lagged
            WHERE
                  lagged.effective_from           < lagged.next_effective_from                     -- Prevent zero duration periods
              AND lagged.externalreference_person = stg.dimpersonrole.externalreference_person
              AND lagged.source_file_name         = stg.dimpersonrole.source_file_name
              AND lagged.source_file_creation     = stg.dimpersonrole.source_file_creation
              AND lagged.effective_from           = stg.dimpersonrole.effective_from
            ;
            """
        )
    conn.commit()

def stageDeactivations(conn):
    LOGGING.debug("stageDeactivations()")
    with conn.cursor() as cur:
        cur.execute(
            """
            INSERT INTO
              stg.dimPersonRole (
                source_file_name,
                source_file_creation,
                externalreference_person,
                externalreference_role,
                effective_from,
                effective_stop,
                is_active,
                _staging_mode
              )
            SELECT
              '<De-Activate>',
              -1,
              stg.externalreference_person,
              dr.externalreference,
              stg.effective_from,
              stg.effective_stop,
              FALSE,
              'U'
            FROM
            (
              SELECT
                externalreference_person,
                effective_from,
                effective_stop
              FROM
                stg.dimPersonRole
              WHERE
                effective_stop IS NOT NULL  -- staging row not superceded by another staging row
              GROUP BY
                externalreference_person,
                effective_from,
                effective_stop
            )
              stg
            INNER JOIN
              dim.dimPerson       AS dp
                ON dp.externalReference = stg.externalreference_person
            INNER JOIN
            (
              SELECT
                personID,
                roleID
              FROM
                dim.dimPersonRole
              GROUP BY
                personID,
                roleID
            )
              dpr
                ON dpr.personID = dp.id
            INNER JOIN
              dim.dimRole         AS dr
                ON dr.id = dpr.roleID
            LEFT JOIN
              stg.dimPersonRole   AS chk
                ON  chk.externalreference_person = dp.externalreference
                AND chk.externalreference_role   = dr.externalreference
                AND chk.effective_from           = stg.effective_from
                AND chk.effective_stop          IS NOT NULL
            WHERE
              chk.externalreference_person IS NULL
            ;
            """
        )
    conn.commit()

def resolveStagingFKs(conn):
    LOGGING.debug("resolveStagingFKs()")
    with conn.cursor() as cur:
        cur.execute(
            """
            UPDATE
              stg.dimPersonRole   AS stg
            SET
              personID               = dpr.personID,
              roleID                 = dpr.RoleID,
              current_effective_from = dpr.effective_from,
              current_effective_stop = dpr.effective_stop
            FROM
              dim.dimPersonRole   AS dpr
            INNER JOIN
              dim.dimPerson       AS dp
                ON dp.id = dpr.personID
            INNER JOIN
              dim.dimRole         AS dr
                ON dr.id = dpr.roleID
            WHERE
                  stg.externalReference_Person  = dp.externalReference
              AND stg.externalReference_Role    = dr.externalReference
              AND stg.effective_from           >= dpr.effective_from
              AND stg.effective_from           <  dpr.effective_stop
            ;
            """
        )
    conn.commit()

def stageInitsTruncsAndUpdates(conn):
    LOGGING.debug("stageInitsTruncsAndUpdates()")
    with conn.cursor() as cur:
        cur.execute(
            """
            INSERT INTO
              stg.dimPersonRole (
                source_file_name,
                source_file_creation,
                externalreference_person,
                externalreference_role,
                effective_from,
                effective_stop,
                is_active,
                _staging_mode
              )
            (
              --------------------------------------------------------------------------------
              --------------------------------------------------------------------------------
              -- Entities that need initialising
              --------------------------------------------------------------------------------
              --------------------------------------------------------------------------------
              SELECT
                '<Initialise>',
                -2,
                externalreference_person,
                externalreference_role,
                0,
                MIN(effective_from),
                FALSE,
                'I'
              FROM
                stg.dimPersonRole
              WHERE
                    personID       IS     NULL  -- New Entity in need of initialisation
                AND effective_stop IS NOT NULL  -- Row not superceded by another staging row
                AND externalreference_role IS NOT NULL -- Is not a dummy record [see dimPerson.pushDeActivations()]
              GROUP BY
                externalreference_person,
                externalreference_role
              HAVING
                MIN(effective_from) >  0  -- Prevent zero length periods
              --------------------------------------------------------------------------------
            )
            ---------
            UNION ALL
            ---------
            (
              --------------------------------------------------------------------------------
              --------------------------------------------------------------------------------
              -- Associations then need updating (new association overlaps existing start)
              --------------------------------------------------------------------------------
              --------------------------------------------------------------------------------
              SELECT
                DISTINCT ON (
                  stg.externalreference_person,
                  stg.externalreference_role,
                  dim.effective_from
                )
                '<'
              || CASE WHEN dim.effective_from < stg.effective_from THEN 'Truncate'
                      WHEN dim.effective_stop > stg.effective_stop THEN 'Truncate'
                                                                  ELSE '' END
              || CASE WHEN dim.effective_from < stg.effective_from THEN ''
                                                                  ELSE 'Update' END
              || '>',
                -3,
                stg.externalreference_person,
                stg.externalreference_role,
                dim.effective_from,
                CASE WHEN dim.effective_from < stg.effective_from THEN stg.effective_from
                    WHEN dim.effective_stop > stg.effective_stop THEN stg.effective_stop
                                                                  ELSE dim.effective_stop END,
                CASE WHEN dim.effective_from < stg.effective_from THEN dim.is_active
                                                                  ELSE stg.is_active END,
                'U'
              FROM
                stg.dimPersonRole   AS stg
              INNER JOIN
                dim.dimPersonRole   AS dim
                  ON  dim.personID        = stg.personID
                  AND dim.roleID          = stg.roleID
                  AND dim.effective_from <  stg.effective_stop
                  AND dim.effective_stop  > stg.effective_from
                  AND dim.effective_from <> stg.effective_from  -- Don't create a duplicate record to update
              ORDER BY
                stg.externalreference_person,
                stg.externalreference_role,
                dim.effective_from,
                stg.effective_from
              --------------------------------------------------------------------------------
            );
            """
        )
    conn.commit()

def applyChanges(conn, source):
    if (source is None):
        cascadeActivations(conn, 'dimPersonRole')
        cascadeRetirements(conn, 'dimPersonRole')

    if (source != 'dimRole'):
        dimRole.applyChanges(conn, 'dimPersonRole')
    if (source != 'dimPerson'):
        dimPerson.applyChanges(conn, 'dimePersonRole')

    LOGGING.debug("applyChanges()")
    _applyChanges(conn)

def _applyChanges(conn):
    with conn.cursor() as cur:
        cur.execute(
            """
            DELETE FROM
              dim.dimPersonRole   d
            USING
              stg.dimPersonRole   s
            WHERE
                  s.PersonID      = d.PersonID
              AND s.RoleID        = d.RoleID
              AND s._staging_mode = 'D'
            ;

            INSERT INTO
              dim.dimPersonRole (
                PersonID,
                RoleID,
                effective_from,
                effective_stop,
                is_active
              )
            SELECT
              dp.id,
              dr.id,
              stg.effective_from,
              CASE WHEN stg.current_effective_stop < stg.effective_stop THEN stg.current_effective_stop ELSE stg.effective_stop END,
              stg.is_active
            FROM
              stg.dimPersonRole   stg
            INNER JOIN
              dim.DimPerson       dp
                ON dp.externalReference = stg.externalReference_Person
            INNER JOIN
              dim.DimRole         dr
                ON dr.externalReference = stg.externalReference_Role
            WHERE
              stg.effective_stop IS NOT NULL
            ON CONFLICT
              (
                personID,
                roleID,
                effective_from
              )
                DO UPDATE
                  SET effective_stop = EXCLUDED.effective_stop,
                      is_active      = EXCLUDED.is_active
            ;

            DELETE FROM
              stg.dimPersonRole
            ;
            """
        )
    conn.commit()


import operator
import torch
from base64 import b64decode
from json import loads
import numpy as np
from matplotlib.pyplot import figure
import matplotlib as plt
from sklearn.naive_bayes import MultinomialNB, BernoulliNB
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.tree import DecisionTreeClassifier
from torch.utils.data import DataLoader
from torch.utils.data import Dataset
import torch.nn as nn

def read_data(json_file):
    json_object = loads(json_file)
    json_data = b64decode(json_object["data"])
    digit_vector = np.fromstring(json_data, dtype=np.ubyte)
    digit_vector = digit_vector.astype(np.float64)
    return json_object["label"], digit_vector

with open("digits.base64.json", "r") as f:
    digits = [read_data(each) for each in f.readlines()]

training_size = int(len(digits) * 0.25)
test = np.array(digits[:600])
validation = np.array(digits[600:training_size])
training = np.array(digits[training_size:])

X = list(training[:, 1])
print(type(X))
print(type(X[0]))

Y = np.array(training[:, 0]).astype("int")

X_val = list(validation[:, 1])
Y_val = np.array(validation[:, 0]).astype("int")

Test_X = list(test[:, 1])
Test_Y = np.array(test[:, 0]).astype("int")

# KNN
#
#
#
# clf = KNeighborsClassifier(n_neighbors=10)
# # clf.fit(X,Y)
# print("Test set accuracy: {:.2f}".format(clf.score(X_val, Y_val)))
#
# pre = clf.predict(Test_X)
#
#
# print(clf.score(Test_X, Test_Y))
#
#
#
# #Decision Tree
#
# tree = DecisionTreeClassifier(random_state=0) # overfit
# tree.fit(X,Y)
# print("Train set accuracy: {:.2f}".format(tree.score(X, Y)))
# print("Test set accuracy: {:.2f}".format(tree.score(X_val, Y_val)))
#
# tree = DecisionTreeClassifier(max_depth=5,random_state=0)
# tree.fit(X,Y)
# print("Train set accuracy: {:.2f}".format(tree.score(X, Y)))
# print("Test set accuracy: {:.2f}".format(tree.score(X_val, Y_val)))

# Naive Bayes

# nbM = MultinomialNB()
# nbM.fit(X,Y)
# print(nbM.score(X_val,Y_val))
#
# nbB = BernoulliNB()
# nbB.fit(X,Y)
# print("Bernoulli form: {:.2f}".format(nbB.score(X_val,Y_val)))

# KNN program
# def classifyKNN(testX, dataset, label, k):
#     datasetSize = dataset.shape[0]
#     diffMat = np.tile(testX, (datasetSize, 1)) - dataset
#     sqDiffMat = diffMat**2
#     sqDistance = sqDiffMat.sum(axis=1)
#     distance = sqDistance ** 0.5
#     sortDistance = distance.argsort()
#
#     classCount = {}
#     for i in range(k):
#         voteLabel = label[sortDistance[i]]
#         classCount[voteLabel] = classCount.get(voteLabel, 0) + 1
#
#     sortedClassCount = sorted(classCount.items(), key=operator.itemgetter(1), reverse=True)
#
#     return sortedClassCount[0][0]
#
#
# predY = []
#
# a = np.array(X)
#
# for testX in Test_X:
#     predY.append(classifyKNN(testX, a, Y, 6))
#
# score = 0
# for i in range(len(predY)):
#     if predY[i] == Test_Y[i]:
#         score += 1
#
# print(score)
# print("KNN score is {:.2f}".format(score / len(Test_Y)))
#

# SVM
# clf = SVC(C = 5)
# clf.fit(X,Y)
#
# print("SVM sklearn RBF kernal: {:.2f}".format(clf.score(X_val,Y_val)))

# clf = SVC(C=5, kernel= 'sigmoid')
# clf.fit(X,Y)
# print("SVM sklearn Sigmoid kernal: {:.2f}".format(clf.score(X_val,Y_val)))

# clf = SVC(C=5, kernel= 'linear')
# clf.fit(X,Y)
# print("SVM sklearn Linear kernal: {:.2f}".format(clf.score(X_val,Y_val)))

# Neural necwork

def get_device():
    return 'cuda' if torch.cuda.is_available() else 'cpu'

class NUMDataset(Dataset):
    def __init__(self, mode="train"):
        self.mode = mode
        if mode == 'train':
            self.X_data = torch.tensor(np.array(X))
            print(self.X_data.shape)
            self.Y_data = torch.tensor(np.array(Y))
            print(self.Y_data.shape)
        elif mode == 'val':
            self.X_data = torch.tensor(np.array(X_val))
            self.Y_data = torch.tensor(np.array(Y_val))
        else:
            self.X_data = torch.tensor(np.array(Test_X))
            self.Y_data = torch.tensor(np.array(Test_Y))

    def __getitem__(self, index):
        if self.mode in ['train', 'val']:
            return self.X_data[index], self.Y_data[index]
        else:
            return self.X_data[index]

    def __len__(self):
        return len(self.X_data)

def prep_dataLoader(mode, batch_size):
    dataset = NUMDataset(mode)
    dataloader = DataLoader(
        dataset, batch_size,
        shuffle=True
    )
    return dataloader

BATCH_SIZE = 1
train_len = NUMDataset('train').__len__()
train_loader = prep_dataLoader(mode='train', batch_size=BATCH_SIZE)
val_len = NUMDataset('val').__len__()
val_loader = prep_dataLoader(mode='val', batch_size=BATCH_SIZE)

class Classifier(nn.Module):
    def __init__(self):
        super(Classifier, self).__init__()

        self.layer1 = nn.Linear(784, 2048)
        self.layer2 = nn.Linear(2048, 512)
        self.layer3 = nn.Linear(512, 256)
        self.out = nn.Linear(256, 10)

        self.act_fn = nn.Sigmoid()

    def forward(self, x):
        x = self.layer1(x)
        x = self.act_fn(x)

        x = self.layer2(x)
        x = self.act_fn(x)

        x = self.layer3(x)
        x = self.act_fn(x)

        x = self.out(x)

        return x

def same_seeds(seed):
    torch.manual_seed(seed)
    if torch.cuda.is_available():
        torch.cuda.manual_seed(seed)
        torch.cuda.manual_seed_all(seed)
    np.random.seed(seed)
    torch.backends.cudnn.benchmark = False
    torch.backends.cudnn.deterministic = True

same_seeds(1)
device = get_device()
print(f'DEVICE: {device}')
num_epoch = 20
learning_rate = 0.001

model_path = './model.ckpt'

model = Classifier().to(device)

criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)

best_acc = 0.0

for epoch in range(num_epoch):
    train_acc = 0.0
    train_loss = 0.0
    val_acc = 0.0
    val_loss = 0.0

    model.train()
    for i, data in enumerate(train_loader):
        inputs, labels = data
        inputs, labels = inputs.to(device), labels.to(device)
        optimizer.zero_grad()

        inputs = inputs.to(torch.float32)
        output = model(inputs)

        labels = labels.to(torch.Tensor().long())
        batch_loss = criterion(output, labels)
        _, train_pred = torch.max(output, 1)
        batch_loss.backward()
        optimizer.step()

        train_acc += (train_pred.cpu() == labels.cpu()).sum().item()

        train_loss = batch_loss.item()

    if len(X_val) > 0:
        model.eval()
        with torch.no_grad():
            for i, data in enumerate(val_loader):
                inputs, labels = data
                inputs, labels = inputs.to(device), labels.to(device)
                optimizer.zero_grad()
                inputs = inputs.to(torch.float32)
                output = model(inputs)
                labels = labels.to(torch.Tensor().long())
                batch_loss = criterion(output, labels)
                _, val_pred = torch.max(output, 1)

                val_acc += (val_pred.cpu() == labels.cpu()).sum().item()
                val_loss = batch_loss.item()
            print('[{:03d}/{:03d}] Train Acc: {:3.6f} Loss: {:3.6f} | Val Acc: {:3.6f} loss: {:3.6f}'.format(
                epoch + 1, num_epoch, train_acc / train_len, train_loss / train_len, val_acc / val_len,
                val_loss / val_len
            ))

        if val_acc > best_acc:
            best_acc = val_acc
            torch.save(model.state_dict(), model_path)
            print('saving model with acc {:.3f}'.format(best_acc / len(X_val)))

    else:
        print('[{:03d}/{:03d}] Train Acc: {:3.6f} Loss: {:3.6f}'.format(
            epoch + 1, num_epoch, train_acc / len(X), train_loss / len(train_loader)
        ))

if len(X_val) == 0:
    torch.save(model.state_dict(), model_path)
    print('saving model at last epoch')


import shelve

from sklearn.metrics import SCORERS
d = shelve.open("hisc")
d["hisc"] = 56
d.close()

print('-----------')

print("Welcome to the Wikipedia quiz show! So many useless facts.")
print("Please input your username and password to continue. ")

playing = input("Join the game? (Yes/No) ")

if playing.lower() != "yes":
    print("Quitting Game")
    quit()

correct_score = 0
incorrect_score = 0

def quizgame():
    print(" Welcome! Welcome! Come on in ")

    def wikipedia_question():
        answer = input(str("Test Question "))

        global correct_score
        global incorrect_score
        def scoreboard():
                print("You have answered " + str(incorrect_score) + " questions incorrectly")
                print("You have answered " + str(correct_score) + " questions correctly")

        if answer ==  (str(45)):
            print("Correct! ")
            correct_score += 1
            scoreboard()
            wikipedia_question()

        else:
            print("Oops, that wasn't the right answer ")
            incorrect_score += 1
            scoreboard()
            print(incorrect_score)
            print(correct_score)

            if incorrect_score > 3:
                print("Game over!")
                print("You got " + str(correct_score) + " questions correct.")
            else:
                wikipedia_question()

    wikipedia_question()

quizgame()


import RPi.GPIO as gpio
import time
import Adafruit_ADS1x15

class LevelSensor:
    def __init__(self):
        self.pool_level = 8.0
        self.adc = Adafruit_ADS1x15.ADS1015()
        self.GAIN = 1
        self.adc_pin = 0
        self.max_reading = 8

    def get_adc_resistance(self, pool_level):
        inches_left = 8.0
        upper_bound = 1130
        lower_bound = 387
        total_boundary = upper_bound - (lower_bound)
        if pool_level > upper_bound:
            pool_level = upper_bound
        elif pool_level < lower_bound:
            pool_level = lower_bound
        tolerance = round((lower_bound/float(total_boundary)) * self.max_reading,2)
        inches_left = round((pool_level/float(total_boundary)) * self.max_reading,2) - tolerance
        if inches_left < 0.2:
            inches_left = 0.0
        return inches_left

    def get_pool_level(self):
        pool_level = self.adc.read_adc(self.adc_pin, gain=self.GAIN)
        #print(pool_level)
        time.sleep(0.25)
        self.pool_level = self.get_adc_resistance(pool_level)
        return self.pool_level

    def check_overflow(self):
        if self.pool_level <= 0.0:
            return 1
        else:
            return 0

def main():
    level = LevelSensor()
    pool_level = 0
    while True:
        pool_level = level.get_pool_level()
        print("The reading of the Level Sensor is: %.2f inches"%(pool_level))
        print("\n")

if __name__ == "__main__":
    main()


import urllib.request
import json

from flask import *
import requests

app = Flask(__name__)

@app.route("/")
def home():
    return render_template('bankingui.html')

@app.route("/success", methods=['GET','POST'])
def home2():
    if request.method == 'POST':

        data = {
            "Inputs": {
                "data":
                [
                    {
                        "Cloumn2":"request.form['x']",
                        "age" : int(request.form['a']),
                        "job" : "request.form['b']",
                        "marital": "request.form['c']",
                        "education": "request.form['d']",
                        "default": "request.form['e']",
                        "housing":"request.form['f']",
                        "loan":"request.form['g']",
                        "contact":"request.form['h']",
                        "month":"request.form['i']",
                        "duration":int(request.form['j']),
                        "campaign":int(request.form['k']),
                        "pdays":int(request.form['l']),
                        "previous":int(request.form['m']),
                        "poutcome":"request.form['n']",
                        "emp.var.rate":float(request.form['o']),
                        "cons.price.idx":float(request.form['p']),
                        "cons.conf.idx":float(request.form['q']),
                        "euribor3m":float(request.form['r']),
                        "nr.employed":float(request.form['s'])
                    },
                ]
            },
            "GlobalParameters": {
                "method": "predict"
            }
        }

        body = str.encode(json.dumps(data))

        url = 'http://ff32e937-0995-4958-a89b-d373e8248865.westus2.azurecontainer.io/score'

        api_key = '' # Replace this with the API key for the web service

        headers = {'Content-Type':'application/json', 'Authorization':('Bearer '+ api_key)}

        req = urllib.request.Request(url, body, headers)

        try:
            response = urllib.request.urlopen(req)

            result = response.read()

            print(result)

        except urllib.error.HTTPError as error:

            print("The request failed with status code: " + str(error.code))

    # Print the headers - they include the requert ID and the timestamp, which are useful for debugging the failure
            print(error.info())
            print(json.loads(error.read().decode("utf8", 'ignore')))

    return render_template('success.html', data = result)

if __name__ == '__main__':
    app.run(debug = True)


#!/usr/bin/env python3
# coding=utf-8
import asyncio

import aiohttp
import pytest

from podcastdownloader.episode import Episode
from podcastdownloader.exceptions import EpisodeException

@pytest.fixture(scope='session')
def client_session() -> aiohttp.ClientSession:
    out = aiohttp.ClientSession()
    return out

@pytest.mark.parametrize(('test_link_dict', 'expected'), (
    ([{
        'rel': 'alternate',
        'type': 'text/html',
        'href': 'http://evenmorenewspodcast.com/usa-still-leaving-afghanistan-still-shooting'
        '-unarmed-people-w-christopher-rivas-ep-142',
    },
        {
        'length': '54468321',
        'type': 'audio/mpeg',
        'href': 'https://dts.podtrac.com/redirect.mp3/chtbl.com/track/242FB3/'
        'traffic.libsyn.com/secure/evenmorenews/EMN_Ep142.mp3?dest-id=695480',
        'rel': 'enclosure',
    }],
        'https://dts.podtrac.com/redirect.mp3/chtbl.com/track/242FB3/'
        'traffic.libsyn.com/secure/evenmorenews/EMN_Ep142.mp3?dest-id=695480'),
))
def test_episode_find_url(test_link_dict: list[dict], expected: str):
    test_dict = {'links': test_link_dict, }
    result = Episode._find_url(test_dict)
    assert result == expected

@pytest.mark.parametrize('test_link_dict', (
    [{
        'rel': 'alternate',
        'type': 'text/html',
        'href': 'http://evenmorenewspodcast.com/usa-still-leaving-afghanistan-still-shooting'
        '-unarmed-people-w-christopher-rivas-ep-142',
    }],
))
def test_episode_find_url_bad(test_link_dict: list[dict]):
    test_dict = {'links': test_link_dict, }
    with pytest.raises(EpisodeException):
        Episode._find_url(test_dict)

@pytest.mark.parametrize(('test_url', 'expected'), (
    ('https://www.example.com/test.png', '.png'),
    ('https://www.example.com/test.mp3', '.mp3'),
    ('https://www.example.com/random/test.flac', '.flac'),
    ('https://www.example.com/test.mp3?test=value', '.mp3'),
    ('https://www.example.com/test.mp3?test=value#test', '.mp3'),
    ('https://www.example.com/test.aac', '.aac'),
))
def test_determine_file_extension_from_url(test_url: str, expected: str, client_session):
    result = asyncio.run(Episode._get_file_extension(test_url, client_session))
    assert result == expected

@pytest.mark.parametrize(('test_name', 'expected'), (
    ('test', 'test'),
    ('te/st', 'test'),
    ('test/test', 'testtest'),
    ('test\0', 'test'),
))
def test_clean_name(test_name: str, expected: str):
    result = Episode._clean_name(test_name)
    assert result == expected


import eg

eg.RegisterPlugin(
    name = "TechnoTrend USB IR Receiver",
    author = "Bitmonster",
    version = "1.1",
    kind = "remote",
    guid = "{66211720-B0DE-46E2-AD57-2D70F477DE02}",
    description = __doc__,
)

import os
from threading import Lock, Timer, Thread
from time import clock, sleep
from functools import partial

from eg.WinApi.Dynamic import (
    cdll, DWORD, POINTER, ULONG, HANDLE, BYTE, c_void_p, c_int, CFUNCTYPE, CDLL
)

USBIR_MODE_DIV = 2

# typedef void (*PIRCBFCN) (PVOID Context,
#                           PVOID Buf,
#                           ULONG len, // buffer length in bytes
#                           USBIR_MODES IRMode,
#                           HANDLE hOpen,
#                           BYTE DevIdx);
IRCALLBACKFUNC = CFUNCTYPE(
    c_void_p, # return type
    c_void_p, # Context
    POINTER(DWORD), # Buf
    ULONG, # len (buffer length in bytes)
    c_int, # IRMode (of enum USBIR_MODES)
    HANDLE, # hOpen
    BYTE # DevIdx
)

class TTIR(eg.IrDecoderPlugin):

    def __init__(self):
        eg.IrDecoderPlugin.__init__(self, 1)

    def __close__(self):
        self.irDecoder.Close()

    def __start__(self):
        self.dll = None
        self.hOpen = None
        pluginDir = os.path.abspath(os.path.dirname(__file__))
        dll = CDLL(os.path.join(pluginDir, "TTUSBIR.dll"))
        self.cCallback = IRCALLBACKFUNC(self.IrCallback)
        self.hOpen = dll.irOpen(0, USBIR_MODE_DIV, self.cCallback, 0)
        if self.hOpen == -1:
            raise self.Exceptions.DeviceNotFound
#        self.irGetUniqueCode = dll.ir_GetUniqueCode
#        self.irGetUniqueCode.restype  = DWORD
        self.dll = dll
        self.data = []
        self.timer = eg.ResettableTimer(self.OnTimeout)
        self.dataLock = Lock()
        self.lastTime = clock()
        self.startByte = 1
        self.dll.irSetPowerLED(self.hOpen, 0)
        self.ledTimer = eg.ResettableTimer(
            partial(self.dll.irSetPowerLED, self.hOpen, 0)
        )

    def OnComputerSuspend(self, dummySuspendType):
        self.dll.irClose(self.hOpen)

    def OnComputerResume(self, dummySuspendType):
        self.hOpen = self.dll.irOpen(
            0, USBIR_MODE_DIV, self.cCallback, 0
        )
        if self.hOpen == -1:
            raise self.Exceptions.DeviceNotFound

    def __stop__(self):
        if self.dll is not None:
            self.dll.irClose(self.hOpen)
            self.dll = None
            self.hOpen = None
            self.cCallback = None
        self.timer.Stop()
        self.ledTimer.Stop()

    def IrCallback(self, context, buf, length, irMode, hOpen, devIdx):
        if irMode == USBIR_MODE_DIV:
            self.dll.irSetPowerLED(self.hOpen, 1)
            self.ledTimer.Reset(1)
            self.timer.Reset(80)
            #self.lastTime = clock()
            self.dataLock.acquire()
            #print "---", length / 4, clock() - self.lastTime
            append = self.data.append
            for i in xrange(self.startByte, min(length/4, 500)):
                value = buf[i] & 0x00ffffff
                append(value)
                if value > 12000:
                    self.irDecoder.Decode(self.data, len(self.data))
                    self.data = []
                    append = self.data.append
            self.startByte = 0
            self.dataLock.release()

    def OnTimeout(self):
        if self.dataLock.acquire(0):
            data = self.data
            self.data = []
            self.startByte = 1
            self.dataLock.release()
            data.append(10000)
            self.irDecoder.Decode(data, len(data))


def data_type(text, num):
    if text == "int":
        return int(num) * 2
    if text == "real":
        result = float(num) * 1.5
        return f"{result:.2f}"
    if text == "string":
        return f"${num}$"

command = input()
number = input()
print(data_type(command, number))


from django.contrib.auth import logout
from django.urls import path
from .views import *

urlpatterns = [
    path('', HomepageView.as_view(), name="home"),
    path('signup/', signup_view, name="signup"),
    path('login/', login_view, name="login"),
    path('logout/', logout_view, name="logout"),
    path('send_friend_request/<int:userID>/',
        send_friend_request, name="send_friend_request"),
    path('accept_friend_request/<int:requestID>/',
        accept_friend_request, name="accept_friend_request"),
    path('friends/', friend_list, name="friend_list"),
    path('remove_friend/<int:userID>/', remove_friend,
        name="remove_friend"),
]


"""Reversing the elements of list of list."""
list1 = [["a", "b", "c"], ["a", "b"], ["a"]]
'''Expected output : [["a"],["b", "a"],["c", "b", "a"]]'''
lis = []
for i in list1:
    # for v in i:
    newlist = i[::-1]
    lis.append(newlist)
lis.reverse()
print(lis)


from secml.testing import CUnitTest

from secml.data.splitter import CTrainTestSplit
from secml.data.loader import CDLRandom
from secml.data import CDataset
from secml.array import CArray

class TestCTrainTestSplit(CUnitTest):
    """Unit test for train/test split."""

    def test_train_test_split(self):

        ds = CDLRandom(n_samples=10, random_state=0).load()

        tts = CTrainTestSplit(train_size=0.5, random_state=0, shuffle=False)

        tr_idx, ts_idx = tts.compute_indices(ds)

        self.logger.info("TR IDX:\n{:}".format(tr_idx))
        self.logger.info("TS IDX:\n{:}".format(ts_idx))

        tr_idx_expected = CArray([0, 1, 2, 3, 4])
        ts_idx_expected = CArray([5, 6, 7, 8, 9])

        self.assertIsInstance(tr_idx, CArray)
        self.assertIsInstance(ts_idx, CArray)

        self.assertFalse((tr_idx != tr_idx_expected).any())
        self.assertFalse((ts_idx != ts_idx_expected).any())

        tr, ts = tts.split(ds)

        tr_expected = ds[tr_idx, :]
        ts_expected = ds[ts_idx, :]

        self.assertIsInstance(tr, CDataset)
        self.assertIsInstance(ts, CDataset)

        self.assertFalse((tr.X != tr_expected.X).any())
        self.assertFalse((tr.Y != tr_expected.Y).any())
        self.assertFalse((ts.X != ts_expected.X).any())
        self.assertFalse((ts.Y != ts_expected.Y).any())

        self.logger.info("Testing splitting of sparse dataset")
        ds = CDLRandom(n_samples=10, random_state=0).load()

        ds = ds.tosparse()

        tts = CTrainTestSplit(train_size=0.25, random_state=0, shuffle=False)
        tr, ts = tts.split(ds)

        self.assertEqual(2, tr.num_samples)
        self.assertEqual(8, ts.num_samples)

        self.assertTrue(tr.issparse)
        self.assertTrue(ts.issparse)

if __name__ == '__main__':
    CUnitTest.main()