#we have data from the ESTO energy data set that the output from this model needs to match. hwoever the way this model works is that its output is a function of the input data, which is activity data (the energy use is the final results). so we need to adjust the input data so that the output matches the ESTO data. because there are so many steps in teh system, this will be a bit complicated. We will do this as follows:
# - biofuels: just base it off the total esto demadn for gasoline adn diesel. calculate share of oil use that the biofuels would make up
# - gasoline/diesel: adjust road: cars and lcvs to make it easy. split the difference in half and then apply half change to each.
# - elec: just decrease ev car stocks
# - non road fuels: decrease use in non road
# - gas: bit more difficult. decrease what vehicle types?

#but also we need to know the expected energy use for the period betwen (and including) BASE_YEAR and OUTLOOK_BASE_YEAR. so we need to run the model for that period first to get the output energy use. so we need to run the model twice. once for the period between BASE_YEAR and OUTLOOK_BASE_YEAR, and then finally for the period between OUTLOOK_BASE_YEAR and OUTLOOK_END_YEAR (not base year and end year bcause its important the results in OUTLOOK_BASE_YEAR are what we expect) then we can adjust the data for the first period so that the output matches the ESTO data. then we can use the data for the second period as the input data for the outlook model.
#adjusting data will involve:
#rescaling energy use for each fuel type so that the total energy use matches the ESTO data > apply this to the most suitable drive types/vehicle types so its less complicated.
#then based on the new energy use for each vehicle type/drive  type, recalcualte the activity and stocks (given the data which should be constant for mielage/occupancy_load).
# #done

#hwoever one thing that will be different is that for biofuels demand, we will maek the 'supplyside share of biofeusl demand equivalent to the share of biofuels in the esto data.

##########
#take in input data we will adjsut: (is this needed. can jsut use model output detailed)
# input_data_new.to_csv('intermediate_data/model_inputs/input_data_new.csv', index=False)
# non_input_data_new.to_csv('intermediate_data/model_inputs/non_input_data_new.csv', index=False)

#take in supply side fuel shares:
# supply_side_fuel_mixing = pd.read_csv('intermediate_data\model_inputs\supply_side_fuel_mixing_COMPGEN.csv')

#take in energy use by fuels
# model_output_all_with_fuels = pd.read_csv('output_data/model_output_with_fuels/{}'.format(model_output_file_name))

#take in detailed data output so we can adjsut the stocks and activity for the first period
# model_output_detailed.to_csv('output_data/model_output_detailed/{}'.format(model_output_file_name), index=False)
#take in esto data:

#%%
#aslo note that esto data is by medium. i think it ha road split into freight and passenger transport types too.
import os
import re
os.chdir(re.split('transport_model_9th_edition', os.getcwd())[0]+'\\transport_model_9th_edition')
from runpy import run_path
import sys
sys.path.append("./workflow")
exec(open("config/config.py").read())#usae this to load libraries and set variables. Feel free to edit that file as you need

import utility_functions

def adjust_supply_side_fuel_share(energy_use_esto,supply_side_fuel_mixing):
    #find portion of '16_06_biodiesel', '16_05_biogasoline', '16_07_bio_jet_kerosene' out of the toal '07_07_gas_diesel_oil', '07_01_motor_gasoline', '07_x_jet_fuel' in the esto data so we can change the supply side fuel mixing to match:
    energy_use_esto_wide = energy_use_esto.groupby(['Economy', 'Date', 'Fuel']).sum(numeric_only=True).reset_index()
    energy_use_esto_wide = energy_use_esto_wide.pivot(index=['Economy', 'Date'], columns='Fuel', values='Energy').reset_index()
    energy_use_esto_wide['share_of_biodiesel'] = energy_use_esto_wide['16_06_biodiesel']/(energy_use_esto_wide['07_07_gas_diesel_oil']+energy_use_esto_wide['16_06_biodiesel'])
    energy_use_esto_wide['share_of_biogasoline'] = energy_use_esto_wide['16_05_biogasoline']/(energy_use_esto_wide['07_01_motor_gasoline']+energy_use_esto_wide['16_05_biogasoline'])

    energy_use_esto_wide['share_of_biogas'] = energy_use_esto_wide['16_01_biogas']/(energy_use_esto_wide['16_01_biogas']+energy_use_esto_wide['07_09_lpg']+energy_use_esto_wide['08_01_natural_gas'])

    try:#some economys dont sue aviation gasoline or kerosene, so we need to catch the error, and just set the value to 0 before doing this:
        energy_use_esto_wide['share_of_bio_jet'] = energy_use_esto_wide['16_07_bio_jet_kerosene']/(energy_use_esto_wide['07_x_jet_fuel']+energy_use_esto_wide['16_07_bio_jet_kerosene']+energy_use_esto_wide['07_02_aviation_gasoline']+energy_use_esto_wide['07_06_kerosene'])
    except:
        if '07_02_aviation_gasoline' not in energy_use_esto_wide.columns:
            energy_use_esto_wide['07_02_aviation_gasoline'] = 0
        if '07_06_kerosene' not in energy_use_esto_wide.columns:
            energy_use_esto_wide['07_06_kerosene'] = 0
        if '07_x_jet_fuel' not in energy_use_esto_wide.columns:
            energy_use_esto_wide['07_x_jet_fuel'] = 0
        energy_use_esto_wide['share_of_bio_jet'] = energy_use_esto_wide['16_07_bio_jet_kerosene']/(energy_use_esto_wide['07_x_jet_fuel']+energy_use_esto_wide['16_07_bio_jet_kerosene']+energy_use_esto_wide['07_02_aviation_gasoline']+energy_use_esto_wide['07_06_kerosene'])

    #manually create dfs then concat them:
    share_of_biodiesel = energy_use_esto_wide[['Economy', 'Date', 'share_of_biodiesel']].copy()
    #create cols for each fuel type:
    share_of_biodiesel['Fuel'] = '07_07_gas_diesel_oil'
    share_of_biodiesel['New_fuel'] = '16_06_biodiesel'
    share_of_biodiesel = share_of_biodiesel.rename(columns={'share_of_biodiesel': 'Supply_side_fuel_share'})

    share_of_biogasoline = energy_use_esto_wide[['Economy', 'Date', 'share_of_biogasoline']].copy()
    share_of_biogasoline['Fuel'] = '07_01_motor_gasoline'
    share_of_biogasoline['New_fuel'] = '16_05_biogasoline'
    share_of_biogasoline = share_of_biogasoline.rename(columns={'share_of_biogasoline': 'Supply_side_fuel_share'})
    #now do it for the ones where we have one biofuel for multiple fuels:
    share_of_biogas = energy_use_esto_wide[['Economy', 'Date', 'share_of_biogas']].copy()
    share_of_biogas['New_fuel'] = '16_01_biogas'
    share_of_biogas = share_of_biogas.rename(columns={'share_of_biogas': 'Supply_side_fuel_share'})
    share_of_biogas_in_lpg = share_of_biogas.copy()
    share_of_biogas_in_lpg['Fuel'] = '07_09_lpg'
    share_of_biogas_in_nat_gas = share_of_biogas.copy()
    share_of_biogas_in_nat_gas['Fuel'] = '08_01_natural_gas'

    share_of_bio_jet = energy_use_esto_wide[['Economy', 'Date', 'share_of_bio_jet']].copy()
    share_of_bio_jet['New_fuel'] = '16_07_bio_jet_kerosene'
    share_of_bio_jet = share_of_bio_jet.rename(columns={'share_of_bio_jet': 'Supply_side_fuel_share'})
    share_of_bio_jet_in_jet = share_of_bio_jet.copy()
    share_of_bio_jet_in_jet['Fuel'] = '07_x_jet_fuel'
    share_of_bio_jet_in_aviation_gasoline = share_of_bio_jet.copy()
    share_of_bio_jet_in_aviation_gasoline['Fuel'] = '07_02_aviation_gasoline'
    share_of_bio_jet_in_kerosene = share_of_bio_jet.copy()
    share_of_bio_jet_in_kerosene['Fuel'] = '07_06_kerosene'

    #concat them and then join to supplu_side_fuel_mixing, then swap supply_side_fuel_share for the new one, where available:
    new_share = pd.concat([share_of_biodiesel, share_of_biogasoline, share_of_biogas_in_lpg, share_of_biogas_in_nat_gas, share_of_bio_jet_in_jet, share_of_bio_jet_in_aviation_gasoline, share_of_bio_jet_in_kerosene]).reset_index(drop=True)
    supply_side_fuel_mixing_new = supply_side_fuel_mixing.merge(new_share, on=['Economy', 'Date', 'Fuel','New_fuel'], how='left', suffixes=('', '_new')).copy()
    #where there is a new share, use that, otherwise use the old one
    supply_side_fuel_mixing_new['Supply_side_fuel_share'] = supply_side_fuel_mixing_new['Supply_side_fuel_share_new'].fillna(supply_side_fuel_mixing_new['Supply_side_fuel_share'])
    #drop Supply_side_fuel_share_new
    supply_side_fuel_mixing_new = supply_side_fuel_mixing_new.drop(columns=['Supply_side_fuel_share_new'])

    return supply_side_fuel_mixing_new
#filter for REF only in input data (since we want both scnearios to have matching input data)
# ['Scenario'] == 'Reference']

#make energy_use_esto match the format of energy_use_output for ref scenario
#drop any aggregate fuels from energy_use_esto

def format_energy_use_for_rescaling(energy_use_esto, energy_use_output, spread_non_specified_and_separate_pipeline = True, remove_annoying_fuels = True, TESTING=False):
    #FORMAT ENERGY USE DFS

    #WITH THIS I DONT THINK THERE IS ANYTHING WE NEED TO DO HERE ANBYNORE. AS THE ROWS ARE NOW IN THE DATA FROM EBT (EXCEPT anything that has changed from what i expect the ebt to have)
    # #these are fuels that arent in the esto data because they arent reported on, so we just need to make sure that their sums are 0 in the mdoel input data,
    # model_fuels_to_ignore = ['16_x_ammonia','16_x_efuel', '16_x_hydrogen', '16_07_bio_jet_kerosene','07_x_other_petroleum_products']

    # missing_fuels = [fuel for fuel in energy_use_output.Fuel.unique() if fuel not in energy_use_esto.Fuel.unique()] + [fuel for fuel in energy_use_esto.Fuel.unique() if fuel not in energy_use_output.Fuel.unique()]

    # missing_fuels = [fuel for fuel in missing_fuels if fuel not in model_fuels_to_ignore]

    # if len(missing_fuels)>0:
    #     print('missing fuels in energy_use_esto')
    #     print(missing_fuels)
    #     raise ValueError('missing fuels in energy_use_esto')

    # #drop nonneeded cols
    # energy_use_esto = energy_use_esto.drop(columns=['Frequency', 'Source', 'Dataset', 'Measure', 'Vehicle Type', 'Drive', 'Transport Type', 'Unit'])

    # if not TESTING:
    #     energy_use_esto['Date'] = energy_use_esto['Date'].apply(lambda x: x[:4])
    #     energy_use_esto['Date'] = energy_use_esto['Date'].astype(int)
    # #drop economy in APEC '22_SEA', '23_NEA', '23b_ONEA', '24_OAM', '24b_OOAM', '25_OCE'
    # energy_use_esto = energy_use_esto.loc[~energy_use_esto['Economy'].isin(['22_SEA', '23_NEA', '23b_ONEA', '24_OAM', '24b_OOAM', '25_OCE', 'APEC'])].copy()
    spread_non_specified_and_separate_pipeline=False#for now ignoring this
    if spread_non_specified_and_separate_pipeline:
        #separate pipeline and nonspecified mediums in energy use esto. they will be dealt with by the transport model at a later date
        energy_use_esto_pipeline = energy_use_esto.loc[energy_use_esto['Medium'] == 'pipeline'].copy()
        energy_use_esto_nonspecified = energy_use_esto.loc[energy_use_esto['Medium'] == 'nonspecified'].copy()
        energy_use_esto = energy_use_esto.loc[~energy_use_esto['Medium'].isin(['pipeline', 'nonspecified'])].copy()

        #add Fuel 07_08_fuel_oil and 07_06_kerosene use in road to the non specified amount and spread it with nonspecifieds energy use. This is because we dont have any info on these uses and they are minmimal so mimght as well drop them in with the nonspecifieds
        if remove_annoying_fuels:
            annoying_road_fuels_df = energy_use_esto.loc[(energy_use_esto['Medium'] == 'road') & (energy_use_esto['Fuel'].isin(['07_08_fuel_oil', '07_06_kerosene']))].copy()
            energy_use_esto = energy_use_esto.loc[~((energy_use_esto['Medium'] == 'road') & (energy_use_esto['Fuel'].isin(['07_08_fuel_oil', '07_06_kerosene'])))].copy()
            #set meidum to nonspecified
            annoying_road_fuels_df['Medium'] = 'nonspecified'
            energy_use_esto_nonspecified = pd.concat([energy_use_esto_nonspecified, annoying_road_fuels_df]).groupby(['Economy', 'Date', 'Fuel', 'Medium']).sum(numeric_only=True).reset_index()

        #spread energy_use_esto_nonspecified among all mediums for that fuel, eocnomy and date. Use the % of each energy use to the total energy use for that fuel, economy and date to do this:
        energy_use_esto['proportion_of_group'] = energy_use_esto.groupby(['Economy', 'Date', 'Fuel'])['Energy'].transform(lambda x: x/x.sum(numeric_only=True))
        #join the nonspec col on
        energy_use_esto = energy_use_esto.merge(energy_use_esto_nonspecified, on=['Economy', 'Date', 'Fuel'], how='left', suffixes=('', '_nonspec'))
        #times the proportion of group by the nonspec energy use to get the new energy use, then add that to enegry
        energy_use_esto['Energy'] = energy_use_esto['Energy'] + (energy_use_esto['proportion_of_group']*energy_use_esto['Energy_nonspec'])
        energy_use_esto = energy_use_esto.drop(columns=['Energy_nonspec', 'proportion_of_group'])
    else:
        energy_use_esto_pipeline = pd.DataFrame()

    ##############

    # #Create a new DataFrame for the new fuels.
    # new_fuels = pd.DataFrame()
    # #Get unique Economy-Date combinations.
    # economy_date_combinations = energy_use_esto[['Economy', 'Date']].drop_duplicates()
    # #Iterate through each unique combination of Economy and Date.
    # for _, row in economy_date_combinations.iterrows():
    #     economy, date = row['Economy'], row['Date']

    #     #For each fuel to ignore, create a new row with the current Economy and Date.
    #     for fuel in model_fuels_to_ignore:
    #         new_row = {'Economy': economy, 'Date': date, 'Fuel': fuel,
    #                 'Energy': 0}
    #         new_fuels = pd.concat([new_fuels, pd.DataFrame(new_row, index=[0])])

    #Concatenate the new fuels with the original DataFrame.
    # energy_use_esto = pd.concat([energy_use_esto, new_fuels])

    ##############NOW HANDLE THE OUTPUT FROM MODEL

    # #We have to adjust the non road mediums drive type to its more specific verison. So pull in model concordances and adjsut them:
    # #load model concordances with fuels
    # model_concordances_fuels = pd.read_csv('config/concordances_and_config_data/computer_generated_concordances/{}'.format(model_concordances_file_name_fuels))
    # #and based on the Fuel and medium col, replace the drive type with the new one, for non road only:
    # model_concordances_fuels = model_concordances_fuels.loc[model_concordances_fuels['Medium'] != 'road'].copy()
    # model_concordances_fuels = model_concordances_fuels[['Fuel', 'Medium', 'Drive']].drop_duplicates()
    # model_concordances_fuels.rename(columns={'Drive': 'New_drive'}, inplace=True)
    # energy_use_output = energy_use_output.merge(model_concordances_fuels, on=['Fuel', 'Medium'], how='left')
    # energy_use_output['Drive'] = energy_use_output['New_drive'].fillna(energy_use_output['Drive']).copy()
    # #drop New_drive
    # energy_use_output = energy_use_output.drop(columns=['New_drive'])

    #NOW CLEAN UP ENERGY_USE_OUTPUT
    energy_use_output_ref = energy_use_output.loc[energy_use_output['Scenario'] == 'Reference'].copy()

    energy_use_output_ref = energy_use_output_ref.drop(columns=[ 'Vehicle Type', 'Drive', 'Transport Type']).groupby(['Economy', 'Date', 'Medium', 'Fuel']).sum(numeric_only=True).reset_index()
    #GRAB DATA ONLY FOR DATES WITH WHICH WE HAVE ESTO DATA
    energy_use_output_ref = energy_use_output_ref.loc[energy_use_output_ref['Date'].isin(energy_use_esto['Date'].unique())].copy()
    #LIEKWISE FOR ESTO
    energy_use_esto = energy_use_esto.loc[energy_use_esto['Date'].isin(energy_use_output_ref['Date'].unique())].copy()
    #########################
    #NOW find the ratio between energy use in the model and energy use in the esto data. So merge the dfs and then find it

    energy_use_merged = energy_use_esto.merge(energy_use_output_ref, on=['Economy', 'Date', 'Medium','Fuel'], how='left', suffixes=('_esto', '_model'))

    #reaplce nans in  Energy_model with 0. they are nan because they arent in the model (the way the model works, it has just removed these rows, so tahts why they are nans)
    #energy_use_merged['Energy_esto'] = energy_use_merged['Energy_esto'].fillna(0)
    energy_use_merged['Energy_model'] = energy_use_merged['Energy_model'].fillna(0)
    # But , for now, if theres any nans in Energy_esto then let the uyser know sicne there shouldnt really be any there.
    if energy_use_merged['Energy_esto'].isna().sum(numeric_only=True) > 0:
        nans = energy_use_merged.loc[energy_use_merged['Energy_esto'].isna(), ['Economy', 'Date', 'Medium','Fuel']].drop_duplicates()
        raise ValueError('There are nans in energy_use_esto for the following rows: {}'.format(nans))
    #
    # if energy_use_merged['Energy'].isna().sum(numeric_only=True) > 0:
    #     nans = energy_use_merged.loc[energy_use_merged['Energy'].isna(), ['Economy', 'Date', 'Medium','Fuel']].drop_duplicates()
    #     raise ValueError('There are nans in energy_use_model for the following rows: {}'.format(nans))

    #calcaulte ratio so that ratio times energy in the model = eneerhy in the esto data. Then we will timesz this ratio by energy uise in the model to get the new energy use (and the effect of timesing by the ratio will be that the total difference for that ['Economy', 'Date', 'Medium','Fuel'] spread equally among all rows for that ['Economy', 'Date', 'Medium','Fuel') (except for supply side fuel mixing fuels, which we will handle separately, and demand side fuel mixing fuels, which we will drop as they are too ahrd to handle)
    #
    energy_use_merged['ratio'] = energy_use_merged['Energy_esto']/energy_use_merged['Energy_model']

    #where ratio becomes inf then this means that ESTO has >0 energy use and the model has 0 energy use. This is because the model didnt assume any use at that point in time. This is a semi common occuraence. So create anotehr col and call it 'addition' and put the Energy_esto value in ther to be split among its users later:
    #but first,m print what fuel, medium, economy combos have this issue:
    inf_rows = energy_use_merged.loc[energy_use_merged['ratio'] == np.inf, ['Fuel', 'Medium', 'Economy']].drop_duplicates()
    if len(inf_rows[~inf_rows.Fuel.isin(['16_05_biogasoline', '16_06_biodiesel', '16_07_bio_jet_kerosene','16_01_biogas'])]) > 0:
        print('The following fuel, medium, economy combos have inf ratio, meaning the model had 0 energy use but the esto data had >0 energy use. This is because the model didnt assume any use at that point in time. So create anotehr col and call it addition and put the Energy_esto value in ther to be split among its users later:')
        print('There are fuels other than 16_05_biogasoline and 16_06_biodiesel that have inf ratio. This is unexpected {inf_rows}. they will be set in the addition column')

    energy_use_merged['addition'] = 0
    energy_use_merged.loc[energy_use_merged['ratio'] == np.inf, 'addition'] = energy_use_merged.loc[energy_use_merged['ratio'] == np.inf, 'Energy_esto']
    #then replace the inf with 1
    energy_use_merged.loc[energy_use_merged['ratio'] == np.inf, 'ratio'] = 1

    #also where ratio is na, its because both values are 0, so just set ratio to 0
    energy_use_merged['ratio'] = energy_use_merged['ratio'].fillna(0)
    ()
    return energy_use_merged, energy_use_esto, energy_use_output_ref, energy_use_esto_pipeline,energy_use_output

#############

def adjust_energy_use_in_input_data(input_data_based_on_previous_model_run,energy_use_merged,road_model_input_wide,non_road_model_input_wide,energy_use_output,TESTING):
    breakpoint()
    #CLEAN INPUT DATA
    #get dates that match the esto data:
    input_data_new = input_data_based_on_previous_model_run.loc[input_data_based_on_previous_model_run['Date'].isin(energy_use_merged['Date'].unique())].copy()
    #Edit the Drives for mediums base don fuel, using concordances:
    #thjois wont work because this df doesnt ahve fuel col
    # #We have to adjust the non road mediums drive type to its more specific verison. So pull in model concordances and adjsut them:
    # #load model concordances with fuels
    # model_concordances_fuels = pd.read_csv('config/concordances_and_config_data/computer_generated_concordances/{}'.format(model_concordances_file_name_fuels))
    # #and based on the Fuel and medium col, replace the drive type with the new one, for non road only:
    # model_concordances_fuels = model_concordances_fuels.loc[model_concordances_fuels['Medium'] != 'road'].copy()
    # model_concordances_fuels = model_concordances_fuels[['Fuel', 'Medium', 'Drive']].drop_duplicates()
    # model_concordances_fuels.rename(columns={'Drive': 'New_drive'}, inplace=True)
    # input_data_new = input_data_new.merge(model_concordances_fuels, on=['Fuel', 'Medium'], how='left')
    # input_data_new['Drive'] = input_data_new['New_drive'].fillna(input_data_new['Drive']).copy()
    # #drop New_drive
    # input_data_new = input_data_new.drop(columns=['New_drive'])

    #TODO WHY ARE WE DOING THIS. WHY NOT MERGE TO energy_use_output?
    # #we want to merge with energy_use_merged to get the ratio, however the detailed input data which we need for its efficiency data and others doesnt contain a fuel col. BUT we can map to the same index cols in energy_use_output then merge.
    # index_cols = ['Date', 'Economy', 'Vehicle Type', 'Medium', 'Transport Type', 'Drive']
    # index_cols_df = energy_use_output.loc[energy_use_output['Scenario'] == 'Reference'].copy()
    # #drop Scenario
    # index_cols_df = index_cols_df.drop(columns=['Scenario'])
    # #and drop any drive in demand side fuel mixing as its to complicated to decrease tehir energy use like we will do. (we also did this in energy_use_output_ref, but we need to do it here again too)
    # demand_side_fuel_mixing_drives = pd.read_csv('intermediate_data\model_inputs\demand_side_fuel_mixing_COMPGEN.csv')['Drive'].unique().tolist()
    # index_cols_df = index_cols_df.loc[~index_cols_df['Drive'].isin(demand_side_fuel_mixing_drives)].copy()

    # #drop nonnecessary cols from energy_use_merged
    # if not TESTING:
    #     energy_use_merged = energy_use_merged.drop(columns=['Energy_esto', 'Energy'])

    # #now do the merge
    # index_cols_merge = energy_use_merged.merge(index_cols_df.drop(columns=['Energy']), on=['Economy', 'Date','Medium', 'Fuel'], how='left')

    # #make sure there are no duplcaites when you drop the fuel column. thi would cause unexpected results (it occurs where there are multiple fuels for the same index cols - which can be expected because things lik rail use coal, elec and diesel)
    # index_cols_merge = index_cols_merge[index_cols_merge.Date>=BASE_YEAR]
    # dupes = index_cols_merge[index_cols_merge.duplicated(subset=index_cols)]
    # if len(dupes) > 0:
    #
    #     raise ValueError('duplicates in index_cols_merge')

    # input_data_new = input_data_new.merge(index_cols_merge, on=index_cols, how='left')

    #drop Fuel
    # input_data_new = input_data_new.drop(columns=['Fuel'])
    ##################

    #merge energy_use_merged to energy_use_output using a right merge and times the ratio by the energy use in the model to get the new energy use (and the effect of timesing by the ratio will be that the total difference for that ['Economy', 'Date', 'Medium','Fuel'] spread equally among all rows for that ['Economy', 'Date', 'Medium','Fuel') (except for supply side fuel mixing fuels, which we will handle separately, and demand side fuel mixing fuels, which we will drop as they are too ahrd to handle)
    energy_use_merged_merged = energy_use_merged.merge(energy_use_output, on=['Economy', 'Date', 'Medium','Fuel'], how='right')
    #To make it a bit more simple, lets called Energy, Energy old. Then when we calcualte Energy*ratio it can be called Energy new. then rename it to energy and drop energy old before we move on:
    energy_use_merged_merged.rename(columns={'Energy': 'Energy_old'}, inplace=True)

    #first, drop any supply side fuel mixing fuels as thier energy use is handled separately
    supply_side_fuel_mixing_fuels = pd.read_csv('intermediate_data\model_inputs\supply_side_fuel_mixing_COMPGEN.csv')['New_fuel'].unique().tolist()
    energy_use_merged_merged = energy_use_merged_merged.loc[~energy_use_merged_merged['Fuel'].isin(supply_side_fuel_mixing_fuels)].copy()
    #then for any phevs sum their electricity row for each economy and date, then minus that off the bev electricity. NOTE that this only works because phevs are currently a small share of use and decreaing their enegry use in elec would probably have a minimal effect. if they were bigger we'd have to find some way to handle this properly
    phev=False#i dont think i need to do this actually. the stocks for phevs will end up half what they should be if we do it, and i think it is fine to keep them in here, as well just sum their energy use like so it is not by fuel later?
    if phev:
        demand_side_fuel_mixing_drives = pd.read_csv('intermediate_data\model_inputs\demand_side_fuel_mixing_COMPGEN.csv')['Drive'].unique().tolist()
        if set(demand_side_fuel_mixing_drives) != set(['phev_d', 'phev_g']):
            raise ValueError('demand_side_fuel_mixing_drives has changed')
        phev_elec = energy_use_merged_merged.loc[energy_use_merged_merged['Drive'].isin(['phev_d', 'phev_g'])].groupby(['Economy', 'Date', 'Scenario']).sum(numeric_only=True).reset_index()
        phev_elec['decrease'] = phev_elec['Energy_old'] - (phev_elec['Energy_old'] * phev_elec['ratio'])#we will decrease bev energy use by this much after applying the ratio to it. this will effectively change bevs to make up the missing electricity use from phevs
        phev_elec['Drive'] = 'bev'
        phev_elec['Transport Type'] = 'passenger'
        #drop electiricty rows for phev
        energy_use_merged_merged = energy_use_merged_merged.loc[~(energy_use_merged_merged['Drive'].isin(['phev_d', 'phev_g'])&energy_use_merged_merged['Fuel'].isin(['17_electricity']))].copy()

    #lastly idenifty any nas. they are probably okay bnut useful to observe for nwo: #the nas that ive seen here are to be expected, such as for bevs and phevs. Its better just to notice potential issues elswehre.
    do_this = False
    if do_this:
        nas = energy_use_merged_merged.loc[energy_use_merged_merged.isna().any(axis=1)]
        if len(nas) > 0:
            breakpoint()
            print(nas)

    #not sure about this one but where ratio is na it is because we either removed it or it wasnt in the estpo data, i think? anyway, set it to 1 and later we can work out whetehr to keep it or not
    # do_this = True
    # if do_this:
    #     input_data_new['ratio'] = input_data_new['ratio'].fillna(1)

    #do the ratio times enegry calc":
    energy_use_merged_merged['Energy_new'] = energy_use_merged_merged['Energy_old']*energy_use_merged_merged['ratio']

    #additions need to be split equally where they are made. do this using the proprtion of each energy use out of the total energy use for that fuel, medium, economy and date to do this:

    #replace energy with 0 where its na for the calcualtion
    energy_use_merged_merged['Energy_new'] = energy_use_merged_merged['Energy_new'].fillna(0)
    energy_use_merged_merged['proportion_of_group'] = energy_use_merged_merged.groupby(['Economy', 'Date', 'Medium'])['Energy_new'].transform(lambda x: x/x.sum(numeric_only=True))

    energy_use_merged_merged['addition'] = energy_use_merged_merged['addition']*energy_use_merged_merged['proportion_of_group'].replace(np.nan, 0)
    #now need to add any additions to the Energy. this is where the ratio was inf because the model had 0 energy use but the esto data had >0. so we will add the esto data energy use to the model data energy use and just times by a ratio of 1
    energy_use_merged_merged['Energy_new'] = energy_use_merged_merged['Energy_new'] + energy_use_merged_merged['addition'].replace(np.nan, 0)
    if phev:
        #now change teh bev energy use by the phev electriicty amount we calculated earlier
        energy_use_merged_merged = energy_use_merged_merged.merge(phev_elec[['Economy', 'Date', 'Scenario', 'Drive', 'Transport Type', 'decrease']], on=['Economy', 'Date', 'Scenario', 'Drive', 'Transport Type'], how='left')
        #where decrease is nan, set it to 0, then decrease it
        energy_use_merged_merged['decrease'] = energy_use_merged_merged['decrease'].fillna(0)
        #if the result will be less than 0 however, then just set it to 0
        energy_use_merged_merged['Energy_new'] = np.where(energy_use_merged_merged['Energy_new'] - energy_use_merged_merged['decrease'] < 0, 0, energy_use_merged_merged['Energy_new'] - energy_use_merged_merged['decrease'])

        #drop decrease
        energy_use_merged_merged = energy_use_merged_merged.drop(columns=['decrease'])

    #drop unneeded cols
    energy_use_merged_merged = energy_use_merged_merged.drop(columns=['ratio','addition','proportion_of_group', 'Energy_esto', 'Energy_model', 'Energy_old', 'Fuel'])
    #and sum now that we've calclate the new enegry use for each 'Economy', 'Date', 'Medium', 'Vehicle Type', 'Transport Type', 'Drive', 'Scenario'
    energy_use_merged_merged = energy_use_merged_merged.groupby(['Economy', 'Date', 'Medium', 'Vehicle Type', 'Transport Type', 'Drive', 'Scenario']).sum(numeric_only=True).reset_index()
    breakpoint()
    # #replicate the dfs for the other scenarios
    # energy_use_merged_merged_copy = energy_use_merged_merged.copy()
    # for scenario in scenario_list:
    #     energy_use_merged_merged_copy['Scenario'] = scenario
    #     energy_use_merged_merged = pd.concat([energy_use_merged_merged, energy_use_merged_merged_copy])
    #####################################
    #Now join on the efficiency, Occupancy_or_load, Mileage, inteisntiy from the detailed data so we can calcualte the new inputs for the model:
    energy_use_merged_merged = energy_use_merged_merged.merge(input_data_new[['Economy', 'Date', 'Medium', 'Vehicle Type', 'Transport Type', 'Drive', 'Scenario', 'Efficiency', 'Occupancy_or_load', 'Mileage', 'Intensity']], on=['Economy', 'Date', 'Medium', 'Vehicle Type', 'Transport Type', 'Drive', 'Scenario'], how='left')
    #split into medium = road and not, then recalcualte the other measures. for road we will calcuialte travel km and activity and stocks, wehreas for non road we will just recalcualte energy use using itnsntiy instead of effcieincy
    #road:
    input_data_new_road = energy_use_merged_merged.loc[energy_use_merged_merged['Medium'] == 'road'].copy()

    input_data_new_road['Travel_km'] = input_data_new_road['Energy_new'] * input_data_new_road['Efficiency']

    input_data_new_road['Activity'] = input_data_new_road['Travel_km'] * input_data_new_road['Occupancy_or_load']

    input_data_new_road['Stocks'] = input_data_new_road['Activity'] / (input_data_new_road['Mileage'] * input_data_new_road['Occupancy_or_load'])

    #rename Energy_new to Energy
    input_data_new_road.rename(columns={'Energy_new': 'Energy'}, inplace=True)

    #########
    # non road:
    input_data_new_non_road = energy_use_merged_merged.loc[energy_use_merged_merged['Medium'] != 'road'].copy()
    input_data_new_non_road['Activity'] = input_data_new_non_road['Energy_new'] / input_data_new_non_road['Intensity']

    input_data_new_non_road.loc[(input_data_new_non_road['Energy_new'] > 0), 'Stocks'] = 1
    input_data_new_non_road.loc[(input_data_new_non_road['Energy_new'] == 0), 'Stocks'] = 0

    #rename Energy_new to Energy
    input_data_new_non_road.rename(columns={'Energy_new': 'Energy'}, inplace=True)
    #drop cols unneeded for non road, by filtering tfor the same cols that are in non_road_model_input_wide
    input_data_new_non_road = input_data_new_non_road.loc[:, input_data_new_non_road.columns.isin(non_road_model_input_wide.columns)].copy()
    #is this right

    ###############
    #replicate the dfs for the other scenarios

    # for scenario in SCENARIOS_LIST:

    # #check for nas and stiff
    # nas = input_data_new_road.loc[input_data_new_road.isna().any(axis=1)]
    # if len(nas) > 0:
    #     #show where they are
    #     print(nas)

    # nas = input_data_new_non_road.loc[input_data_new_non_road.isna().any(axis=1)]
    # if len(nas) > 0:
    #     #show where they are
    #     print(nas)
    ##########

    #now, waht we have is meant to be the input used by run road model and non road. we nmeed ot merge it to the other inputs needed for it to be used by run_road_model.py and run_non_road_model.py.
    # missing_cols_road = [col for col in road_model_input_wide.columns if col not in input_data_new_road.columns]#['Non_road_intensity_improvement', 'Population', 'Gdp', 'Gdp_per_capita', 'Activity_growth']
    # missing_cols_non_road = [col for col in non_road_model_input_wide.columns if col not in input_data_new_non_road.columns]#['Non_road_intensity_improvement', 'Population', 'Gdp', 'Gdp_per_capita', 'Activity_growth']
    #what if non road missing cols are empty can we jsut carry on? or probably  throw an error if NOT empty
    #merge and add tehse missing cols back on usaing the original input data:
    road_all_wide = input_data_new_road.copy()
    road_all_wide = road_all_wide.merge(road_model_input_wide, on=['Date', 'Economy', 'Medium', 'Transport Type','Vehicle Type',  'Drive', 'Scenario'], how='left', suffixes=('', '_new'))
    #check what new cols there are by seeing what cols are different between input_data_new_road and road_all_wide
    new_cols = [col for col in road_all_wide.columns if col not in input_data_new_road.columns]
    #drop cols that end with _new
    road_all_wide = road_all_wide.loc[:,~road_all_wide.columns.str.endswith('_new')].copy()

    #NOW FOR NON ROAD
    non_road_all_wide = input_data_new_non_road.copy()
    non_road_all_wide = non_road_all_wide.merge(non_road_model_input_wide, on=['Date', 'Economy', 'Medium','Vehicle Type', 'Transport Type', 'Drive', 'Scenario'], how='left', suffixes=('', '_new'))
    #check what new cols there are by seeing what cols are different between input_data_new_road and non_road_model_input_wide
    new_cols = [col for col in input_data_new_non_road.columns if col not in non_road_all_wide.columns]
    #drop cols that end with _new
    non_road_all_wide = non_road_all_wide.loc[:,~non_road_all_wide.columns.str.endswith('_new')].copy()
    # #so we'll melt both the new input data and the orignal input data for road and non road, so that the measures are in one col and the values are in another. then we can merge them on the measures col and  replace values where they are new, then pivot bacc to wide format
    # #
    # input_data_new_non_road = input_data_new_non_road.melt(id_vars=['Date', 'Economy', 'Medium','Vehicle Type', 'Transport Type', 'Drive'], var_name='Measure', value_name='Value')
    # non_road_model_input = non_road_model_input_wide.melt(id_vars=['Date', 'Economy', 'Medium','Vehicle Type', 'Transport Type', 'Drive', 'Scenario'], var_name='Measure', value_name='Value')
    # #join to original input data
    # non_road_all = non_road_model_input.merge(input_data_new_non_road, on=['Date', 'Economy', 'Medium','Vehicle Type', 'Transport Type', 'Drive', 'Measure'], how='left', suffixes=('', '_new'))

    # #for the measures we havent already calcualted suing new energy, we should fillna with the old values.
    # measures_to_not_fillna = ['Activity', 'Energy']
    # non_road_all_measures_to_fillna = non_road_all.loc[~non_road_all['Measure'].isin(measures_to_not_fillna)].copy()
    # non_road_all_measures_to_fillna['Value'] = non_road_all_measures_to_fillna['Value_new'].fillna(non_road_all_measures_to_fillna['Value'])
    # non_road_other = non_road_all.loc[non_road_all['Measure'].isin(measures_to_not_fillna)].copy()
    # non_road_other['Value'] = non_road_other['Value_new']
    # non_road_all = pd.concat([non_road_all_measures_to_fillna, non_road_other]).copy()

    # #drop Value_new
    # non_road_all = non_road_all.drop(columns=['Value_new'])
    # #pivot back to wide format
    # non_road_all_wide = non_road_all.pivot_table(index=['Date', 'Economy', 'Medium', 'Transport Type', 'Drive','Vehicle Type',  'Scenario'], columns='Measure', values='Value').reset_index()
    # #
    # ########
    # #do same for road
    # input_data_new_road_long = input_data_new_road.melt(id_vars=['Date', 'Economy', 'Medium', 'Transport Type','Vehicle Type',  'Drive'], var_name='Measure', value_name='Value')
    # road_model_input = road_model_input_wide.melt(id_vars=['Date', 'Economy', 'Medium', 'Transport Type','Vehicle Type',  'Scenario', 'Drive'],  var_name='Measure', value_name='Value')
    # #join to original input data
    # road_all = road_model_input.merge(input_data_new_road_long, on=['Date', 'Economy', 'Medium', 'Transport Type','Vehicle Type', 'Drive', 'Measure'], how='left', suffixes=('', '_new'))

    # #for the measures we havent already calcualted suing new energy, we should fillna with the old values. otherwise only use the new values.
    # measures_to_not_fillna = ['Activity', 'Stocks', 'Travel_km', 'Energy']
    # road_all_measures_to_fillna = road_all.loc[~road_all['Measure'].isin(measures_to_not_fillna)].copy()
    # road_all_measures_to_fillna['Value'] = road_all_measures_to_fillna['Value_new'].fillna(road_all_measures_to_fillna['Value'])
    # road_other = road_all.loc[road_all['Measure'].isin(measures_to_not_fillna)].copy()
    # road_other['Value'] = road_other['Value_new']
    # road_all = pd.concat([road_all_measures_to_fillna, road_other]).copy()

    #drop Value_new
    # road_all = road_all.drop(columns=['Value_new'])
    #pivot back to wide format
    # road_all_wide = road_all.pivot_table(index=['Date', 'Economy', 'Medium', 'Transport Type','Vehicle Type',  'Scenario', 'Drive'], columns='Measure', values='Value').reset_index()

    ##########TESTINGOVER###############
    # #remove Activity_growth freom both road and non road as its been a bit corrupted by the above process
    # road_all_wide = road_all_wide.drop(columns=['Activity_growth'])
    # non_road_all_wide = non_road_all_wide.drop(columns=['Activity_growth'])
    #then add it back fromroad_model_input_wide amnd non_road_model_input_wide:
    # road_all_wide = road_all_wide.merge(road_model_input_wide[['Date', 'Economy', 'Medium', 'Transport Type','Vehicle Type',  'Scenario', 'Drive', 'Activity_growth']], on=['Date', 'Economy', 'Medium', 'Transport Type','Vehicle Type',  'Scenario', 'Drive'], how='left')
    # non_road_all_wide = non_road_all_wide.merge(non_road_model_input_wide[['Date', 'Economy', 'Medium', 'Transport Type','Vehicle Type',  'Scenario', 'Drive', 'Activity_growth']], on=['Date', 'Economy', 'Medium', 'Transport Type','Vehicle Type',  'Scenario', 'Drive'], how='left')

    return road_all_wide, non_road_all_wide

def adjust_data_to_match_esto(road_model_input_wide,non_road_model_input_wide,advance_base_year=True, TESTING=False, TEST_ECONOMY='19_THA'):
    energy_use_esto = format_9th_input_energy_from_esto()

    input_data_based_on_previous_model_run = pd.read_csv('output_data/model_output_detailed/NON_ROAD_DETAILED_{}'.format(model_output_file_name))
    supply_side_fuel_mixing = pd.read_csv('intermediate_data\model_inputs\supply_side_fuel_mixing_COMPGEN.csv')
    energy_use_output = pd.read_csv('output_data/model_output_with_fuels/NON_ROAD_DETAILED_{}'.format(model_output_file_name))

    if TESTING:

        #filter for only 01_AUS and year <= 2025 to make it run faster
        road_model_input_wide = road_model_input_wide[(road_model_input_wide['Economy'] == TEST_ECONOMY ) & (road_model_input_wide['Date'] <= 2025)]
        non_road_model_input_wide = non_road_model_input_wide[(non_road_model_input_wide['Economy'] == TEST_ECONOMY) & (non_road_model_input_wide['Date'] <= 2025)]

        #filter for only 01_AUS to make it run faster  and year <= 2025
        supply_side_fuel_mixing = supply_side_fuel_mixing[supply_side_fuel_mixing['Economy'] == TEST_ECONOMY].copy()
        input_data_based_on_previous_model_run = input_data_based_on_previous_model_run[input_data_based_on_previous_model_run['Economy'] == TEST_ECONOMY].copy()
        #filter for only 01_AUS to make it run faster  and year <= 2025
        energy_use_output = energy_use_output[energy_use_output['Economy'] == TEST_ECONOMY].copy()
        energy_use_esto = energy_use_esto[energy_use_esto['Economy'] == TEST_ECONOMY].copy()
        #  and year <= 2025
        supply_side_fuel_mixing = supply_side_fuel_mixing[supply_side_fuel_mixing['Date'] <= 2025].copy()
        input_data_based_on_previous_model_run = input_data_based_on_previous_model_run[input_data_based_on_previous_model_run['Date'] <= 2025].copy()

        energy_use_esto = energy_use_esto[energy_use_esto['Date'] <= 2025].copy()
        energy_use_output = energy_use_output[energy_use_output['Date'] <= 2025].copy()

    energy_use_merged, energy_use_esto, energy_use_output_ref,energy_use_esto_pipeline,energy_use_output = format_energy_use_for_rescaling(energy_use_esto, energy_use_output,spread_non_specified_and_separate_pipeline = True, remove_annoying_fuels = True, TESTING=TESTING)

    supply_side_fuel_mixing_new = adjust_supply_side_fuel_share(energy_use_esto,supply_side_fuel_mixing)

    road_all_wide, non_road_all_wide = adjust_energy_use_in_input_data(input_data_based_on_previous_model_run,energy_use_merged,road_model_input_wide,non_road_model_input_wide,energy_use_output,TESTING)

    #########
    #now do tests to check data matches expectations:
    #test that the total road enegry use matches the total energy use in the esto data:
    test_output_matches_expectations(road_all_wide, non_road_all_wide, energy_use_merged, advance_base_year=True)

    return road_all_wide, non_road_all_wide, supply_side_fuel_mixing_new

#why is ratio so high in places? maybe need to fix.
def test_output_matches_expectations(road_all_wide, non_road_all_wide, energy_use_merged, advance_base_year=True):
    #calcauklte total energy use by year and economy for both road and non road.
    #first rmeove the supply_side_fuel_mixing_fuels from esto data!
    supply_side_fuel_mixing_fuels = pd.read_csv('intermediate_data\model_inputs\supply_side_fuel_mixing_COMPGEN.csv')['New_fuel'].unique().tolist()
    energy_use_merged = energy_use_merged.loc[~energy_use_merged['Fuel'].isin(supply_side_fuel_mixing_fuels)].copy()
    ################
    road_all_wide_total_energy_use = road_all_wide.groupby(['Economy','Scenario', 'Date'])['Energy'].sum(numeric_only=True).reset_index()
    non_road_all_wide_total_energy_use = non_road_all_wide.groupby(['Economy','Scenario', 'Date'])['Energy'].sum(numeric_only=True).reset_index()
    esto_total_energy_use_non_road = energy_use_merged.loc[(energy_use_merged['Medium'] != 'road')].groupby(['Economy', 'Date'])['Energy_esto'].sum(numeric_only=True).reset_index()
    esto_total_energy_use_road = energy_use_merged.loc[(energy_use_merged['Medium'] == 'road')].groupby(['Economy', 'Date'])['Energy_esto'].sum(numeric_only=True).reset_index()
    #print the differentce between total energy in the years 2017 to 2022
    print('road energy use difference (PJ)')
    diff = road_all_wide_total_energy_use.merge(esto_total_energy_use_road, on=['Economy', 'Date'], how='left', suffixes=('', '_esto'))
    if advance_base_year:
        diff = diff.loc[(diff.Date>BASE_YEAR) & (diff.Date<=OUTLOOK_BASE_YEAR)]
    else:
        diff = diff.loc[(diff.Date>=BASE_YEAR) & (diff.Date<=OUTLOOK_BASE_YEAR)]
    print(diff['Energy'].sum(numeric_only=True) - diff['Energy_esto'].sum(numeric_only=True))

    diff2 = non_road_all_wide_total_energy_use.merge(esto_total_energy_use_non_road, on=['Economy', 'Date'], how='left', suffixes=('', '_esto'))
    if advance_base_year:
        diff2 = diff2.loc[(diff2.Date>BASE_YEAR) & (diff2.Date<=OUTLOOK_BASE_YEAR)]
    else:
        diff2 = diff2.loc[(diff2.Date>=BASE_YEAR) & (diff2.Date<=OUTLOOK_BASE_YEAR)]
    print('non road energy use difference (PJ)')
    print(diff2['Energy'].sum(numeric_only=True) - diff2['Energy_esto'].sum(numeric_only=True))
    ###################TESTING###############
    #Another test using the proportion difference between teh two:
    total_road_energy_use = road_all_wide.groupby(['Economy','Scenario', 'Date']).sum(numeric_only=True).reset_index()
    total_esto_road_energy_use = energy_use_merged.loc[(energy_use_merged['Medium'] == 'road')].groupby(['Economy', 'Date']).sum(numeric_only=True).reset_index()

    total_non_road_energy_use = non_road_all_wide.groupby(['Economy','Scenario', 'Date']).sum(numeric_only=True).reset_index()
    total_esto_non_road_energy_use = energy_use_merged.loc[(energy_use_merged['Medium'] != 'road')].groupby(['Economy', 'Date']).sum(numeric_only=True).reset_index()

    diff_road = total_road_energy_use.merge(total_esto_road_energy_use, on=['Economy', 'Date'], how='left', suffixes=('', '_esto'))
    #filter for dates after base year
    if advance_base_year:
        diff_road = diff_road.loc[diff_road.Date>=OUTLOOK_BASE_YEAR]
    else:
        diff_road = diff_road.loc[diff_road.Date>=BASE_YEAR]
    diff_road_proportion = sum(diff_road['Energy'].dropna()) / sum(diff_road['Energy_esto'].dropna())

    diff_non_road = total_non_road_energy_use.merge(total_esto_non_road_energy_use, on=['Economy', 'Date'], how='left', suffixes=('', '_esto'))
    if advance_base_year:
        diff_non_road = diff_non_road.loc[diff_non_road.Date>=OUTLOOK_BASE_YEAR]
    else:
        diff_non_road = diff_non_road.loc[diff_non_road.Date>=BASE_YEAR]
    diff_non_road_proportion = sum(diff_non_road['Energy'].dropna()) / sum(diff_non_road['Energy_esto'].dropna())
    if diff_road_proportion > 1.01 or diff_road_proportion < 0.99:
        breakpoint()
        #saev output to csv
        diff_road.to_csv('intermediate_data/errors/ajust_data_to_match_esto_road_energy_use_diff.csv')
        raise ValueError('road energy use does not match esto')
    if diff_non_road_proportion > 1.01 or diff_non_road_proportion < 0.99:
        breakpoint()
        diff_non_road.to_csv('intermediate_data/errors/ajust_data_to_match_esto_non_road_energy_use_diff.csv')
        raise ValueError('non road energy use does not match esto')
    ##########TESTING###############

#%%
def format_9th_input_energy_from_esto():
    #take in data from the EBT system of 9th and format it so that it can be used to create the energy data to whcih the model will be rescaled:
    #load the 9th data
    date_id = utility_functions.get_latest_date_for_data_file('input_data/9th_model_inputs', 'model_df_wide_')
    energy_use_esto = pd.read_csv(f'input_data/9th_model_inputs/model_df_wide_{date_id}.csv')
    #TEMP replace 15_PHL with 15_RP and 17_SGP with 17_SIN in the eocnomy col. make sure to let user know they can klet hyguga know:
    if len(energy_use_esto.loc[energy_use_esto['economy'].isin(['15_PHL', '17_SGP'])]) > 0:
        print('########################\n\n there are some economies in the esto data that are not in the model data. we will replace 15_PHL with 15_RP and 17_SGP with 17_SIN. AKA TELL HYUGA wats up \n\n########################')
    energy_use_esto['economy'] = energy_use_esto['economy'].replace({'15_PHL': '15_RP', '17_SGP': '17_SIN'})

    #reverse the mappings:
    medium_mapping_reverse = {v: k for k, v in medium_mapping.items()}

    #now format it so we only have the daata we need:
    # cols:'scenarios', 'economy', 'sectors', 'sub1sectors', 'sub2sectors',
    #    'sub3sectors', 'sub4sectors', 'fuels', 'subfuels', '1980'...
    #first filter so teh sector is transport:
    energy_use_esto = energy_use_esto.loc[energy_use_esto['sectors'] == '15_transport_sector'].copy()
    #and fiulter for ref scenario:
    energy_use_esto = energy_use_esto.loc[energy_use_esto['scenarios'] == 'reference'].copy()
    #and drop aggregate fuels:
    aggregate_cols = ['19_total', '20_total_renewables', '21_modern_renewables']
    energy_use_esto = energy_use_esto.loc[~energy_use_esto['fuels'].isin(aggregate_cols)].copy()
    #then do the mappings:
    #map the subfuel to the fuel
    #now map the subfuels to the subfuels in the esto data
    energy_use_esto['Fuel'] = energy_use_esto['subfuels'].map(temp_esto_subfuels_to_new_subfuels_mapping)
    #where subfuel is x then set Fuel to the value in fuels column:
    energy_use_esto.loc[energy_use_esto['subfuels'] == 'x', 'Fuel'] = energy_use_esto['fuels'].map(x_subfuel_mappings)
    if len(energy_use_esto.loc[energy_use_esto['Fuel'].isna()]) > 0:

        # drop anyrows where the fuels column is 06_crude_oil_and_ngl, as we are going to remove that column on the input data side anyway (i.e. tell hyuga to drop it)
        nas = energy_use_esto.loc[energy_use_esto['Fuel'].isna()].loc[~energy_use_esto['fuels'].isin(['06_crude_oil_and_ngl'])]
        if len(nas) > 0:
            raise ValueError('there are nans in Fuel because there was an x in subfuel and the fuel was not in the x_subfuel_mapping, {}'.format(energy_use_esto.loc[energy_use_esto['Fuel'].isna(), 'fuels'].unique()))
        else:
            #write big warnign just to rmeind you to remind hyuga to drop 06_crude_oil_and_ngl!
            print('##########################\n\n there are nans in Fuel because there was an x in subfuel and the fuel was not in the x_subfuel_mapping, but they are all 06_crude_oil_and_ngl, so we will drop that column on the input data side. AKA TELL HYUGA TO DROP 06_crude_oil_and_ngl\n\n##########################')

    #map the medium to the sub1sector then drop the fuel and sectors cols since weve dfone all the mapping we can:
    energy_use_esto['Medium'] = energy_use_esto['sub1sectors'].map(medium_mapping_reverse)
    energy_use_esto = energy_use_esto.drop(columns=['sub1sectors', 'sub2sectors', 'sub3sectors', 'sub4sectors', 'sectors', 'fuels','subfuels', 'scenarios'])

    #then sum up the energy use by scenarios, economy, medium and subfuel:
    energy_use_esto = energy_use_esto.groupby(['economy', 'Medium', 'Fuel']).sum(numeric_only=True).reset_index()
    #melt so that the years are in one col and the energy use is in another:
    energy_use_esto = energy_use_esto.melt(id_vars=['economy', 'Medium', 'Fuel'], var_name='Date', value_name='Energy_esto')

    #rename economy to Economy,
    energy_use_esto.rename(columns={'economy': 'Economy'}, inplace=True)

    # #drop any 0's or nas:
    energy_use_esto = energy_use_esto.loc[energy_use_esto['Energy_esto'] > 0].copy()

    #drop data that is less than the BASE_YEAR and more than OUTLOOK_BASE_YEAR
    energy_use_esto['Date'] = energy_use_esto['Date'].apply(lambda x: x[:4])
    energy_use_esto['Date'] = energy_use_esto['Date'].astype(int)
    energy_use_esto = energy_use_esto.loc[energy_use_esto['Date'] >= BASE_YEAR].copy()
    energy_use_esto = energy_use_esto.loc[energy_use_esto['Date'] <= OUTLOOK_BASE_YEAR].copy()

    #drop '22_SEA', '23_NEA', '23b_ONEA', '24_OAM','24b_OOAM', '25_OCE', 'APEC' Economys
    energy_use_esto = energy_use_esto.loc[~energy_use_esto['Economy'].isin(['22_SEA', '23_NEA', '23b_ONEA', '24_OAM','24b_OOAM', '25_OCE', 'APEC'])].copy()

    #lastly, using the concordances, we will identify any fuel/medium combinations that arent in either and notify the user. for ones that are dealt with in the dicitonary below, do that, for the others, throw an error:
    missing_fuels_and_mediums_to_new_fuels_and_mediums = {
        'road':{
            '07_x_other_petroleum_products': ('07_x_other_petroleum_products', 'nonspecified'),
            '16_09_other_sources':
            ('16_09_other_sources', 'nonspecified'),
            '07_02_aviation_gasoline': ('07_02_aviation_gasoline', 'nonspecified'),
            '07_06_kerosene': ('07_06_kerosene', 'nonspecified'),
            '07_08_fuel_oil': ('07_08_fuel_oil', 'nonspecified'),
        },
        'rail':{
            '07_x_other_petroleum_products': ('07_x_other_petroleum_products', 'nonspecified'),
            '16_09_other_sources':
            ('16_09_other_sources', 'nonspecified'),
        },
        'air':{
            '07_x_other_petroleum_products': ('07_x_other_petroleum_products', 'nonspecified')
        },
        'ship':{
            '16_09_other_sources':
            ('16_09_other_sources', 'nonspecified')
    }}

    #dso mapping now ewith the new fuels and mediums:
    for medium, fuels in missing_fuels_and_mediums_to_new_fuels_and_mediums.items():
        for fuel, new_fuel_and_medium in fuels.items():
            energy_use_esto.loc[(energy_use_esto['Medium'] == medium) & (energy_use_esto['Fuel'] == fuel), 'Fuel'] = new_fuel_and_medium[0]
            energy_use_esto.loc[(energy_use_esto['Medium'] == medium) & (energy_use_esto['Fuel'] == fuel), 'Medium'] = new_fuel_and_medium[1]

    concordances_fuels = pd.read_csv('config/concordances_and_config_data/computer_generated_concordances/{}'.format(model_concordances_file_name_fuels))
    concordances_fuels = concordances_fuels[['Fuel', 'Medium']].drop_duplicates()
    energy_use_esto_fuel_medium = energy_use_esto[['Fuel', 'Medium']].drop_duplicates()
    #drop nonspecified and pipeline from energy_use_esto_fuel_medium
    energy_use_esto_fuel_medium = energy_use_esto_fuel_medium.loc[~energy_use_esto_fuel_medium['Medium'].isin(['nonspecified', 'pipeline'])].copy()

    #do an outer join and then identify any nans:
    outer_join = concordances_fuels.merge(energy_use_esto_fuel_medium, on=['Fuel', 'Medium'], how='outer', indicator=True)

    #TODO MAKE THIS ACTIVIE ONCE WE HAVE THE CONCORDANCES FOR THE NEW FUELS AND MEDIUMS
    # #where merge is 'left_only' then throw an error. this is wherte a fuel is used in tthe model but isnt in esto. for now we will add these as 0 rows to the data but let the user now with a very visible message!
    left_only = outer_join.loc[outer_join['_merge'] == 'left_only']
    # ignored_fuels_and_mediums = ['07_x_other_petroleum_products', '16_09_other_sources', '07_02_aviation_gasoline', '07_06_kerosene', '07_08_fuel_oil']
    if len(left_only) > 0:
        # energy_use_esto_new = energy_use_esto.copy()
        #for each row in left_only, add a row to energy_use_esto_new with the fuel and medium and energy set to 0, for every economy and date:
        for index, row in left_only.iterrows():
            for economy in energy_use_esto['Economy'].unique():
                for date in energy_use_esto['Date'].unique():
                    energy_use_esto_new_df_row = pd.DataFrame({'Economy': [economy], 'Date': [date], 'Medium': [row['Medium']], 'Fuel': [row['Fuel']], 'Energy_esto': [0]})
                    #concat to energy_use_esto
                    energy_use_esto = pd.concat([energy_use_esto, energy_use_esto_new_df_row])
            print('###############################\n')
            print('there is a fuel/medium combination in the model that is not in the esto data. These should at least have values = to 0. it is {} and {}'.format(row['Fuel'], row['Medium']))

    #     raise ValueError('there are some fuel/medium combinations in energy_use_esto that are not in the concordances_fuels {}'.format(left_only))
    #where merge is right only, this is where a fuel is used in esto but not asccoiunted for in the model. The suer will have to deal with it manually:
    # right_only = outer_join.loc[outer_join['_merge'] == 'right_only']
    # if len(right_only) > 0:
    #     breakpoint()
    #     raise ValueError('there are some fuel/medium combinations in concordances_fuels that are not in the energy_use_esto {}'.format(right_only))

    #and now drop pipeline and nonspecified from energy_use_esto:
    energy_use_esto = energy_use_esto.loc[~energy_use_esto['Medium'].isin(['nonspecified', 'pipeline'])].copy()
    energy_use_esto = energy_use_esto.groupby(['Economy', 'Medium','Date', 'Fuel']).sum(numeric_only=True).reset_index()
    #reame Energy_esto to Energy:
    energy_use_esto.rename(columns={'Energy_esto': 'Energy'}, inplace=True)
    return energy_use_esto

#%%


#! /usr/bin/env python3

### Exercises: Level 1

# 1. Declare an empty list
lst = []
print(lst)

# 2. Declare a list with more than 5 items
lst = ['item1', 'item2', 'item3', 'item4', 'item5', 'item6', 'item7']
print(lst)

# 3. Find the length of your list
print('Length: \t', len(lst))

# 4. Get the first item, the middle item and the last item of the list
print('First: \t\t',lst[0])
print('Middle: \t', lst[len(lst)//2])
print('Last: \t\t', lst[-1])

# 5. Declare a list called mixed_data_types, put your(name, age, height, marital status, address)
mixed_data_types = ['Sam', 25, '6ft 3in', 'Single', {'address':'6735 Golden Briar'}]

# 6. Declare a list variable named it_companies and assign initial values Facebook, Google, Microsoft, Apple, IBM, Oracle and Amazon.
it_companies = ['Facebook', 'Google', 'Microsoft', 'Apple', 'IBM', 'Oracle', 'Amazon']

# 7. Print the list using _print()_
print(it_companies)

# 8. Print the number of companies in the list
print('Company Count: \t', len(it_companies))

# 9. Print the first, middle and last company
print('First Company: \t', it_companies[0])
print('Middle Company: ', it_companies[len(it_companies)//2])
print('Last Company: \t', it_companies[-1])

# 10. Print the list after modifying one of the companies
it_companies[4] = 'Twitter'
print('Changed List: \t', it_companies)

# 11. Add an IT company to it_companies
it_companies.append('Coalfire')
print('Appended list: \t', it_companies)

# 12. Insert an IT company in the middle of the companies list
it_companies.insert(3, 'FireEye')
print('Inserted list: \t', it_companies)

# 13. Change one of the it_companies names to uppercase (IBM excluded!)
it_companies[1] = it_companies[1].upper()
print('Upper\'d List: \t', it_companies)

# 14. Join the it_companies with a string '#; '
print('Joined List: \t','#; '.join(it_companies))

# 15. Check if a certain company exists in the it_companies list.
print('GOOGLE Exist: \t', 'GOOGLE' in it_companies)

# 16. Sort the list using sort() method
it_companies.sort()
print('Sorted List: \t', it_companies)

# 17. Reverse the list in descending order using reverse() method
it_companies.reverse()
print('Reversed List: \t', it_companies)

# 18. Slice out the first 3 companies from the list
print('Sliced List: \t', it_companies[:3])

# 19. Slice out the last 3 companies from the list
print('Sliced List2: \t', it_companies[-3:])

# 20. Slice out the middle IT company or companies from the list
print('Middle Slice: \t', it_companies[len(it_companies)//2])

# 21. Remove the first IT company from the list
it_companies.pop(0)
print('First Remove: \t', it_companies)

# 22. Remove the middle IT company or companies from the list
it_companies.pop(len(it_companies)//2)
print('Middle Remove: \t', it_companies)

# 23. Remove the last IT company from the list
it_companies.pop(-1)
print('Last Remove: \t', it_companies)

# 24. Remove all IT companies from the list
it_companies.clear()
print('Cleared List: \t', it_companies)

# 25. Destroy the IT companies list
del it_companies

# 26. Join the following lists:

front_end = ['HTML', 'CSS', 'JS', 'React', 'Redux']
back_end = ['Node','Express', 'MongoDB']
joined = front_end + back_end

print('Joined Lists: \t', joined)

# 27. After joining the lists in question 26. Copy the joined list and assign it to a variable full_stack. Then insert Python and SQL after Redux.
full_stack = joined.copy()
print('Full Stack: \t', full_stack)

# ### Exercises: Level 2

# 1. The following is a list of 10 students ages:

ages = [19, 22, 19, 24, 20, 25, 26, 24, 25, 24]

# - Sort the list and find the min and max age
ages.sort()
print('Min Age: \t', ages[0])
print('Max Age: \t', ages[-1])

# - Add the min age and the max age again to the list
ages.insert(0,ages[0])
ages.insert(-1,ages[-1])
print('Ages List: \t', ages)

# - Find the median age (one middle item or two middle items divided by two)
age_median = len(ages) // 2
print('Median Age: \t', ages[age_median])

# - Find the average age (sum of all items divided by their number )
age_sum = 0

for ele in range(0, len(ages)):
    age_sum += ages[ele]

age_avg = age_sum / len(ages)

print('Average Age: \t', age_avg)

# - Find the range of the ages (max minus min)
age_rang = ages[-1] - ages[0]
print('Range of Age: \t', age_rang)

# - Compare the value of (min - average) and (max - average), use _abs()_ method
print('Abs of Min: \t', abs(ages[0] - age_avg))
print('Abs of Max: \t', abs(ages[-1] - age_avg))

# 1. Find the middle country(ies) in the [countries list](https://github.com/Asabeneh/30-Days-Of-Python/tree/master/data/countries.py)
countries = [
  'Afghanistan',
  'Albania',
  'Algeria',
  'Andorra',
  'Angola',
  'Antigua and Barbuda',
  'Argentina',
  'Armenia',
  'Australia',
  'Austria',
  'Azerbaijan',
  'Bahamas',
  'Bahrain',
  'Bangladesh',
  'Barbados',
  'Belarus',
  'Belgium',
  'Belize',
  'Benin',
  'Bhutan',
  'Bolivia',
  'Bosnia and Herzegovina',
  'Botswana',
  'Brazil',
  'Brunei',
  'Bulgaria',
  'Burkina Faso',
  'Burundi',
  'Cambodia',
  'Cameroon',
  'Canada',
  'Cape Verde',
  'Central African Republic',
  'Chad',
  'Chile',
  'China',
  'Colombi',
  'Comoros',
  'Congo (Brazzaville)',
  'Congo',
  'Costa Rica',
  "Cote d'Ivoire",
  'Croatia',
  'Cuba',
  'Cyprus',
  'Czech Republic',
  'Denmark',
  'Djibouti',
  'Dominica',
  'Dominican Republic',
  'East Timor (Timor Timur)',
  'Ecuador',
  'Egypt',
  'El Salvador',
  'Equatorial Guinea',
  'Eritrea',
  'Estonia',
  'Ethiopia',
  'Fiji',
  'Finland',
  'France',
  'Gabon',
  'Gambia, The',
  'Georgia',
  'Germany',
  'Ghana',
  'Greece',
  'Grenada',
  'Guatemala',
  'Guinea',
  'Guinea-Bissau',
  'Guyana',
  'Haiti',
  'Honduras',
  'Hungary',
  'Iceland',
  'India',
  'Indonesia',
  'Iran',
  'Iraq',
  'Ireland',
  'Israel',
  'Italy',
  'Jamaica',
  'Japan',
  'Jordan',
  'Kazakhstan',
  'Kenya',
  'Kiribati',
  'Korea, North',
  'Korea, South',
  'Kuwait',
  'Kyrgyzstan',
  'Laos',
  'Latvia',
  'Lebanon',
  'Lesotho',
  'Liberia',
  'Libya',
  'Liechtenstein',
  'Lithuania',
  'Luxembourg',
  'Macedonia',
  'Madagascar',
  'Malawi',
  'Malaysia',
  'Maldives',
  'Mali',
  'Malta',
  'Marshall Islands',
  'Mauritania',
  'Mauritius',
  'Mexico',
  'Micronesia',
  'Moldova',
  'Monaco',
  'Mongolia',
  'Morocco',
  'Mozambique',
  'Myanmar',
  'Namibia',
  'Nauru',
  'Nepal',
  'Netherlands',
  'New Zealand',
  'Nicaragua',
  'Niger',
  'Nigeria',
  'Norway',
  'Oman',
  'Pakistan',
  'Palau',
  'Panama',
  'Papua New Guinea',
  'Paraguay',
  'Peru',
  'Philippines',
  'Poland',
  'Portugal',
  'Qatar',
  'Romania',
  'Russia',
  'Rwanda',
  'Saint Kitts and Nevis',
  'Saint Lucia',
  'Saint Vincent',
  'Samoa',
  'San Marino',
  'Sao Tome and Principe',
  'Saudi Arabia',
  'Senegal',
  'Serbia and Montenegro',
  'Seychelles',
  'Sierra Leone',
  'Singapore',
  'Slovakia',
  'Slovenia',
  'Solomon Islands',
  'Somalia',
  'South Africa',
  'Spain',
  'Sri Lanka',
  'Sudan',
  'Suriname',
  'Swaziland',
  'Sweden',
  'Switzerland',
  'Syria',
  'Taiwan',
  'Tajikistan',
  'Tanzania',
  'Thailand',
  'Togo',
  'Tonga',
  'Trinidad and Tobago',
  'Tunisia',
  'Turkey',
  'Turkmenistan',
  'Tuvalu',
  'Uganda',
  'Ukraine',
  'United Arab Emirates',
  'United Kingdom',
  'United States',
  'Uruguay',
  'Uzbekistan',
  'Vanuatu',
  'Vatican City',
  'Venezuela',
  'Vietnam',
  'Yemen',
  'Zambia',
  'Zimbabwe',
]

def middleList(input_list):
    middle = float(len(input_list)/2)
    if middle %2 != 0: # If length of list is odd, print both sides
        return input_list[int(middle)]
    else:
        return input_list[int(middle) - 1], input_list[int(middle)]

print('Middle List: \t', middleList(countries))

# 1. Divide the countries list into two equal lists if it is even if not one more country for the first half.
def splitList(input_list):
    mid_ele = middleList(input_list) # Git the mid points

    if type(mid_ele) == tuple:
        # First half of list, from 0th ele to index of mid_ele + 1 --- adding 1 to include mid_ele
        first_half = input_list[:input_list.index(mid_ele[0])+1]
        second_half = input_list[input_list.index(mid_ele[1]):] # Start from second mid_ele to ned

        return first_half, second_half

    else:
        #Same reasoning as above, +1 to include element
        first_half = input_list[:input_list.index(mid_ele) + 1]
        second_half = input_list[input_list.index(mid_ele)+1:] # Adding 1 here to not include mid_ele

        return first_half, second_half

#Pass Large Countries list to splitList Function
first_half, second_half = splitList(countries)
print('Len of First: \t', len(first_half))
print('Len of Second: \t', len(second_half))

# 1. Unpack the first three countries and the rest as scandic countries.
countries = ['China', 'Russia', 'USA', 'Finland', 'Sweden', 'Norway', 'Denmark']

CHI, RUS, USA, *scadic = countries

print('Unpacked CHI: \t', CHI)
print('Unpacked RUS: \t', RUS)
print('Unpacked USA: \t', USA)
print('Unpacked Other: ', scadic)


import sys
import time
import subprocess
import signal
import pexpect
import re

from functools import partial
from os.path import exists

from pylib.Tracker import Tracker
from guilib.LHWorker import LHWorker
from guilib.TrackerWindow import TrackerWindow
from guilib.QToaster import QToaster
from pexpect import popen_spawn

from PyQt5 import QtCore, QtWidgets
from PyQt5.QtCore import Qt
from PyQt5.QtGui import *
from PyQt5.QtWidgets import (
    QApplication, QMainWindow, QPushButton, QLabel, QMenuBar, QMenu, QAction, QVBoxLayout,
    QHBoxLayout, QStackedLayout, QWidget, QTableWidget, QTableWidgetItem, QFrame
)

from guilib.MainWindowStyle import main_window_style

lighthouse_connected = False

class MainWindow(QMainWindow):
    def __init__(self, parent=None, appconfig=None):
        super().__init__()
        self.appConfig = appconfig
        self.startUI()

    def load_stylesheet(self):
        self.setStyleSheet(main_window_style)

    def startUI(self):
        self.lhworker = LHWorker(None, self.appConfig)
        self.lhworker.console_open.connect(self.update_connect_ui_on)
        self.lhworker.console_close.connect(self.update_connect_ui_off)
        self.lhworker.devinfo_ready.connect(self.update_devtable_ui)

        #if self.style_path != None:
            #with open(self.style_path, "r") as spfh:
                #self.setStyleSheet(spfh.read())

        self.load_stylesheet()
        self.setObjectName("MainWindow")
        self.setWindowTitle("Tundra Debugger")
        self.resize(800, 800)
        self.create_menu_bar()

        self.identify_buttons = []
        self.select_buttons = []
        self.connected_serial = None

        page_layout = QVBoxLayout()
        status_label_layout = QHBoxLayout()
        top_label_layout = QHBoxLayout()
        top_button_layout = QHBoxLayout()
        middle_table_layout = QHBoxLayout()
        stack_layout = QStackedLayout()

        page_layout.addLayout(top_label_layout)
        page_layout.addLayout(top_button_layout)
        page_layout.addLayout(status_label_layout)
        page_layout.addLayout(middle_table_layout)
        page_layout.addLayout(stack_layout)

        self.status_label = QLabel("Not Connected")
        self.status_label.setObjectName("StatusLabel")
        status_label_layout.addWidget(self.status_label)
        status_label_layout.setAlignment(Qt.AlignHCenter | Qt.AlignTop)

        top_label = QLabel("Tundra Tracker Debugging Tool")
        top_label.setObjectName("TitleLabel")
        top_label_layout.addWidget(top_label)
        top_label_layout.setAlignment(Qt.AlignHCenter | Qt.AlignTop)

        self.button = QPushButton("Scan For Devices")
        self.button.setCheckable(True)
        self.button.setFixedWidth(240)
        self.button.setFixedHeight(30)
        self.button.clicked.connect(self.scan_button_pressed)
        top_button_layout.addWidget(self.button)

        self.tableWidget = QTableWidget()
        self.tableWidget.horizontalHeader().setSectionResizeMode(QtWidgets.QHeaderView.Stretch)
        self.tableWidget.setSizeAdjustPolicy(QtWidgets.QAbstractScrollArea.AdjustToContents)
        self.tableWidget.setObjectName("DeviceTable")
        self.tableWidget.verticalHeader().setVisible(False)
        self.tableWidget.horizontalHeader().setVisible(False)
        self.tableWidget.setColumnCount(3)
        #self.tableWidget.setColumnWidth(0, 150)
        #self.tableWidget.setColumnWidth(1, 220)
        #self.tableWidget.setColumnWidth(2, 230)
        self.tableWidget.setLineWidth(0)
        self.tableWidget.setFrameStyle(QFrame.Box | QFrame.Plain)
        self.tableWidget.setFixedWidth(800)
        self.tableWidget.setFixedHeight(400)
        middle_table_layout.addWidget(self.tableWidget, alignment=Qt.AlignCenter)

        self.widget = QWidget()
        self.widget.setLayout(page_layout)
        self.setCentralWidget(self.widget)

    def format_table_cell(self, content):
        result = QTableWidgetItem(content)
        result.setTextAlignment(Qt.AlignHCenter)
        return result

    def update_devtable_ui(self, devices):
        self.devices = devices
        numdevs = len(devices)

        self.tableWidget.setRowCount(numdevs)
        self.identify_buttons = []
        self.select_buttons = []
        rowct = 0
        for device in devices:
            cell0 = self.format_table_cell(device.serial)
            self.tableWidget.setItem(rowct, 0, cell0)
            cell1 = self.format_table_cell(device.devclass)
            self.tableWidget.setItem(rowct, 1, cell1)

            conbutton = QPushButton("Connect")
            conbutton.setObjectName(device.serial)
            conbutton.setCheckable(True)
            conbutton.clicked.connect(partial(self.connect_button_pressed, device.serial))
            self.identify_buttons.append(conbutton)
            self.tableWidget.setCellWidget(rowct, 2, conbutton)
            rowct = rowct + 1

        self.tableWidget.resizeColumnsToContents()

    def update_connect_ui_on(self):
        self.status_label.setText("Connected.")
        corner = QtCore.Qt.BottomRightCorner
        QToaster.showMessage(self, "Connected to lighthouse console service.", corner=corner, desktop=False)

    def update_connect_ui_off(self):
        self.status_label.setText("Not Connected.")

    def create_menu_bar(self):
        menu_bar = self.menuBar()
        newAct = QAction('&New', self)
        openAct = QAction('&Open', self)
        saveAct = QAction('&Save', self)
        exitAct = QAction('&Exit', self)
        exitAct.triggered.connect(self.exitAction)

        copyAct = QAction('&Copy', self)
        pasteAct = QAction('&Paste', self)
        helpContentAct = QAction('&Help', self)
        aboutAct = QAction('&About', self)

        file_menu = menu_bar.addMenu('&File')
        file_menu.addAction(newAct)
        file_menu.addAction(openAct)
        file_menu.addAction(saveAct)
        file_menu.addAction(exitAct)

        edit_menu = menu_bar.addMenu('&Edit')
        edit_menu.addAction(copyAct)
        edit_menu.addAction(pasteAct)

        help_menu = menu_bar.addMenu('&Help')
        help_menu.addAction(helpContentAct)
        help_menu.addSeparator()
        help_menu.addAction(aboutAct)

    def newAction(self):
        print("New Action selected")

    def openAction(self):
        print("Open Action selected")

    def saveAction(self):
        print("Save Action selected")

    def exitAction(self):
        print("Exit Action selected")
        self.lhworker.close_console()
        sys.exit(0)

    def copyAction(self):
        print("Copy Action selected")

    def pasteAction(self):
        print("Paste Action selected")

    def helpContentAction(self):
        print("Help Content Action selected")

    def aboutAction(self):
        print("New Action selected")

    def id_button_pressed(self, serial):
        self.lhworker.identify_device(serial)

    def connect_button_pressed(self, serial):
        sending_button = self.sender()
        self.lhworker.connect_device(serial)
        self.connected_serial = serial
        self.device_window = TrackerWindow(self, serial, self.lhworker, self.appConfig)
        self.device_window.show()
        sending_button.setDefault(True)
        self.device_window.serial = serial;

    def scan_button_pressed(self):
        self.lhworker.start()
        #self.lhworker.get_devinfo()


import pandas as pd
#import matplotlib.pyplot as plt
import numpy as np

#df = pd.read_csv('data.csv')
#print(df.info())

#df = pd.read_csv('dirtydata.csv')
#new_df = df.dropna()
#print(new_df.to_string())

#df = pd.read_csv('data1.csv')
#print(df.corr())
#df.plot( kind = 'scatter', x= 'Duration', y = 'Calories')
#plt.show()
#print(df.to_string())
#
#x = df["Calories"].mean()
#df["Calories"].fillna(x, inplace = True)
#df.fillna(130, inplace=True)
#df.dropna(inplace = True)
#print(df.to_string())
#x = "2020/12/22 "

#df["Date"] = pd.to_datetime(df['Date'])
#df.dropna(subset = ['Date'], inplace = True)
#df.loc[7, 'Duration'] = 45
#print(df.to_string())
#for x in df.index:
#    if df.loc[x, "Duration"] > 120:
#        df.loc[x, "Duration"] =120

#for x in df.index:
#    if df.loc[x, "Duration"] > 120:
#        df.drop(x, inplace = True)
#print(df.to_string())
#print(df.to_string())
#print(df.duplicated())

#df.drop_duplicates(inplace = True)
#print(df.duplicated())

#df.corr()

#d = {'Name':pd.Series(['Tom','James','Ricky','Vin','Steve','Smith','Jack',
#   'Lee','David','Gasper','Betina','Andres']),
#   'Age':pd.Series([25,26,25,23,30,29,23,34,40,30,51,46]),
#   'Rating':pd.Series([4.23,3.24,3.98,2.56,3.20,4.6,3.8,3.78,2.98,4.80,4.10,3.65])
#}

#df = pd.DataFrame(d)
#print(df)

#def adder(ele1,ele2):
#   return ele1+ele2

#df = pd.DataFrame(np.random.randn(5,3),columns=['col1','col2','col3'])
#df.pipe(adder,2)
#print(df.apply(np.mean))
s1 = pd.Series([28220804648, 16179776932, 21229909340, 16126522783, 14666227351, 11172062661, 9501221177, 13778234614, 13194846235])
s2 = pd.Series([2440.352295, 2455.935059, 2468.030273, 2423.001221, 2547.092041, 2597.084717, 2603.466553, 2688.278809, 2792.117188])
df = {'Volume': s1, 'close': s2}
d = pd.DataFrame(df)
d['divide'] = d['Volume']/d['close']
d.pop('divide')
print(d)


class Solution:
    def convertToTitle(self, columnNumber: int) -> str:
        res = []

        while columnNumber > 0:
            digit = columnNumber % 26
            columnNumber = columnNumber // 26
            if digit == 0:
                digit = 26
                columnNumber -= 1
            string_val   = chr(digit+ord('A')-1)
            res.append(string_val)

        return "".join(res[::-1])


import json, requests
from wagtail.admin.edit_handlers import FieldPanel
from wagtail.core.models import Page, Orderable
from django.db import models
from django.utils.translation import ugettext_lazy as _
from django.core.cache import cache
from django.conf import settings
import django.utils.dateparse as dateparse
from datetime import datetime, time
import pytz
from enumfields import Enum
from enumfields.fields import EnumIntegerField

class DataSources(Enum):
    FACEBOOK = 0
    LINKEDEVENTS = 1

class EventsIndexPage(Page):
    type = _('Events')
    data_source = EnumIntegerField(DataSources, verbose_name=_('Event data source'), default=DataSources.LINKEDEVENTS)
    facebook_page_id = models.CharField(default='1415745085336451', max_length=200)
    linkedevents_params = models.CharField(default='?keyword=yso:p8692,yso:p26655', max_length=200)

    content_panels = Page.content_panels + [
        FieldPanel('data_source'),
        FieldPanel('facebook_page_id'),
        FieldPanel('linkedevents_params'),
     ]

    urls = {DataSources.FACEBOOK: "graph.facebook.com/v2.5/",
            DataSources.LINKEDEVENTS: "api.hel.fi/linkedevents/v1/"}

    def _facebook_events(self):
        if not hasattr(settings, 'FACEBOOK_APP_ID') or not hasattr(settings, 'FACEBOOK_APP_SECRET'):
            return []
        events = cache.get('facebook')
        if events:
            return events
        events = []
        # facebook feed returns events latest first
        url = 'https://{}{}?fields=feed{{link,message,object_id}}&access_token={}|{}'.format(
            self.urls[self.data_source],
            self.facebook_page_id,
            str(settings.FACEBOOK_APP_ID),
            settings.FACEBOOK_APP_SECRET)
        feed = requests.get(url).json()['feed']['data']

        # filter the events from the feed

        for item in feed:
            if 'link' in item:
                if 'https://www.facebook.com/events/' in str(item['link']):
                    events.append(item)

        # fetch details for the events

        event_ids = ','.join([event['object_id'] for event in events])
        details = []
        url = 'https://{}?ids={}&fields=description,cover,end_time,name,start_time,id,picture,place&access_token={}|{}'.format(
            self.urls[self.data_source],
            event_ids,
            str(settings.FACEBOOK_APP_ID),
            settings.FACEBOOK_APP_SECRET)
        details = requests.get(url).json()
        for event in events:
            event['details'] = details[event['object_id']]
        cache.add('facebook', events, 3600)
        return events

    def _linked_events(self):
        events = cache.get('linkedevents')
        if events:
            return events
        # the methods are assumed to return events latest first
        url = 'https://{}event/{}&include=location&sort=-end_time&page_size=100'.format(
            self.urls[self.data_source],
            self.linkedevents_params)
        event_list = requests.get(url).json()
        events = event_list.get('data')

        # we will be happy with 100 latest events for now
        cache.add('linkedevents', events, 3600)
        return events

    _event_methods = {DataSources.FACEBOOK: _facebook_events,
                     DataSources.LINKEDEVENTS: _linked_events}

    def events(self, future=False):
        try:
            events = self._event_methods[self.data_source](self)
        except (TimeoutError, ConnectionError, LookupError):
            # if the event source is unreachable or down or data is invalid
            events = []
        if not future:
            return json.dumps(events)
        # the methods are assumed to return events latest first, reverse the order
        tz = pytz.timezone(settings.TIME_ZONE)
        for event in events:
            # for future filtering, make sure all events have end times not null
            try:
                end = event['end_time']
                if not end:
                    event['end_time'] = event['start_time']
            except LookupError:
                event['end_time'] = event['start_time']
            # check the datetimes first
            start = dateparse.parse_datetime(event['start_time'])
            end = dateparse.parse_datetime(event['end_time'])
            # linkedevents may not have exact times, parse_datetime may fail
            # we have to append time, assume server time zone and convert to utc for filtering
            if not start:
                start = tz.localize(datetime.combine(dateparse.parse_date(event['start_time']), time()))
                event['start_time'] = start.astimezone(pytz.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
            if not end:
                end = tz.localize(datetime.combine(dateparse.parse_date(event['end_time']), time(23,59,59)))
                event['end_time'] = end.astimezone(pytz.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
        # we want the next event first
        return json.dumps(list(reversed([event for event in events
                                         if dateparse.parse_datetime(event['end_time']) > datetime.now(tz)])))


from LogisticRegr import *

if __name__ == '__main__':

    df = pd.read_csv('clean_data.csv', delimiter=',')

    card = df['card']  # goal parameter
    reports = df['reports']
    age = df['age']
    income = df['income']
    expenditure = df['expenditure']
    owner = df['owner']
    selfemp = df['selfemp']
    dependents = df['dependents']
    majorcards = df['majorcards']
    active = df['active']

    train_size = int(len(card) * .75)

    train_x = np.array(card[:train_size])  # goal parameter
    train_a = np.array(reports[:train_size])
    train_b = np.array(age[:train_size])
    train_c = np.array(income[:train_size])
    train_d = np.array(expenditure[:train_size])
    train_e = np.array(owner[:train_size])
    train_f = np.array(selfemp[:train_size])
    train_g = np.array(dependents[:train_size])
    train_i = np.array(majorcards[:train_size])
    train_j = np.array(active[:train_size])

    test_x = np.array(card[train_size:len(card)])  # goal parameter
    test_a = np.array(reports[train_size:len(reports)])
    test_b = np.array(age[train_size:len(age)])
    test_c = np.array(income[train_size:len(income)])
    test_d = np.array(expenditure[train_size:len(expenditure)])
    test_e = np.array(owner[train_size:len(owner)])
    test_f = np.array(selfemp[train_size:len(selfemp)])
    test_g = np.array(dependents[train_size:len(dependents)])
    test_i = np.array(majorcards[train_size:len(majorcards)])
    test_j = np.array(active[train_size:len(active)])

    dataset_sizes = {'train': train_x.shape[0], 'test': test_x.shape[0]}

    learningRate = .0001
    epochs = 50

    # Initialize model with saved weights:

    # model = LogisticRegr(9, 1)
    # model.load_state_dict(torch.load('log_regr-clean_data.pt'))
    # model.eval()

    model = LogisticRegr(9, 1)
    criterion = torch.nn.BCELoss()
    optimizer = torch.optim.Adam(model.parameters(), lr=learningRate)

    y_loss = {'train': [], 'test': []}
    y_err = {'train': [], 'test': []}
    x_epoch = []
    fig = plt.figure()
    ax0 = fig.add_subplot(121, title="Loss")
    ax1 = fig.add_subplot(122, title="Accuracy")

    def draw_curve(current_epoch):
        x_epoch.append(current_epoch)
        ax0.plot(x_epoch, y_loss['train'], 'b-', label='train', linewidth=1)
        ax0.plot(x_epoch, y_loss['test'], 'r-', label='test', linewidth=1)
        ax1.plot(x_epoch, y_err['train'], 'b-', label='train', linewidth=1)
        ax1.plot(x_epoch, y_err['test'], 'r-', label='test', linewidth=1)
        if current_epoch == 0:
            ax0.legend()
            ax1.legend()

    for epoch in range(epochs):
        for phase in ['train', 'test']:
            if phase == 'train':
                model.train(True)
            else:
                model.train(False)

            running_loss = 0
            running_corrects = 0

            for i in range(dataset_sizes[phase]):
                if torch.cuda.is_available():
                    inputs = Variable(torch.from_numpy(np.array([train_a[i], train_b[i], train_c[i],
                                                                 train_d[i], train_e[i], train_f[i],
                                                                 train_g[i], train_i[i],
                                                                 train_j[i]])).cuda()).double()
                    labels = Variable(torch.from_numpy(np.array([train_x[i]])).cuda()).double()
                else:
                    inputs = Variable(torch.from_numpy(np.array([train_a[i], train_b[i], train_c[i],
                                                                 train_d[i], train_e[i], train_f[i],
                                                                 train_g[i], train_i[i],
                                                                 train_j[i]]))).double()
                    labels = Variable(torch.from_numpy(np.array([train_x[i]]))).double()

                optimizer.zero_grad()
                outputs = model(inputs)
                preds = round(outputs.data.item())
                loss = criterion(outputs, labels)
                if phase == 'train':
                    loss.backward()
                    optimizer.step()

                running_loss += loss.item()
                running_corrects += int(preds == int(labels.data.item()))

            epoch_loss = running_loss / dataset_sizes[phase]
            epoch_acc = running_corrects / dataset_sizes[phase]
            y_loss[phase].append(epoch_loss)
            y_err[phase].append(epoch_acc)
            if phase == 'test':
                draw_curve(epoch)

            print(f'Epoch {epoch + 1}/{epochs}:')
            print(f'{phase} Loss: {epoch_loss} Acc: {epoch_acc}\n\n')

    fig.show()
    # fig.savefig('loss_acc_plot.png')

    # torch.save(model.state_dict(), 'log_regr-clean_data.pt')


"""
I downloaded using the instructions here and verified that it worked
http://www.sfu.ca/biology2/staff/dc/raxml/
For me, I had to add the argument -T 2 or I got an error about threads

Then I copied the raxml executable to /usr/local/bin/raxmlHPC
Dendropy will be looking for an executable with the name raxmlHPC
Make sure to include `raxml_args=["-T 2"]`

On other operating systems you may need to compile yourself using the directions here
https://cme.h-its.org/exelixis/web/software/raxml/hands_on.html
"""

import os.path
import platform
import unittest

from dendropy.interop import raxml
from dendropy.model.discrete import simulate_discrete_chars, Jc69
from dendropy.calculate.treecompare import symmetric_difference
from numpy.random import default_rng

import spectraltree

class TestRaxml(unittest.TestCase):
    def setUp(self):
        self.reference_tree = spectraltree.balanced_binary(32)

    def test_dendropy_data(self):
        data = simulate_discrete_chars(
            seq_len=2000,
            tree_model=self.reference_tree,
            seq_model=Jc69(),
            mutation_rate=spectraltree.Jukes_Cantor().p2t(0.95),
            rng=default_rng(123))

        raxml = spectraltree.RAxML()
        tree = raxml(data)
        RF = spectraltree.compare_trees(self.reference_tree, tree)
        print("RF (test_without_class):", RF[0])
        self.assertEqual(RF[0], 0)

    def test_numpy_data(self):
        jc = spectraltree.Jukes_Cantor()
        observations, taxa_meta = spectraltree.simulate_sequences(
            seq_len=2000,
            tree_model=self.reference_tree,
            seq_model=jc,
            mutation_rate=jc.p2t(0.95),
            rng=default_rng(123),
            alphabet="DNA")

        raxml = spectraltree.RAxML()
        tree = raxml(observations, taxa_meta)
        RF = spectraltree.compare_trees(self.reference_tree, tree)
        print("RF (test_without_class):", RF[0])
        self.assertEqual(RF[0], 0)

if __name__ == "__main__":
    unittest.main()


'''
	__author__ = "Will Russell"
'''

import csv

weatherFiles = ["train_data_1.txt","train_data_2.txt","train_data_3.txt",]

'''

'''
def retrieve_weather_data(fileName):
	weather_data = []
	with open(fileName, newline="\n") as csvfile:
		csvreader = csv.reader(csvfile, delimiter=",")
		for row in csvreader:
			weather_data.append({"hour":int(row[0]), "temp":float(row[1])})
	return weather_data

'''

'''
def retrieve_training_data(relative_path="./", files=weatherFiles):
	total_training_data = []
	for f in files:
		data = retrieve_weather_data(relative_path + f)
		total_training_data.append(data)
	return total_training_data

def get_all_training_data(relative_path="./", files=weatherFiles):
	total_training_data = []
	for f in files:
		data = retrieve_weather_data(relative_path + f)


MOD = int(1e9 + 7)

def binomial(n, k):
    """
    A fast way to calculate binomial coefficients by Andrew Dalke.
    See http://stackoverflow.com/questions/3025162/statistics-combinations-in-python
    """
    if 0 <= k <= n:
        ntok = 1
        ktok = 1
        for t in range(1, min(k, n - k) + 1):
            ntok *= n
            ktok *= t
            n -= 1
        return (ntok // ktok) % MOD
    else:
        return 0

T = int(input())
for _ in range(T):
    n, k = [int(x) for x in input().split()]
    s = 0
    for r in range(1, k + 1):
        t = pow(2, r, MOD) * binomial(k - 1, r - 1) * binomial(n - k + 1, r)
        t %= MOD
        s += t
        s %= MOD
    print(s)


import cv2
import imutils
import os
import matplotlib.pyplot as plt

# Imgage path
image_path = r'C:\Users\UTILISATEUR\Pictures\logoPython.png'
# Image directory
directory = r'C:\Users\UTILISATEUR\Pictures'
img = cv2.imread(image_path) # lire une image
# change the current directory to specified directory
os.chdir(directory)
print("Before saving image")
print(os.listdir(directory))
resizeImg = imutils.resize(img,width=100) # redimensionner une image
cv2.imwrite("resizedImg.jpg",resizeImg) # sauvegarder l'image

img = cv2.imread("logoPython.jpg") # lire une image
grayImg = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
thresImg = cv2.threshold(grayImg,180,255,cv2.THRESH_BINARY)[1]
plt.imshow(thresImg,cmap='gray') # afficher l'image
cv2.imwrite("imgThresholdGraylogoPython.jpg",thresImg)

# List files in directory C:\Users\caspary5\Documents\hello_world\images
print("After processing images")
print(os.listdir(directory))
print("Successfully saved")


import json
import sys
import subprocess
import re

class ModuleProxy():
  """
  This class provides a simple sandbox that allows code from arbitrary Python
  modules to be evaluated in a subprocess. Said code cannot access the state
  of the calling process, though it has the same user privileges as the caller.
  """
  def __init__(self, module_name):
    self.module_name = module_name

  def __getattr__(self, symbol_name):
    return ObjectProxy(self, symbol_name)

  def _get(self, symbol_name, call=False, *args):
    if call:
      script_tmpl = "import {}; print(repr({}.{}(*{})))"
      script = script_tmpl.format(
        self.module_name,
        self.module_name,
        symbol_name,
        repr(args)
      )
    else:
      script_tmpl = "import {}; print(repr({}.{}))"
      script = script_tmpl.format(self.module_name, self.module_name, symbol_name)

    raw_output = subprocess.check_output([sys.executable, '-c', script])
    # the Python JSON module won't parse string literals enclosed by single-quote
    # characters, so...
    output = re.sub(r"'", '"', raw_output.decode("utf-8").rstrip())
    # allows for simple scalar types and lists to be returned by evaluated code
    # without breaking sandbox
    return json.loads(output)

class ObjectProxy():
  def __init__(self, mod_proxy, symbol_name):
    self.mod_proxy = mod_proxy
    self.symbol_name = symbol_name

  def __call__(self, *args):
    return self.mod_proxy._get(self.symbol_name, True, *args)

  def __repr__(self):
    return repr(self.value())

  def value(self):
    return self.mod_proxy._get(self.symbol_name)

if __name__ == "__main__":
  # simple tests of this module
  fake_sys = ModuleProxy('sys')
  assert isinstance(fake_sys.executable.value(), str)
  fake_math = ModuleProxy('math')
  assert fake_math.floor(3.5) is 3
  print("tests ok")


import sys

# * ========================================================
# * @Title       : 2447
# * @Path        : python\문제풀이\boj\z.분류별\2447_별 찍기 10.py
# * @Link        : https://www.acmicpc.net/problem/2447
# * @Description : 별 찍기 10
# * @Note        :
# * @Date        : 2021-08-12 18:45:41
# * --------------------------------------------------------
# * @Author      : Inseong-so(https://github.com/inseong-so)
# * ========================================================

# Solve 함수
def solution(N):
    matrix = [[' ' for _ in range(N)] for _ in range(N)]

    def stars(x, y, size):
        if size == 1:
            matrix[x][y] = '*'
            return

        for i in range(3):
            for j in range(3):
                if i != 1 or j != 1:
                    stars(x + i * (size // 3), y + j * (size // 3), size // 3)

    stars(0, 0, N)

    for i in range(len(matrix)):
        for j in range(len(matrix[i])):
            print(matrix[i][j], end="")
        print()

    return 1

# # 입력
# input()
# int(input())
# list(map(int, sys.stdin.readline().split()))
# solution()

def test_01(capfd):
    # solution(27)
    # out, err = capfd.readouterr()
    # result = '\n'
    # assert out == result
    assert solution(27) == 0

# def test_02(capfd):
    # solution()
    # out, err = capfd.readouterr()
    # result = '\n'
    # assert out == result

# def test_03(capfd):
    # solution()
    # out, err = capfd.readouterr()
    # result = '\n'
    # assert out == result

# def test_04(capfd):
    # solution()
    # out, err = capfd.readouterr()
    # result = '\n'
    # assert out == result

# # 숏코딩
# def solution(n):
#     s = '*'
#     while n > 1:
#         t = [i*3 for i in s]
#         s = t+[i+' '*len(i)+i for i in s]+t
#         n //= 3
#     print('\n'.join(s))

# # 내 풀이
# def solution(N):
#     matrix = [[' ' for _ in range(N)] for _ in range(N)]

#     def stars(x, y, sz):
#         if sz == 1:
#             matrix[x][y] = '*'
#             return
#         for i in range(3):
#             for j in range(3):
#                 if i != 1 or j != 1:
#                     stars(x + sz // 3 * i, y + sz // 3 * j, sz // 3)
#     stars(0, 0, N)
#     for i in range(len(matrix)):
#         for j in range(len(matrix[i])):
#             print(matrix[i][j], end="")
#         print()


from .helpers import *

path = __file__.replace('\\', '/').replace('__main__.py', '')

with open(f'{path}/input') as f:
    data = f.read()

positions = [int(x) for x in data.split(',')]

def get_fuel_cost(position: int, acceleration: bool) -> int:
    total = 0
    for x in positions:
        distance = abs(x-position)
        if acceleration:
            for i in range(1, distance+1):
                total += i
        else:
            total += distance
    return total

def get_min_cost(acceleration: bool) -> int:
    costs = []
    for i in range(max(positions)):
        cost = get_fuel_cost(i, acceleration)
        costs.append(cost)

    return min(costs)

print('--- PART 1 (START) ---')
print(get_min_cost(False))
print('--- PART 1 (END) ---')

print('--- PART 2 (START) ---')
print(get_min_cost(True))
print('--- PART 2 (END) ---')


import time
import arcade

SCREEN_WIDTH = 600
SCREEN_HEIGHT = 600
OPENING_SCREEN_DURATION = 0.5
OPENING_SCREEN_IMAGE = "images/background.jpg"

class OpeningScreen(arcade.View):
    def __init__(self):
        super().__init__()
        self.opening_time = None

    def on_show(self):
        self.opening_time = time.monotonic()

    def on_draw(self):
        arcade.start_render()
        arcade.draw_texture_rectangle(SCREEN_WIDTH / 2, SCREEN_HEIGHT / 2, SCREEN_WIDTH, SCREEN_HEIGHT,
                                      arcade.load_texture(OPENING_SCREEN_IMAGE))

    def on_update(self, delta_time):
        if time.monotonic() - self.opening_time >= OPENING_SCREEN_DURATION:
            from Learn_With_Mazes.Learn_With_Mazes_Django.src.screens.MainMenu import MainMenu
            main_menu = MainMenu()
            self.window.show_view(main_menu)