import numpy as np
import sqlite3 as sq
import datetime as dt
import subprocess as sp
import glob as gb
import os

import matplotlib.pyplot as plt

from PyFVCOM.grid import vincenty_distance
from PyFVCOM.read import FileReader
from PyFVCOM.plot import Time, Plotter
from PyFVCOM.stats import calculate_coefficient, rmse

SQL_UNIX_EPOCH = dt.datetime(1970, 1, 1, 0, 0, 0)

"""
Validation of model outputs against in situ data stored and extracted from a database.

This also includes the code to build the databases of time series data sets.

"""

class validation_db():
    """ Work with an SQLite database. """

    def __init__(self, db_name):
        if db_name[-3:] != '.db':
            db_name += '.db'
        self.conn = sq.connect(db_name)
        self.create_table_sql = {}
        self.retrieve_data_sql = {}
        self.c = self.conn.cursor()

    def execute_sql(self, sql_str):
        """
        Execute the given SQL statement.

        Parameters
        ----------
        sql_str : str
            SQL statement to execute.

        Returns
        -------
        results : np.ndarray
            Data from the database which matches the SQL statement.
        """

        self.c.execute(sql_str)

        return self.c.fetchall()

    def make_create_table_sql(self, table_name, col_list):
        """
        Create an SQL table if no such table exists.

        Parameters
        ----------
        table_name : str
            Table name to create.
        col_list : list
            List of column names to add to the table.

        """

        create_str = 'CREATE TABLE IF NOT EXISTS ' + table_name + ' ('
        for this_col in col_list:
            create_str += this_col
            create_str += ', '
        create_str = create_str[0:-2]
        create_str += ');'
        self.create_table_sql['create_' + table_name] = create_str

    def insert_into_table(self, table_name, data):
        """
        Insert data into a table.

        Parameters
        ----------
        table_name : str
            Table name into which to insert the given data.
        data : np.ndarray
            Data to insert into the database.

        """
        no_rows = len(data)
        no_cols = len(data[0])
        qs_string = '('
        for this_x in range(no_cols):
            qs_string += '?,'
        qs_string = qs_string[:-1]
        qs_string += ')'

        if no_rows > 1:
            self.c.executemany('insert or ignore into ' + table_name + ' values ' + qs_string, data)
        elif no_rows == 1:
            self.c.execute('insert into ' + table_name + ' values ' + qs_string, data[0])
        self.conn.commit()

    def select_qry(self, table_name, where_str, select_str='*', order_by_str=None, inner_join_str=None, group_by_str=None):
        """
        Extract data from the database which matches the given SQL query.

        Parameters
        ----------
        table_name : str
            Table name to query.
        where_str : str
            Where statement.
        select_str : str, optional
            Optionally give a set of columns to select.
        order_by_str : str, optional
            Optionally give a set of columns by which to order the results.
        inner_join_str : str, optional
            Optionally give an inner join string.
        group_by_str : str, optional
            Optionally give a string by which to group the results.

        Returns
        -------
        results : np.ndarray
            The data which matches the given query.

        """
        qry_string = 'select ' + select_str + ' from ' + table_name
        if inner_join_str:
            qry_string += ' inner join ' + inner_join_str
        if where_str:
            qry_string += ' where ' + where_str
        if order_by_str:
            qry_string += ' order by ' + order_by_str
        if group_by_str:
            qry_string += ' group by ' + group_by_str

        return self.execute_sql(qry_string)

    def close_conn(self):
        """ Close the connection to the database. """
        self.conn.close()

def dt_to_epochsec(time_to_convert):
    """
    Convert a datetime to our SQL database epoch.

    Parameters
    ----------
    time_to_convert : datetime.datetime
        Datetime to convert.

    Returns
    -------
    epoched : int
        Converted datetime (in seconds).

    """

    return (time_to_convert - SQL_UNIX_EPOCH).total_seconds()

def epochsec_to_dt(time_to_convert):
    """

    Parameters
    ----------
    time_to_convert : int
        Seconds in the SQL database epoch.

    Return
    ------
    unepoched : datetime.datetime.
        Converted time.

    """
    return SQL_UNIX_EPOCH + dt.timedelta(seconds=time_to_convert)

def plot_map(fvcom, tide_db_path, threshold=np.inf, legend=False, **kwargs):
    """
    Plot the tide gauges which fall within the model domain (in space and time) defined by the given FileReader object.

    Parameters
    ----------
    fvcom : PyFVCOM.read.FileReader
        FVCOM model data as a FileReader object.
    tide_db_path : str
        Path to the tidal database.
    threshold : float, optional
        Give a threshold distance (in spherical units) beyond which a gauge is considered too far away.
    legend : bool, optional
        Set to True to add a legend to the plot. Defaults to False.

    Any remaining keyword arguments are passed to PyFVCOM.plot.Plotter.

    Returns
    -------
    plot : PyFVCOM.plot.Plotter
        The Plotter object instance for the map

    """

    tide_db = db_tide(tide_db_path)
    gauge_names, gauge_locations = tide_db.get_gauge_locations(long_names=True)

    gauges_in_domain = []
    fvcom_nodes = []
    for gi, gauge in enumerate(gauge_locations):
        river_index = fvcom.closest_node(gauge, threshold=threshold)
        if river_index:
            gauge_id, gauge_dist  = tide_db.get_nearest_gauge_id(*gauge)
            times, data = tide_db.get_tidal_series(gauge_id, np.min(fvcom.time.datetime), np.max(fvcom.time.datetime))
            if not np.any(data):
                continue

            gauges_in_domain.append(gi)
            fvcom_nodes.append(river_index)

    plot = Plotter(fvcom, **kwargs)
    fx, fy = plot.m(fvcom.grid.lon, fvcom.grid.lat)
    plot.plot_field(-fvcom.grid.h)
    plot.axes.plot(fx[fvcom_nodes], fy[fvcom_nodes], 'ro', markersize=3, zorder=202, label='Model')
    # Add the gauge locations.
    rx, ry = plot.m(gauge_locations[:, 0], gauge_locations[:, 1])
    plot.axes.plot(rx, ry, 'wo', label='Gauges')
    for xx, yy, name in zip(rx, ry, gauge_names[gauges_in_domain]):
        plot.axes.text(xx, yy, name, fontsize=10, rotation=45, rotation_mode='anchor', zorder=203)

    if legend:
        plot.axes.legend(numpoints=1, scatterpoints=1, ncol=2, loc='upper center', fontsize=10)

    return plot

def plot_tides(fvcom, db_name, threshold=500, figsize=(10, 10), **kwargs):
    """
    Plot model and tide gauge data.

    Parameters
    ----------
    fvcom : PyFVCOM.read.FileReader
        FVCOM model data as a FileReader object.
    db_name : str
        Database name to interrogate.
    threshold : float, optional
        Give a threshold distance (in spherical units) to exclude gauges too far from a model node.
    figsize : tuple
        Give a figure size (units are inches).

    Remaining keyword arguments are passed to PyFVCOM.plot.Time.

    Returns
    -------
    time : PyFVCOM.plot.Time
        Time series plot object.
    gauge_obs : dict
        Dictionary with the gauge and model data.

    """

    tide_db = db_tide(db_name)

    # Get all the gauges in the database and find the corresponding model nodes.
    gauge_names, gauge_locations = tide_db.get_gauge_locations(long_names=True)

    gauge_obs = {}
    gauges_in_domain = []
    fvcom_nodes = []
    for gi, gauge in enumerate(gauge_locations):
        river_index = fvcom.closest_node(gauge, threshold=threshold)
        if river_index:
            current_gauge = {}
            current_gauge['gauge_id'], current_gauge['gauge_dist'] = tide_db.get_nearest_gauge_id(*gauge)
            current_gauge['times'], current_gauge['data'] = tide_db.get_tidal_series(current_gauge['gauge_id'],
                                                                                np.min(fvcom.time.datetime),
                                                                                np.max(fvcom.time.datetime))
            if not np.any(current_gauge['data']):
                continue

            current_gauge['lon'], current_gauge['lat'] = gauge_locations[gi, :]

            current_gauge['gauge_clean'] = current_gauge['data'][:, 1] == 0
            current_gauge['gauge_obs_clean'] = {'times': np.copy(current_gauge['times'])[current_gauge['gauge_clean']],
                                                'data': np.copy(current_gauge['data'])[current_gauge['gauge_clean'], 0]}
            current_gauge['rescale_zeta'] = fvcom.data.zeta[:, river_index] - np.mean(fvcom.data.zeta[:, river_index])
            current_gauge['rescale_gauge_obs'] = current_gauge['gauge_obs_clean']['data'] - np.mean(current_gauge['gauge_obs_clean']['data'])

            current_gauge['dates_mod'] = np.isin(fvcom.time.datetime, current_gauge['gauge_obs_clean']['times'])
            current_gauge['dates_obs'] = np.isin(current_gauge['gauge_obs_clean']['times'], fvcom.time.datetime)
            # Skip out if we don't have any coincident data (might simply be a sampling issue) within the model
            # period. We should interpolate here.
            if not np.any(current_gauge['dates_mod']) or not np.any(current_gauge['dates_obs']):
                continue

            current_gauge['r'], current_gauge['p'] = calculate_coefficient(current_gauge['rescale_zeta'][current_gauge['dates_mod']], current_gauge['rescale_gauge_obs'][current_gauge['dates_obs']])
            current_gauge['rms'] = rmse(current_gauge['rescale_zeta'][current_gauge['dates_mod']], current_gauge['rescale_gauge_obs'][current_gauge['dates_obs']])
            current_gauge['std'] = np.std(current_gauge['rescale_zeta'][current_gauge['dates_mod']] - current_gauge['rescale_gauge_obs'][current_gauge['dates_obs']])

            gauges_in_domain.append(gi)
            fvcom_nodes.append(river_index)

            name = gauge_names[gi]
            gauge_obs[name] = current_gauge
            del current_gauge

    tide_db.close_conn()  # tidy up after ourselves

    # Now make a figure of all that data.
    if len(gauge_obs) > 5:
        cols = np.ceil(len(gauge_obs) ** (1.0 / 3)).astype(int) + 1
    else:
        cols = 1
    rows = np.ceil(len(gauge_obs) / cols).astype(int)
    fig = plt.figure(figsize=figsize)
    for count, site in enumerate(sorted(gauge_obs)):
        ax = fig.add_subplot(rows, cols, count + 1)
        time = Time(fvcom, figure=fig, axes=ax, hold=True, **kwargs)
        time.plot_line(gauge_obs[site]['rescale_zeta'], label='Model', color='k')
        # We have to use the raw plot function for the gauge data as the plot_line function assumes we're using model
        # data.
        time.axes.plot(gauge_obs[site]['gauge_obs_clean']['times'], gauge_obs[site]['rescale_gauge_obs'], label='Gauge', color='m')
        # Should add the times of the flagged data here.
        time.axes.set_xlim(fvcom.time.datetime.min(), fvcom.time.datetime.max())
        time.axes.set_ylim(np.min((gauge_obs[site]['rescale_gauge_obs'].min(), gauge_obs[site]['rescale_zeta'].min())),
                           np.max((gauge_obs[site]['rescale_gauge_obs'].max(), gauge_obs[site]['rescale_zeta'].max())))
        time.axes.set_title(site)

    return time, gauge_obs

def _make_normal_tide_series(h_series):
    height_series = h_series - np.mean(h_series)
    return height_series

class db_tide(validation_db):
    """ Create a time series database and query it. """

    def make_bodc_tables(self):
        """ Make the complete set of empty tables for data to be inserted into (as defined in _add_sql_strings) """

        # Insert information into the error flags table
        self._add_sql_strings()
        for this_table, this_str in self.create_table_sql.items():
            self.execute_sql(this_str)
        error_data = [(0, '', 'No error'), (1, 'M', 'Improbable value flagged by QC'),
                        (2, 'N', 'Null Value'), (3, 'T', 'Value interpolated from adjacent values')]
        self.insert_into_table('error_flags', error_data)

    def insert_tide_file(self, file_list):
        """
        Add data from a set of files to the database.

        Parameters
        ----------
        file_list : list
            List of file names.

        """
        for this_file in file_list:
            print('Inserting data from file: ' + this_file)
            this_file_obj = bodc_annual_tide_file(this_file)
            try:
                site_id = self.select_qry('sites', "site_tla == '" + this_file_obj.site_tla + "'", 'site_id')[0][0]
            except:
                try:
                    current_id_max = np.max(self.select_qry('sites', None, 'site_id'))
                    site_id = int(current_id_max + 1)
                except:
                    site_id = 1

                site_data = [(site_id, this_file_obj.site_tla, this_file_obj.site_name, this_file_obj.lon, this_file_obj.lat, '')]
                self.debug_data = site_data
                self.insert_into_table('sites', site_data)

            site_id_list = [site_id] * len(this_file_obj.seconds_from_ref)
            table_data = list(zip(site_id_list, this_file_obj.seconds_from_ref, this_file_obj.elevation_data,
                            this_file_obj.elevation_flag, this_file_obj.residual_data, this_file_obj.residual_flag))
            self.insert_into_table('gauge_obs', table_data)

    def get_tidal_series(self, station_identifier, start_date_dt=None, end_date_dt=None):
        """
        Extract a time series of tidal elevations for a given station.

        Parameters
        ----------
        station_identifier : str
            Database station identifier.
        start_date_dt, end_date_dt : datetime.datetime, optional
            Give start and/or end times to extract from the database. If omitted, all data are returned.

        Returns
        -------
        dates : np.ndarray
            Array of datetime objects.
        data : np.ndarray
            Surface elevation and residuals from the database for the given station.

        """
        select_str = "time_int, elevation, elevation_flag"
        table_name = "gauge_obs as go"
        inner_join_str = "sites as st on st.site_id = go.site_id"

        if isinstance(station_identifier, str):
            where_str = "st.site_tla = '" + station_identifier + "'"
        else:
            where_str = "st.site_id = " + str(int(station_identifier))

        if start_date_dt is not None:
            start_sec = dt_to_epochsec(start_date_dt)
            where_str += " and go.time_int >= " + str(start_sec)
        if end_date_dt is not None:
            end_sec = dt_to_epochsec(end_date_dt)
            where_str += " and go.time_int <= " + str(end_sec)
        order_by_str = 'go.time_int'
        return_data = self.select_qry(table_name, where_str, select_str, order_by_str, inner_join_str)
        if not return_data:
            print('No data available')
            dates, data = None, None
        else:
            return_data = np.asarray(return_data)
            date_list = [epochsec_to_dt(this_time) for this_time in return_data[:,0]]
            dates, data = np.asarray(date_list), return_data[:, 1:]

        return dates, data

    def get_gauge_locations(self, long_names=False):
        """
        Extract locations and names of the tide gauges from the database.

        Parameters
        ----------
        long_names : bool, optional
            If True, return the 'nice' long names rather than the station identifiers.

        Returns
        -------
        tla_name : np.ndarray
            List of tide gauge names.
        lon_lat : np.ndarray
            Positions of the gauges.

        """
        gauge_site_data = np.asarray(self.select_qry('sites', None))
        if long_names:
            tla_name = gauge_site_data[:, 2]
        else:
            tla_name = gauge_site_data[:, 1]
        lon_lat = np.asarray(gauge_site_data[:, 3:5], dtype=float)

        return tla_name, lon_lat

    def get_nearest_gauge_id(self, lon, lat, threshold=np.inf):
        """
        Get the ID of the gauge closest to the position given by `lon' and `lat'.

        lon, lat : float
            Position for which to search for the nearest tide gauge.

        Returns
        -------
        closest_gauge_id : int
            Database ID for the gauge closest to `lon' and `lat'.
        min_dist : float
            Distance in metres between `lon' and `lat' and the gauge.
        threshold : float
            Threshold distance in metres (inclusive) within which gauges must be from the given position. If no
            gauges are found within this distance, the gauge ID is None.

        """

        sites_lat_lon = np.asarray(self.select_qry('sites', None, 'site_id, lat, lon'))
        min_dist = np.inf
        closest_gauge_id = None  # we should make this False or None or something
        for this_row in sites_lat_lon:
            this_dist = vincenty_distance([lat, lon], [this_row[1], this_row[2]])
            if this_dist < min_dist:
                min_dist = this_dist
                closest_gauge_id = this_row[0]
        if min_dist >= threshold:
            closest_gauge_id = None
        else:
            closest_gauge_id = int(closest_gauge_id)

        return closest_gauge_id, min_dist

    def _add_sql_strings(self):
        """ Function to define the database structure. """
        bodc_tables = {'gauge_obs': ['site_id integer NOT NULL', 'time_int integer NOT NULL',
                                     'elevation real NOT NULL', 'elevation_flag integer', 'residual real', 'residual_flag integer',
                                     'PRIMARY KEY (site_id, time_int)', 'FOREIGN KEY (site_id) REFERENCES sites(site_id)',
                                     'FOREIGN KEY (elevation_flag) REFERENCES error_flags(flag_id)',
                                     'FOREIGN KEY (residual_flag) REFERENCES error_flags(flag_id)'],
                       'sites': ['site_id integer NOT NULL', 'site_tla text NOT NULL', 'site_name text', 'lon real', 'lat real',
                                 'other_stuff text', 'PRIMARY KEY (site_id)'],
                       'error_flags': ['flag_id integer NOT NULL', 'flag_code text', 'flag_description text']}

        for this_key, this_val in bodc_tables.items():
            self.make_create_table_sql(this_key, this_val)

class bodc_annual_tide_file():
    def __init__(self, file_name, header_length=11):
        """
        Assumptions: file name of the form yearTLA.txt

        """
        bodc_annual_tide_file._clean_tide_file(file_name, header_length)
        with open(file_name) as f:
            header_lines = [next(f) for this_line in range(header_length)]

        for this_line in header_lines:
            if 'ongitude' in this_line:
                self.lon = [float(s) for s in this_line.split() if bodc_annual_tide_file._is_number(s)][0]
            if 'atitude' in this_line:
                self.lat = [float(s) for s in this_line.split() if bodc_annual_tide_file._is_number(s)][0]
            if 'Site' in this_line:
                site_str_raw = this_line.split()[1:]
                if len(site_str_raw) == 1:
                    site_str = site_str_raw[0]
                else:
                    site_str = ''
                    for this_str in site_str_raw:
                        site_str += this_str
        self.site_name = site_str
        self.site_tla = file_name.split('/')[-1][4:7]

        raw_data = np.loadtxt(file_name, skiprows=header_length, dtype=bytes).astype(str)

        seconds_from_ref = []
        for this_row in raw_data:
            this_dt_str = this_row[1] + ' ' + this_row[2]
            this_seconds_from_ref = dt_to_epochsec(dt.datetime.strptime(this_dt_str, '%Y/%m/%d %H:%M:%S'))
            seconds_from_ref.append(int(this_seconds_from_ref))
        self.seconds_from_ref = seconds_from_ref

        elevation_data = []
        elevation_flag = []
        residual_data = []
        residual_flag = []
        for this_row in raw_data:
            meas, error_code = bodc_annual_tide_file._parse_tide_obs(this_row[3])
            elevation_data.append(meas)
            elevation_flag.append(error_code)
            meas, error_code = bodc_annual_tide_file._parse_tide_obs(this_row[4])
            residual_data.append(meas)
            residual_flag.append(error_code)
        self.elevation_data = elevation_data
        self.elevation_flag = elevation_flag
        self.residual_data = residual_data
        self.residual_flag = residual_flag

    @staticmethod
    def _parse_tide_obs(in_str):
        error_code_dict = {'M':1, 'N':2, 'T':3}
        try:
            int(in_str[-1])
            error_code = 0
            meas = float(in_str)
        except:
            error_code_str = in_str[-1]
            meas = float(in_str[0:-1])
            try:
                error_code = error_code_dict[error_code_str]
            except:
                print('Unrecognised error code')
                return
        return meas, error_code

    @staticmethod
    def _is_number(s):
        try:
            float(s)
            return True
        except ValueError:
            return False

    @staticmethod
    def _clean_tide_file(file_name, header_length):
        sed_str = "sed -i '"+ str(header_length + 1) + ",$ {/^ *[0-9]/!d}' " + file_name
        sp.call([sed_str], shell=True)

#################################################################################################################
"""

Validation against L4 and E1 CTD and buoy data

observations_meta_data = {'buoy_name':'E1', 'year':'2006', 'ctd_new_file_type': False,
            'ctd_datadir':'/data/euryale4/backup/mbe/Data/WCO_data/E1/CTD_data/2006',
            'buoy_filepath':None, 'lon':-4.368, 'lat':50.035}

observations_meta_data = {'buoy_name':'L4', 'year':'2015', 'ctd_new_file_type': True, 'ctd_filepath':'./data/e1_data_2015.txt',
            'buoy_filepath': , '/data/euryale4/backup/mbe/Data/WCO_data/L4/Buoy_data/l4_cont_data_2015.txt', 'lon':-4.217, 'lat':50.250}

model_filestr_lambda = lambda m: '/data/euryale4/backup/mbe/Models/FVCOM/tamar_v2/run/output/depth_tweak2/2006/{:02d}/tamar_v2_0001.nc'.format(m)
available_months = np.arange(1,13)
model_file_list = [model_filestr_lambda(this_month) for this_month in available_months]

"""

class db_wco(validation_db):
    """ Work with an SQL database of data from PML's Western Channel Observatory. """

    def make_wco_tables(self):
        """
        Make the complete set of empty tables for data to be inserted into (as defined in _add_sql_strings).

        """

        # Insert information into the error flags table
        self._add_sql_strings()
        for this_table, this_str in self.create_table_sql.items():
            self.execute_sql(this_str)
        sites_data = [(0, 'L4',-4.217,50.250, ' '), (1, 'E1',-4.368,50.035,' ')]
        self.insert_into_table('sites', sites_data)
        measurement_type_data = [(0,'CTD measurements'), (1, 'Surface buoy measurements')]
        self.insert_into_table('measurement_types', measurement_type_data)
        self.execute_sql('create index date_index on obs (time_int);')

    def _add_sql_strings(self):
        wco_tables = {'obs':['buoy_id integer NOT NULL', 'time_int integer NOT NULL',
                                    'depth real NOT NULL', 'temp real', 'salinity real', 'measurement_flag integer NOT NULL',
                                    'PRIMARY KEY (buoy_id, depth, measurement_flag, time_int)', 'FOREIGN KEY (buoy_id) REFERENCES sites(buoy_id)',
                                    'FOREIGN KEY (measurement_flag) REFERENCES measurement_types(measurement_flag)'],
                       'sites':['buoy_id integer NOT NULL', 'buoy_name text', 'lon real', 'lat real',
                                    'other_stuff text', 'PRIMARY KEY (buoy_id)'],
                       'measurement_types':['measurement_flag integer NOT NULL', 'measurement_description text', 'PRIMARY KEY (measurement_flag)']}

        for this_key, this_val in wco_tables.items():
            self.make_create_table_sql(this_key, this_val)

    def insert_CTD_file(self, filestr, buoy_id):
        file_obj = WCO_obs_file(filestr)
        self._insert_obs(file_obj, buoy_id, 0.0)

    def insert_buoy_file(self, filestr, buoy_id):
        file_obj = WCO_obs_file(filestr, depth=0)
        self._insert_obs(file_obj, buoy_id, 1.0)

    def insert_CTD_dir(self, dirstr, buoy_id):
        file_obj = CTD_dir(dirstr)
        self._insert_obs(file_obj, buoy_id, 0.0)

    def _insert_obs(self, file_obj, buoy_id, measurement_id):
        epoch_sec_timelist = []
        for this_time in file_obj.observation_dict['dt_time']:
            epoch_sec_timelist.append(dt_to_epochsec(this_time))
        buoy_id_list = np.tile(buoy_id, len(epoch_sec_timelist))
        measurement_id_list = np.tile(measurement_id, len(epoch_sec_timelist))

        table_data = list(zip(buoy_id_list, epoch_sec_timelist, file_obj.observation_dict['depth'], file_obj.observation_dict['temp'],
                        file_obj.observation_dict['salinity'], measurement_id_list))
        self.insert_into_table('obs', table_data)

    def get_observations(self, buoy_name, start_date_dt=None, end_date_dt=None, measurement_id=None):
        select_str = "time_int, depth, temp, salinity"
        table_name = "obs as go"
        inner_join_str = "sites as st on st.buoy_id = go.buoy_id"

        where_str = "st.buoy_name = '" + buoy_name + "'"

        if start_date_dt is not None:
            start_sec = dt_to_epochsec(start_date_dt)
            where_str += " and go.time_int >= " + str(start_sec)
        if end_date_dt is not None:
            end_sec = dt_to_epochsec(end_date_dt)
            where_str += " and go.time_int <= " + str(end_sec)
        order_by_str = 'go.time_int, go.depth'

        return_data = self.select_qry(table_name, where_str, select_str, order_by_str, inner_join_str)

        if not return_data:
            dates, data = None, None
            print('No data available')
        else:
            return_data = np.asarray(return_data)
            date_list = [epochsec_to_dt(this_time) for this_time in return_data[:,0]]
            dates, data = np.asarray(date_list), return_data[:,1:]

        return dates, data

class WCO_obs_file():
    def __init__(self, filename, depth=None):
        self._setup_possible_vars()
        self.observation_dict = self._add_file(filename)
        if depth is not None:
            self.observation_dict['depth'] = np.tile(depth, len(self.observation_dict['dt_time']))

    def _add_file(self,filename,remove_undated=True):
        # remove duff lines
        sed_str = "sed '/^-9.990e/d' " + filename + " > temp_file.txt"
        sp.call(sed_str, shell=True)

        # some files have multiple records of differing types...helpful
        temp_str = 'YKA123ASD'
        file_split_str = '''awk '/^[^0-9]/{g++} { print $0 > "''' + temp_str + '''"g".txt"}' temp_file.txt'''
        sp.call(file_split_str, shell=True)
        temp_file_list = gb.glob(temp_str + '*')

        obs_dict_list = []
        for this_file in temp_file_list:
            this_obs = self._add_file_part(this_file)
            if not remove_undated or 'dt_time' in this_obs:
                obs_dict_list.append(this_obs)
        rm_file = [os.remove(this_file) for this_file in temp_file_list]
        return {this_key:np.hstack([this_dict[this_key] for this_dict in obs_dict_list]) for this_key in obs_dict_list[0]}

    def _add_file_part(self, filename):
        # seperate header and clean out non numeric lines
        head_str ="head -1 " + filename + " > temp_header_file.txt"
        sed_str = "sed '/^[!0-9]/!d' " + filename + " > temp_file.txt"
        sp.call(head_str, shell=True)
        sp.call(sed_str, shell=True)

        # Load the files, some use semi-colon delimiters, some whitespace...
        if ';' in str(np.loadtxt('temp_header_file.txt', delimiter='no_delimination_needed', dtype=str)):
            observations_raw = np.loadtxt('temp_file.txt', delimiter=';',dtype=str)
            observations_header = np.loadtxt('temp_header_file.txt', delimiter=';',dtype=str)
        elif ',' in str(np.loadtxt('temp_header_file.txt', delimiter='no_delimination_needed', dtype=str)):
            observations_raw = np.loadtxt('temp_file.txt', delimiter=',',dtype=str)
            observations_header = np.loadtxt('temp_header_file.txt', delimiter=',',dtype=str)
        else:
            observations_raw = np.loadtxt('temp_file.txt', dtype=str)
            observations_header = np.loadtxt('temp_header_file.txt', dtype=str)

        # Clean up temp files
        os.remove('temp_file.txt')
        os.remove('temp_header_file.txt')

        # Find the relevant columns and pull out temp, salinity, date, etc if available
        observation_dict = {}
        time_vars = []

        for this_var, this_possible in self.possible_vars.items():
            if np.any(np.isin(this_possible, observations_header)):
                this_col = np.where(np.isin(observations_header, this_possible))[0]
                if this_var == 'time' or this_var =='date' or this_var=='Jd':
                    observation_dict[this_var] = np.squeeze(np.asarray(observations_raw[:,this_col], dtype=str))
                    time_vars.append(this_possible[np.isin(this_possible, observations_header)])
                else:
                    observation_dict[this_var] = np.squeeze(np.asarray(observations_raw[:,this_col], dtype=float))
        if 'date' in observation_dict:
            observation_dict['dt_time'] = self._parse_dates_to_dt(observation_dict, time_vars)
        return observation_dict

    def _setup_possible_vars(self):
        self.possible_vars = {'temp':np.asarray(['Tv290C', 'SST', ' Mean SST (degC)']), 'salinity':np.asarray(['Sal00', 'Sal', ' Mean SST (degC)']),
                        'depth':np.asarray(['DepSM']),    'date':np.asarray(['mm/dd/yyyy', 'Year', ' Date (YYMMDD)']),
                        'julian_day':np.asarray(['Jd']), 'time':np.asarray(['hh:mm:ss', 'Time', ' Time (HHMMSS)'])}

    @staticmethod
    def _parse_dates_to_dt(obs_dict, time_vars):
        dt_list = []
        if np.any(np.isin('mm/dd/yyyy', time_vars)):
            for this_time, this_date in zip(obs_dict['time'], obs_dict['date']):
                dt_list.append(dt.datetime.strptime(this_date + ' ' + this_time, '%m/%d/%Y %H:%M:%S'))
        elif np.any(np.isin('Year', time_vars)):
            for this_time, (this_jd, this_year) in zip(obs_dict['time'], zip(obs_dict['julian_day'], obs_dict['date'])):
                dt_list.append(dt.datetime(int(this_year),1,1) + dt.timedelta(days=int(this_jd) -1) +
                                dt.timedelta(hours=int(this_time.split('.')[0])) + dt.timedelta(minutes=int(this_time.split('.')[1])))
        elif np.any(np.isin(' Date (YYMMDD)', time_vars)):
            for this_time, this_date in zip(obs_dict['time'], obs_dict['date']):
                dt_list.append(dt.datetime.strptime(this_date + ' ' + this_time, '%y%m%d %H%M%S'))
        else:
            print('Date parser not up to date with possible vars')
            dt_list = None

        return np.asarray(dt_list)

class CTD_dir(WCO_obs_file):
    def __init__(self, dirname):
        all_files = os.listdir(dirname)
        dt_list = []
        observation_dict_list = []
        self._setup_possible_vars()

        for this_file in all_files:
            print('Processing file {}'.format(this_file))
            try:
                observation_dict_list.append(self._add_file(dirname + this_file, remove_undated=False))
                date_str = '20' + this_file[0:2] + '-' + this_file[2:4] + '-' + this_file[4:6]
                this_dt = dt.datetime.strptime(date_str, '%Y-%m-%d') + dt.timedelta(hours=12)
                dt_list.append(np.tile(this_dt, len(observation_dict_list[-1]['temp'])))

            except ValueError:
                print('Error in file {}'.format(this_file))
        # Flatten the list of dictionaries to one dictionary
        self.observation_dict = {this_key:np.hstack([this_dict[this_key] for this_dict in observation_dict_list]) for this_key in observation_dict_list[0]}
        self.observation_dict['dt_time'] = np.hstack(dt_list)

"""
Do the comparison

"""

class comp_data():
    def __init__(self, buoy_list, file_list_or_probe_dir, wco_database, max_time_threshold=dt.timedelta(hours=1), max_depth_threshold = 100, probe_depths=None):
        self.file_list_or_probe_dir
        self.database_obj = wco_database
        self.buoy_list = buoy_list

        self.model_data = {}
        if probe_depths is not None:
            for this_ind, this_buoy in enumerate(buoy_list):
                self.model_data[this_buoy]['depth'] = probe_depths[this_ind]

        self.observations = {}
        for this_buoy in self.buoy_list:
            self.observations[this_buoy] = {}

    def retrieve_file_data(self):
        pass

    def retrieve_obs_data(self, buoy_name, var, measurement_type=None):
        if not hasattr(self.model_date_mm):
            print('Retrieve model data first')
            return
        obs_dt, obs_raw = self.database_obj.get_obs_data(buoy_name, var, self.model_date_mm[0], self.model_date_mm[1], measurement_type)
        obs_depth = obs_raw[:,0]
        obs_var = obs_raw[:,1]
        self.observations[buoy_name][var] = obs_dict

    def get_comp_data_interpolated(self, buoy_name, var_list):
        if not hasattr(self, comp_dict):
            self.comp_dict = {}

        obs_dates = np.unique([this_date.date() for this_date in observations['time']])
        obs_comp = {}
        for this_ind, this_obs_time in enumerate(obs_dates):
            if this_obs_date >= model_time_mm[0].date() and this_obs_date <= model_time_mm[1].date():
                this_obs_choose = [this_time.date() == this_obs_date for this_time in self.observations[buoy_name]['dt_time']]
                t
                this_time_before, this_time_after = self.model_closest_both_times(this_obs_time)

                this_obs_deps = self.observations[buoy_name]['depth'][this_obs_choose]
                for var in var_list:
                    this_obs = self.observations[buoy_name][var][this_obs_choose]
                    this_model = np.squeeze(fvcom_data_reader.data.temp[this_time_close,...])
                    this_model_interp = np.squeeze(np.interp(this_obs_deps, model_depths, this_model))

                    try:
                        obs_comp[var].append(this_comp)
                    except KeyError:
                        obs_comp[var] = this_comp

                    max_obs_depth.append(np.max(this_obs_deps))

        self.comp_dict[buoy_name] = obs_comp

    def comp_data_nearest(self, buoy_name, var_list):
        pass

    def model_closest_time():
        pass

class comp_data_filereader(comp_data):
    def retrieve_file_data(self):
        where_str = 'buoy_name in ('
        for this_buoy in self.buoy_list:
            where_str+=this_buoy + ','
        where_str = where_str[0:-1] + ')'
        buoy_lat_lons = self.wco_database.select_query('sites', where_str, 'buoy_name, lon, lat')

        first_file = True
        model_all_dicts = {}

        for this_file in self.file_list:
            if first_file:
                fvcom_data_reader = FileReader(this_file, ['temp', 'salinity'])
                close_node = []
                for this_buoy_ll in buoy_lat_lons:
                    close_node.append()

                model_depths = fvcom_data_reader.grid.siglay * fvcom_data_reader.grid.h * -1

                for this_buoy in self.buoy_list:
                    model_all_dicts[this_buoy] = {'dt_time': mod_times, 'temp': mod_t_vals, 'salinity': mod_s_vals}

                first_file = False
            else:
                fvcom_data_reader = FileReader(this_file, ['temp', 'salinity'], dims={'node':[close_node]})
                for this_buoy in self.buoy_list:
                    model_all_dicts

        model_depths = fvcom_data_reader.grid.siglay * fvcom_data_reader.grid.h * -1
        for this_buoy in self.buoy_list:
            model_all_dicts[this_buoy]['depth'] = model_dp

        for this_buoy in self.buoy_list:
            self.model_data[this_buoy] = model_all_dicts[this_buoy]

    def model_closest_time(self, find_time):
        return closest_time

class comp_data_probe(comp_data):
    def retrieve_file_data():
        for this_buoy in self.buoy_list:
            t_filelist = []
            s_filelist = []
            for this_dir in self.file_or_probe_dir_list:
                t_filelist.append(this_dir + this_buoy + '_t1.dat')
                s_filelist.append(this_dir + this_buoy + '_s1.dat')
            mod_times, mod_t_vals, mod_pos = pf.read.read_probes(t_filelist, locations=True, datetimes=True)
            mod_times, mod_s_vals, mod_pos = pf.read.read_probes(s_filelist, locations=True, datetimes=True)
            model_dict = {'dt_time':mod_times, 'temp':mod_t_vals, 'salinity':mod_s_vals}
            self.model_data[this_buoy] = model_dict

def wco_model_comparison(model_file_list, obs_database_file):

    temp_comp = []
    sal_comp = []
    dates_comp = []
    max_obs_depth = []

    obs_dates = np.unique([this_date.date() for this_date in observations['time']])
    for this_ind, this_obs_date in enumerate(obs_dates):
        if this_obs_date >= model_time_mm[0].date() and this_obs_date <= model_time_mm[1].date():
            this_obs_choose = [this_time.date() == this_obs_date for this_time in observations['time']]
            this_obs_time = np.min(observations['time'][this_obs_choose])
            this_time_close = fvcom_data_reader.closest_time(this_obs_time)

            this_obs_deps = observations['h'][this_obs_choose]
            this_obs_temp = observations['temp'][this_obs_choose]
            this_obs_salinity = observations['salinity'][this_obs_choose]

            this_obs_temp_interp = np.squeeze(np.interp(model_depths, this_obs_deps, this_obs_temp))
            this_obs_salinity_interp = np.squeeze(np.interp(model_depths, this_obs_deps, this_obs_salinity))

            this_model_temp = np.squeeze(fvcom_data_reader.data.temp[this_time_close,...])
            this_model_salinity = np.squeeze(fvcom_data_reader.data.salinity[this_time_close,...])

            temp_comp.append(np.asarray([this_obs_temp_interp, this_model_temp]))
            sal_comp.append(np.asarray([this_obs_salinity_interp, this_model_salinity]))
            dates_comp.append(this_obs_time)
            max_obs_depth.append(np.max(this_obs_deps))

    ctd_comp = {'temp':temp_comp, 'salinity':sal_comp, 'dates':dates_comp, 'max_depth':max_obs_depth}

    observations = get_buoy_obs(obs_meta_data)

    if observations:
        buoy_temp_comp = []
        buoy_sal_comp = []
        buoy_dates_comp = []

        for this_ind, this_obs_time in enumerate(observations['time']):
            if this_obs_time >= model_time_mm[0] and this_obs_time <= model_time_mm[1]:
                this_time_close = fvcom_data_reader.closest_time(this_obs_time)
                buoy_temp_comp.append([observations['temp'][this_ind], fvcom_data_reader.data.temp[this_time_close,0]])
                buoy_sal_comp.append([observations['salinity'][this_ind], fvcom_data_reader.data.salinity[this_time_close,0]])
                buoy_dates_comp.append([this_obs_time, this_time_close])

        buoy_comp = {'temp':buoy_temp_comp, 'salinity':buoy_sal_comp, 'dates':buoy_dates_comp}

    else:
        buoy_comp = {}

    return ctd_comp, buoy_comp


#!/usr/bin/python3
""" Rectangle """
BaseGeometry = __import__('7-base_geometry').BaseGeometry

class Rectangle(BaseGeometry):
    """ Rectangle Class inheriting from BaseGeometry Class """
    def __init__(self, width, height):
        """ Init """
        try:
            super().integer_validator("width", width)
            self.__width = width
        except Exception as e:
            raise e

        try:
            super().integer_validator("height", height)
            self.__height = height
        except Exception as e:
            raise e

    def area(self):
        """ Returns area of Rectangle """
        return (self.__width * self.__height)

    def __str__(self):
        """ Returns str() representation of Rectangle """
        return "[Rectangle] {}/{}".format(self.__width, self.__height)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import division

from expyriment import control, stimuli, io, design, misc

class Grid:
    positions_locations = {"TopLeft": (-200,150), "TopMiddle": (0, 150), "TopRight": (200, 150), \
                        "Left": (-200, 0), "Right": (200, 0), \
                        "BottomLeft": (-200, -150), "BottomMiddle": (0, -150), "BottomRight": (200, -150)}

    positions_indices = ["TopLeft", "TopMiddle", "TopRight", "Left", "Right", "BottomLeft", "BottomMiddle", "BottomRight"]

    positions_indices2 = {"1": "TopLeft", "2": "TopMiddle", "3": "TopRight", "4": "Left", "5": "Right", \
                         "6": "BottomLeft", "7": "BottomMiddle", "8": "BottomRight"}

    def __init__(self, should_paint_grid):
        self.should_paint_grid = should_paint_grid
        self.line_horsizontal1 = stimuli.Line((-300, 80), (300, 80), 5)
        self.line_horsizontal2 = stimuli.Line((-300, -80), (300, -80), 5)
        self.line_vertical1 = stimuli.Line((-100, 200), (-100, -200), 5)
        self.line_vertical2 = stimuli.Line((100, 200), (100, -200), 5)
        self.cross = stimuli.FixCross((50, 50), (0, 0), 5)

        self.line_horsizontal1.preload()
        self.line_horsizontal2.preload()
        self.line_vertical1.preload()
        self.line_vertical2.preload()

    def paint_grid(self, canvas):
        time_delay = 0;

        time_delay += self.cross.plot(canvas)
        if self.should_paint_grid == 0:
            return 0;

        time_delay += self.line_vertical1.plot(canvas)
        time_delay += self.line_vertical2.plot(canvas)
        time_delay += self.line_horsizontal1.plot(canvas)
        time_delay += self.line_horsizontal2.plot(canvas)
        return time_delay;


class GsmUtil(object):
    gsm_unicode = (
        u"@£$¥èéùìòÇ\nØø\rÅåΔ_ΦΓΛΩΠΨΣΘΞ\x1bÆæßÉ !\"#¤%&'()*+,-./0123456789:;<=>?¡ABCDEFGHIJKLMNOPQRSTUVWXYZÄÖÑÜ¿abcdefghijklmnopqrstuvwxyzäöñüà")
    gsm_ext_unicode = u"^{}\\[~]|€"
    gsm_all_unicode = gsm_unicode + gsm_ext_unicode

    @staticmethod
    def encode_to_gsm(s, out_of_range_char_to='?', charset='utf8'):
        if isinstance(s, str):
            str_type = True
        # elif isinstance(s, unicode):
        #     str_type = False
        else:
            return s

        table = GsmUtil.gsm_all_unicode
        # s = unicode(s)
        res = ""
        for c in s:
            idx = table.find(c)
            if idx != -1:
                res += c
                continue
            else:
                res += out_of_range_char_to

        if str_type:
            return res.encode(charset)
        else:
            return res

def main():
    # print('Ç' * 160  # gsm
    # print('Î' * 160  # no gsm
    # print('Î' * 67  # no gsm
    input_string = r"""abcdef...z 0123456789 ÎÂÇÅÄÛşŞßÜÖîâçåäûüöĞğẞİıÈÉÊËÛÙÏÎÀÂÔèéêëûùïîàâôç ~!`&*;):)<>[]{}><=÷×+'"/ | \ \\ ※♂♀¿ ♡☆ 한글은 안 보임"""
    print(input_string)
    print(GsmUtil.encode_to_gsm(input_string))

if __name__ == '__main__':
    main()


import pandas as pd
#import numpy as np

#ward_g2v = cluster.ward_tree(g2v)
#ward_child = ward_g2v[0]
#n_samples = g2v.shape[0]
#genes = g2v.index

def ward_tree_2_label_mat(ward_child, genes):
    """Create a label matrix from ward tree children array.

    Create a matrix containing all possible clustering solutions from the Ward tree
    for all number of clusters `k` in `[1, n_samples]`.

    Parameters :
    ----------
    ward_child : int 2D array, shape=(n_samples-1, 2)
        Chilrend from Ward tree computation, each row gives the nodes being merged
        into a new node at each iteration.

    genes : sting array, shape=(n_samples,)
        List of all genes in the dataset.

    Returns :
    -------
    lab_df : pandas DataFrame, shape=(n_samples, n_samples)
        Data frame containing all possible solutions for the different cuts of the
        Ward tree in column. Column names gives the number of clusters in the
        clustering solution stored as a label list (corresponding column content).
    """
    n_samples = len(genes)
    # initialy 1 gene is 1 cluster in itself
    n_cl = n_samples
    lab = pd.Series([i for i in range(n_samples)],index=genes)

    # number of clusters at each iteration
    n_cl = [n_samples-i for i in range(len(ward_child)+1)]
    # matrix to fill with labels at each iteration
    lab_df = pd.DataFrame(0, index=genes, columns=n_cl)
    lab_df.iloc[:,0] = lab
    for i in range(len(ward_child)):
        a,b = ward_child[i]
        # at iteration i, clusters a and b are merged into one cluster
        new_node = n_samples + i # label of the new cluster
        # the new label list is inialised as equal to the last one
        new_lab = lab_df.iloc[:,i].copy(deep=True)

        # merged cluster lables are modified
        new_lab[lab_df.iloc[:,i]==a] = new_node
        new_lab[lab_df.iloc[:,i]==b] = new_node
        lab_df.iloc[:,i+1] = new_lab # new label list is added to the matrix

    return lab_df


import sys,os
commands = ['makemigrations', 'migrate', 'shell']
def managecommand(command):
    for com in command:
        os.system(f'python manage.py {com}')

cd =  {1:'makemigrations',2:"migrate", 3:"shell",4:'runserver'}
buc = []
for arg in sys.argv:
    buc.append(arg)
buc = buc[1:]
print(buc)
if buc[0] == '1':
    managecommand([cd[1], buc[0]])
elif buc[0] == '2':
    managecommand([cd[2], buc[0]])
elif buc[0] == "3":
    managecommand([cd[3], buc[0]+" "+int(buc[1])])
elif buc[0] == "4":
    managecommand([cd[4], buc[0]])
elif buc[0] == "all":
    managecommand([cd[1], buc[0]])
    managecommand([cd[2], buc[0]])
else:print('not found')


from django.contrib import admin as dj_admin
from django.contrib import admin
from myapp.models import Project, ModelGroup, AutodeskExtractorRule, BimModel, Job
from myapp.models import URN, ActiveLink

class WorkAdmin(dj_admin.ModelAdmin):
    list_display = ("id", "name")

class ProjectAdmin(admin.ModelAdmin):
    list_display = (
        "name",
        "description",
        "created_at",
        "updated_at",
        "created_by",
        "updated_by",
        "deleted_at",
        "deleted_by",
    )

class ModelGroupAdmin(admin.ModelAdmin):
    list_display = (
        "name",
        "description",
        "created_at",
        "updated_at",
        "created_by",
        "updated_by",
        "deleted_at",
        "deleted_by",
    )

class AutodeskExtractorRuleAdmin(admin.ModelAdmin):
    list_display = (
        "project",
        "model_group",
        "name",
        "description",
        "created_at",
        "updated_at",
        "created_by",
        "updated_by",
        "deleted_at",
        "deleted_by",
    )

class BimModelAdmin(admin.ModelAdmin):
    list_display = (
        "project",
        "model_group",
        "name",
        "description",
        "created_at",
        "updated_at",
        "created_by",
        "updated_by",
        "deleted_at",
        "deleted_by",
    )

class JobAdmin(admin.ModelAdmin):
    list_display = (
        "project",
        "task_type",
        "created_at",
        "task_id",
    )

admin.site.register(Project, ProjectAdmin)
admin.site.register(ModelGroup, ModelGroupAdmin)
admin.site.register(AutodeskExtractorRule, AutodeskExtractorRuleAdmin)
admin.site.register(BimModel, BimModelAdmin)
admin.site.register(Job, JobAdmin)

dj_admin.site.register(ActiveLink)
dj_admin.site.register(URN)


import os
import time
import csv
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from pathlib import Path
import Assistant

import sys
if '..\\..' not in sys.path:
    sys.path.append("..\\..")

if 'Assistant' not in sys.modules:
    import Assistant

class Spain:

    def __init__(self):
        self.casosActivos = None
        self.casosDiarios = None
        self.DecesosDiarios = None
        self.RecuperacionesDiarias = None
        self.all_data = None
        self.pollution = Assistant.Pollution("madrid")

    def get_all_data(self):
        import Assistant
        self.casosActivos = Assistant.getCases('Spain','Active Cases')
        self.casosDiarios = Assistant.getCases('Spain','Daily New Cases')
        self.DecesosDiarios = Assistant.getCases('Spain','Daily New Deaths')
        self.RecuperacionesDiarias = Assistant.getCases('Spain','Newly Recovered')
        self.dates = self.casosActivos['Date']
        self.all_data = self.casosActivos
        self.all_data['Daily New Cases'] = self.casosDiarios['Daily New Cases']
        self.all_data['Daily New Deaths'] = self.DecesosDiarios['Daily New Deaths']
        self.all_data['Newly Recovered'] = self.RecuperacionesDiarias['Newly Recovered']
        self.all_data['PM2.5'] = self.pollution.extract_data('PM2.5', self.dates)
        self.all_data['PM10'] = self.pollution.extract_data('PM10', self.dates)
        self.all_data['O3'] = self.pollution.extract_data('O3', self.dates)
        self.all_data['NO2'] = self.pollution.extract_data('NO2', self.dates)
        self.all_data['SO2'] = self.pollution.extract_data('SO2', self.dates)
        self.all_data['CO'] = self.pollution.extract_data('CO', self.dates)
        print(self.all_data)


import json
import pandas as pd
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error, r2_score
import joblib
import sys

def train_model():
    jsondata = json.loads(sys.argv[1])

    # Create a dictionary to store models for each variable
    models = {}

    for variable_data in jsondata:
        variable_name = variable_data['dataName']
        df = pd.DataFrame(variable_data['dataSource'])

        df['timeStamp'] = pd.to_datetime(df['timeStamp'])

        df['hour'] = df['timeStamp'].dt.hour
        df['minute'] = df['timeStamp'].dt.minute
        df['second'] = df['timeStamp'].dt.second

        target = 'dataValue'

        X = df[['hour', 'minute', 'second']]
        y = df[target]

        model = LinearRegression()
        model.fit(X, y)

        models[variable_name] = model

        joblib.dump(model, f'trained_model_{variable_name}.joblib')

    response_data = {"models": models, "status": "trained"}

    try:
        print(json.dumps(response_data))
    except Exception as e:
        print(f"An error occurred: {e}")

train_model()


class Solution:
    def intToRoman(self, num: int) -> str:
        roman = {1: "I", 4: "IV", 5: "V", 9: "IX", 10: "X", 40: "XL", 50: "L",
                 90: "XC", 100: "C", 400: "CD", 500: "D", 900: "CM", 1000: "M"}
        roman = dict(reversed(list(roman.items())))
        res = ""
        for key, value in roman.items():
            if not num:
                break
            div = num//key
            res += div*value
            num %= key
        return res


from flask import Flask, request, jsonify
from tf_idf import get_text, scrape_google, tf_idf_analysis

app = Flask(__name__)

@app.route('/get_text')
def get_text_endpoint():
    url = request.args.get('url')
    text = get_text(url)
    return jsonify(text=text)

@app.route('/scrape_google')
def scrape_google_endpoint():
    query = request.args.get('query')
    links = scrape_google(query)
    return jsonify(links=links)

@app.route('/tf_idf_analysis')
def tf_idf_analysis_endpoint():
    keyword = request.args.get('keyword')
    result = tf_idf_analysis(keyword)
    return jsonify(result=result)

if __name__ == '__main__':
    app.run(debug=True)


hint = ""

def display_user_position(player):
    print("Current Floor:  " + str(player[0] + 1) + ", Current Room: " + str(player[1] + 1))
    return player

def display_user_items(items):
    print("Current items: " + str(items))
    return items

def is_invalidate_command(command, valid_commands):
    if command not in {"left", "right", "up", "down", "grab", "fight", "help"} or \
            command not in set(valid_commands):
        set_hint("Hint: Invalid command, try again!")
        return True
    return False

def update_user_position(command, valid_commands, player, floors):
    if is_invalidate_command(command, valid_commands) or command not in {"left", "right", "up", "down"}:
        return display_user_position(player)

    position_changed = True
    if command == "left" and player[1] - 1 >= 0:
        player[1] = player[1] - 1
    elif command == "right" and player[1] + 1 < len(floors[0]):
        player[1] = player[1] + 1
    elif command == "up" and player[0] + 1 < len(floors):
        player[0] = player[0] + 1
    elif command == "down" and player[0] - 1 >= 0:
        player[0] = player[0] - 1
    else:
        position_changed = False

    if not position_changed:
        set_hint("Hint: No more room in direction - " + command)

    return display_user_position(player)

def update_user_items(encounter, command, valid_commands, items, player, floors):
    if is_invalidate_command(command, valid_commands) or command not in {"grab", "fight"}:
        return display_user_items(items), "in progress", floors

    status = "in progress"

    if command == "grab":
        if len(items) >= 3:
            set_hint("Hint: Cannot grab. You can only hold maximum 3 items!")
            return display_user_items(items), status, floors
        items.append(encounter)
        floors[player[0]][player[1]] = ""
    else:
        status = "loss"
        if encounter == "monster":
            for i in items[:]:
                if i == "sword":
                    items.remove(i)
                    status = "success"
                    floors[player[0]][player[1]] = ""
                    break
        elif encounter == "boss-monster":
            for i in items[:]:
                if i == "sword":
                    items.remove(i)
                    break
            for i in items[:]:
                if i == "magic-stone":
                    items.remove(i)
                    status = "win"
                    floors[player[0]][player[1]] = ""
                    break

    return display_user_items(items), status, floors

def reverse(prev_command):
    res = ""
    if prev_command == "left":
        res = "right"
    elif prev_command == "right":
        res = "left"
    elif prev_command == "up":
        res = "down"
    elif prev_command == "down":
        res = "up"
    return res

# left, right, up, down, grab, fight, help.
def display_commands(encounter, prev_command):
    commands = []
    if encounter == "sword" or encounter == "magic-stone":
        commands.extend(["left", "right", "grab", "help"])
    elif encounter == "up-staircase":
        commands.extend(["left", "right", "up", "help"])
    elif encounter == "down-staircase":
        commands.extend(["left", "right", "down", "help"])
    elif encounter == "monster" or encounter == "boss-monster":
        if prev_command != "":
            commands.append(reverse(prev_command))
        commands.extend(["fight", "help"])
    else:
        commands.extend(["left", "right", "help"])

    print(commands)

    return commands

def help(encounter, command, valid_commands, items, player, floors):
    if is_invalidate_command(command, valid_commands) or command not in {"help"}:
        return items
    pass

def get_hint():
    global hint
    ret = hint
    set_hint("")
    return ret

def set_hint(str):
    global hint
    hint = str
    print(hint)
    pass

def game_cli():
    floors = [["", "sword", "up-staircase", "sword", "monster"],
              ["magic-stone", "monster", "up-staircase", "monster", "down-staircase"],
              ["down-staircase", "", "sword", "", "boss-monster"]]
    player = [0, 0]
    items = []
    status = "in progress"
    current_command = ""

    while status != "win" and status != "loss":
        print("-- This room has: [" + floors[player[0]][player[1]] + "] --")
        commands = display_commands(floors[player[0]][player[1]], current_command)
        current_command = raw_input("From above select your command: ")
        print("\n-----------------")
        items, status, floors = update_user_items(floors[player[0]][player[1]],
                                                  current_command, commands, items, player, floors)
        player = update_user_position(current_command, commands, player, floors)

    print("Game Over! You " + status)
    print("Thanks for playing!")

if __name__ == "__main__":
    game_cli()


class Solution:
    def lengthOfLongestSubstring(self, s: str) -> int:

        # Time: O(n) | Space: O(n)

        l, r = 0, 0
        hashset = set()
        max_len = 0

        while r < len(s):
            if s[r] not in hashset:
                hashset.add(s[r])

                max_len = max(max_len, r - l + 1)
                r += 1

            else:
                hashset.remove(s[l])
                l += 1

        return max_len

        """
        Intuition

        Use a hashset.

        Iterate over s and store in set.
        IF not in set add cur num.
        find the max_len.
        ELSE:  remove l pointer from set.
        l += 1

        return max_len
        """

3. Longest Substring Without Repeating Characters.py


from tests.baseTest import *
from lib.random_generator import RandomGenerator
from lib.api_requests import RequestManager
from grappa import should
from api_endpoints.signup_endpoint import SignupMethods
from api_endpoints.threads_endpoint import ThreadsMethods
from lib.data_encoder import Encoder

class ViewThreadsTest(BaseTest):
    rand = RandomGenerator()
    rm = RequestManager()
    encoder = Encoder()

    @classmethod
    def setUpClass(cls):
        BaseTest.setUpClass()
        account_data = SignupMethods().create_test_account(generate_fields=True)[0]
        data_to_encode = account_data['username'] + ':' + account_data['password']
        encoded_credentials = cls.encoder.encode_data(data_to_encode)
        cls.thread_auth_headers = {'Authorization': 'Basic ' + encoded_credentials}
        sample_thread = cls.rand.generate_random_string(10)
        ThreadsMethods().create_sample_thread(authorization=cls.thread_auth_headers, thread_name=sample_thread,
                                              private=False)

    def setUp(self):
        BaseTest.setUp(self)
        self.threads_url = self.CONFIG['API_ADDRESS'] + '/threads'

    def test_01_get_last_threads(self):
        logging.info('Trying to get last threads')
        result = ThreadsMethods().get_threads(authorization=self.thread_auth_headers)
        logging.info('Server responded with %s' % result)
        result['code'] | should.be.equal.to(200)
        result['response']['itemsFound'] | should.be.a(int)
        result['response']['limit'] | should.be.a(int)
        result['response']['limit'] | should.be.equal.to(100)
        len(result['response']['items']) | should.be.higher.than(0)
        len(result['response']['items']) | should.be.lower.than(101)
        result['response']['items'][0]['createdAt'] | should.be.a(int)
        result['response']['items'][0]['updatedAt'] | should.be.a(int)
        result['response']['items'][0]['id'] | should.be.a(str)
        result['response']['items'][0]['id'] | should.not_be.none
        result['response']['items'][0]['modelType'] | should.be.equal.to('ThreadModel')
        result['response']['items'][0]['name'] | should.be.a(str)
        result['response']['items'][0]['owner'] | should.be.a(str)
        result['response']['items'][0]['users'] | should.be.a(list)
        result['response']['items'][0]['private'] | should.be.a(bool)
        result['response']['items'][0]['deleted'] | should.be.a(bool)

    def test_02_get_last_100_threads(self):
        logging.info('Generating 100 threads')
        for i in range(0, 102):
            sample_thread = self.rand.generate_random_string(10)
            ThreadsMethods().create_sample_thread(authorization=self.thread_auth_headers, thread_name=sample_thread,
                                                  private=False)
        logging.info('Trying to get last 100 threads')
        result = ThreadsMethods().get_threads(authorization=self.thread_auth_headers)
        logging.info('Server responded with %s' % result)
        result['code'] | should.be.equal.to(200)
        len(result['response']['items']) | should.be.equal.to(100)


import random
import tensorflow as tf
from AI.agent import Agent
from AI.ROSEnvironment import ArmControlPlateform
from config import get_config

flags = tf.app.flags

# Model
flags.DEFINE_string('model', 'DQN', 'Type of model')
# Environment
#flags.DEFINE_string('env_name', 'Acrobot-v1', 'The name of gym environment to use')
# Etc
#flags.DEFINE_boolean('display', True, 'Whether to do display the game screen or not')
flags.DEFINE_integer('random_seed', 123, 'Value of random seed')

FLAGS = flags.FLAGS

# Set random seed
tf.set_random_seed(FLAGS.random_seed)
random.seed(FLAGS.random_seed)

def main(_):
    config = get_config(FLAGS) or FLAGS
    env = ArmControlPlateform(config)
    for _ in range(1):
        env.reset()
        agent = Agent(config, env)
        agent.activate(learn=True)

def test():
    config = get_config(FLAGS) or FLAGS
    env = ArmControlPlateform(config)
    env.reset()
    for i in range(30):
        sleep(0.1)
        action = env.action_space.sample()
        ob,re,ter = env.step(action)
        print(ob,re,ter)

if __name__ == '__main__':
    tf.app.run()
    #test()


from typing import List, Tuple

import numpy as np
import torch
import torch.nn.functional as F
from torch import Tensor, nn

def get_cnn_output_dim(
    input_size: int, conv_kernel_size: int, padding_size: int, conv_stride_size: int
) -> int:
    return (input_size + 2 * padding_size - conv_kernel_size) / conv_stride_size + 1

def get_pool_output_dim(
    input_size, pool_kernel_size: int, pool_stride_size: int
) -> int:
    return np.floor((input_size - pool_kernel_size) / pool_stride_size + 1)

def get_cnn_layer_output_dim(
    n_layers: int,
    input_size: int,
    conv_kernel_size: int,
    padding_size: int = 0,
    conv_stride_size: int = 1,
    pool_kernel_size: int = 2,
    pool_stride_size: int = 2,
) -> int:
    if n_layers > 1:
        cnn_output = get_cnn_output_dim(
            input_size, conv_kernel_size, padding_size, conv_stride_size
        )
        pool_output = get_pool_output_dim(
            cnn_output, pool_kernel_size, pool_stride_size
        )
        n_layers -= 1
        return int(
            get_cnn_layer_output_dim(
                n_layers,
                pool_output,
                conv_kernel_size,
                padding_size,
                conv_stride_size,
                pool_kernel_size,
                pool_stride_size,
            )
        )
    else:
        cnn_output = get_cnn_output_dim(
            input_size, conv_kernel_size, padding_size, conv_stride_size
        )
        pool_output = get_pool_output_dim(
            cnn_output, pool_kernel_size, pool_stride_size
        )
        return int(pool_output)

class MultivariateMLP(nn.Module):
    def __init__(
        self,
        input_dimension: Tuple,
        in_channels: int,
        n_outputs: int,
    ):
        super().__init__()
        self.in_channels = in_channels
        self.encoder = nn.Sequential(
            FlattenMLP(),
            nn.Linear(input_dimension[0] * input_dimension[1] * 1 * in_channels, 564),
            # nn.BatchNorm1d(256),
            nn.ReLU(),
            nn.Dropout(0.5),
        )
        self.linear_classifiers = [
            nn.Linear(564, n_outputs) for _ in range(in_channels)
        ]

    def forward(self, x):
        out = self.encoder(x)
        outputs = [
            F.softmax(classifier(out), dim=-1) for classifier in self.linear_classifiers
        ]
        return outputs

class Flatten(nn.Module):
    """A custom layer that views an input as 1D."""

    def forward(self, input):
        return input.view(input.size(0), -1)

class FlattenMLP(nn.Module):
    """A custom layer that views an input as 1D."""

    def forward(self, input):
        return input.reshape(-1)

class MultivariateCNN(nn.Module):
    """https://github.com/ArminBaz/UTK-Face/blob/master/src/MultNN.py"""

    def __init__(
        self,
        input_dimension: Tuple,
        in_channels: int,
        n_outputs: int,
        n_cnn_layers: int = 2,
        conv_kernel_size: int = 2,
        pool_kernel_size: int = 2,
        conv_channels_1: int = 256,
        conv_channels_2: int = 512,
        linear_hidden_cells: int = 256,
        linear_dropout: float = 0.5,
    ):
        super(MultivariateCNN, self).__init__()
        self.in_channels = in_channels
        linear_dim1 = get_cnn_layer_output_dim(
            n_layers=n_cnn_layers,
            input_size=input_dimension[0],
            conv_kernel_size=conv_kernel_size,
            pool_kernel_size=pool_kernel_size,
        )
        linear_dim2 = get_cnn_layer_output_dim(
            n_layers=n_cnn_layers,
            input_size=input_dimension[1],
            conv_kernel_size=conv_kernel_size,
            pool_kernel_size=pool_kernel_size,
        )
        self.encoder = nn.Sequential(
            nn.Conv2d(
                in_channels=in_channels,
                out_channels=conv_channels_1,
                kernel_size=conv_kernel_size,
            ),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=pool_kernel_size),
            nn.Conv2d(
                in_channels=conv_channels_1,
                out_channels=conv_channels_2,
                kernel_size=conv_kernel_size,
            ),
            # nn.BatchNorm2d(64),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=pool_kernel_size),
            #               nn.Conv2d(in_channels, out_channels=8, kernel_size=conv_kernel_size),
            #               nn.ReLU(),
            #               nn.MaxPool2d(kernel_size=pool_kernel_size),
            Flatten(),
            nn.Linear(linear_dim1 * linear_dim2 * conv_channels_2, linear_hidden_cells),
            # nn.BatchNorm1d(256),
            nn.ReLU(),
            nn.Dropout(linear_dropout),
        )
        self.linear_classifiers = [
            nn.Linear(linear_hidden_cells, n_outputs) for _ in range(in_channels)
        ]

    def forward(self, x):
        out = self.encoder(x)
        outputs = [
            F.softmax(classifier(out), dim=-1) for classifier in self.linear_classifiers
        ]
        return outputs


import pygame
from random import randint

pygame.init()

#game window
WIDTH = 600
HEIGHT = 400
win = pygame.display.set_mode((WIDTH, HEIGHT))
pygame.display.set_caption("Shot Clock")
pygame.display.set_icon(pygame.image.load('icon.png'))

#gmae font
pygame.font.init()
GAME_FONT = pygame.font.Font('gamera.TTF', 35)

#game start and timing variables
game_start = False  # boolean that controls main loop
start_time = 0  # Start time for game timer
TIME_ALLOWED = 60  # amount of time player has (1 minute)

#ball properties
MAX_BALL_SPEED = 17
MIN_BALL_SPEED = 4

#game colors
BLACK = (0, 0, 0)
ORANGE = (252, 102, 0)
WHITE  = (255, 255, 255)

#sprites
HOOP = pygame.image.load('hoop.png')
BASKETBALL = pygame.image.load('ball.png')

class Ball:
    def __init__(self,x, y,vel):
        self.x = x
        self.y = y
        self.vel = vel

    def draw_ball(self, surface):
        surface.blit(BASKETBALL,(self.x, self.y))

    def update_position(self):
        self.y += self.vel

    def is_off_screen(self):
        return self.y > 420

# game start loop
def start_game():
    global game_start
    global start_time

    line1 = GAME_FONT.render("Make Shots! You Have 60s!!", True, ORANGE)
    line2 = GAME_FONT.render("Press SPACE To Start!!", True, ORANGE)
    line3 = GAME_FONT.render("ESC to Exit  R to Reset", True, ORANGE)
    line4 = GAME_FONT.render("A:Left   D:Right", True, ORANGE)

    win.blit(line1, (50, 50))
    win.blit(line2, (80, 130))
    win.blit(line3, (80, 210))
    win.blit(line4, (150, 290))

    pygame.display.update()

    while not game_start:

        if game_start:
            break

        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                pygame.quit()
                exit()
            elif event.type == pygame.KEYDOWN:
                if event.key == pygame.K_SPACE:
                    game_start = True
                    start_time = pygame.time.get_ticks()
                if event.key == pygame.K_ESCAPE:
                    pygame.quit()
                    exit()

def is_time_left():
    return TIME_ALLOWED > int((pygame.time.get_ticks() - start_time) / 1000)

game_balls = [
    Ball(randint(30, 570), 0, randint(1, MAX_BALL_SPEED)),
    Ball(randint(30, 570), 0, randint(1, MAX_BALL_SPEED)),
    Ball(randint(30, 570), 0, randint(1, MAX_BALL_SPEED)),
    Ball(randint(30, 570), 0, randint(1, MAX_BALL_SPEED))
]

def new_ball(index):
    global MAX_BALL_SPEED
    global MIN_BALL_SPEED

    game_balls[index].x = randint(30, 570)
    game_balls[index].y = 0
    game_balls[index].vel = randint(MIN_BALL_SPEED, MAX_BALL_SPEED)

    if MIN_BALL_SPEED < MAX_BALL_SPEED:
        MIN_BALL_SPEED += 1

def main():

    global game_start
    global start_time

    hoop_x = 260
    hoop_y = 360

    hitbox_x = 280
    hitbox_y = 380

    clock = pygame.time.Clock()
    SCORE = 0  # player score

    start_game()
    if not game_start:
        print("\nYou Didn't Press Space :( ")
    else:
        print("\nGame starting....")

    while game_start and is_time_left():
        TIME_LEFT = TIME_ALLOWED -  int((pygame.time.get_ticks() - start_time) / 1000)
        clock.tick(30)

        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                game_start = False

        keys = pygame.key.get_pressed()

        # if press left(A):
        # move the basket to left (-1 to y value)
        if keys[pygame.K_a]:
            hoop_x -= 40
            hitbox_x -= 40
            # after moving the platform. if its x or y match those of the platform
            # they have collided/touched (the ball is caught) add one

        # if press right(D):
        # move the basket to the right(+1 to y value)
        if keys[pygame.K_d]:
            hoop_x += 30
            hitbox_x += 30

        # if press escape(ESC)
        # quit game
        if keys[pygame.K_ESCAPE]:
            print("Exiting Game......")
            game_start = False

        # if press reset(R)
        # resets timer to 60s
        if keys[pygame.K_r]:
            start_time = pygame.time.get_ticks()
            SCORE = 0

        win.fill((0, 0, 0))

        # basket/hoop
        win.blit(HOOP, (hoop_x,hoop_y))

        # balls
        # Loop the same (4) balls with a new speed and position each time they come back.
        # ball is looped after it registers as a point or falls off screen
        for i in range(4):

            if game_balls[i].is_off_screen():
                new_ball(i)

            game_balls[i].draw_ball(win)
            game_balls[i].update_position()

            # if any part of the hoop/platform is touched by a ball as a collision
            # ball is cleared from the display and new ball appears
            if (hitbox_y) < game_balls[i].y < (hitbox_y + 20) and (hitbox_x) < \
                    game_balls[i].x < (hoop_x + 120):
                    # hitbox is simplified to be 120x20 rectangle starting from the top left corner of the rim
                    # if a ball falls within this region it is considered a score.
                SCORE += 1
                new_ball(i)

        # prints time on screen
        time_tracker = GAME_FONT.render(F"Time:{TIME_LEFT}", True, ORANGE)
        win.blit(time_tracker, (455, 0))

        # prints score on screen
        score_tracker = GAME_FONT.render(F"Score:{SCORE}", True, ORANGE)
        win.blit(score_tracker, (0, 0))

        pygame.display.update()

    # End Game Loop
    # End screen lasts for 10 secs and exits automatically
    close_window = False
    while not close_window:

        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                pygame.quit()
                exit()

            elif event.type == pygame.KEYDOWN:
                if event.key == pygame.K_ESCAPE:
                    close_window = True

                if event.key == pygame.K_r:
                    win.fill(BLACK)
                    main()

        win.fill(BLACK)

        game_over = GAME_FONT.render("Game Over!", True, ORANGE)
        win.blit(game_over, (215, 50))

        final_score = GAME_FONT.render(F"Final Score: {SCORE} ", True, WHITE)
        win.blit(final_score, (185, 130))

        new_game = GAME_FONT.render("Press R to Start New Game", True, ORANGE)
        win.blit(new_game, (50,210))

        exit_game = GAME_FONT.render("Press ESC to Close Game", True, ORANGE)
        win.blit(exit_game, (70, 290))

        pygame.display.update()

    # gg
    pygame.quit()

if __name__ == "__main__":
     main()


# with open("binary", 'bw') as bin_file:             # this is writing into a binary file
#     for i in range(17):
#         bin_file.write(bytes([i]))                 # this is passing a single number to bytes function

# above code can also be written as shown below
# with open("binary", 'bw') as bin_file:
#     bin_file.write(bytes(range(17)))
#
# with open("binary", 'br') as binfile:              # this is reading from the binary file
#     for b in binfile:
#         print(b)

a = 65534      # FF FE
b = 65535      # FF FF
c = 65536      # 00 01 00 00
x = 2998302    # 00 2D C0 1E

with open("binary2", 'bw') as bin_file:
    bin_file.write(a.to_bytes(2, 'big'))
    bin_file.write(b.to_bytes(2, 'big'))
    bin_file.write(c.to_bytes(4, 'big'))
    bin_file.write(x.to_bytes(4, 'big'))
    bin_file.write(x.to_bytes(4, 'little'))

with open("binary2", 'br') as binfile:
    e = int.from_bytes(binfile.read(2), 'big')
    print(e)
    f = int.from_bytes(binfile.read(2), 'big')
    print(f)
    g = int.from_bytes(binfile.read(4), 'big')
    print(g)
    h = int.from_bytes(binfile.read(4), 'big')
    print(h)
    i = int.from_bytes(binfile.read(4), 'big')
    print(i)


import numpy as np

class TrackBall( object ):
    """
    Class used for controlling the rotation of the scene via mouse or joystick, by generating the virtual trackball effect

    Constructor

        ========== =========================
        Arguments
        ========== =========================
        w_size     size of the window
        ========== =========================

        Example:
            Event On click: ::

                ( x , y ) = get_click_coords_on_window()
                trk.track_ball_mapping( [ x , y ] )

            Event On Move: ::

                ( x , y ) = get_current_coords_on_window()
                ( rot_axis , rot_angle ) = trk.on_move( [ x , y ] )
                glRotatef( rot_angle , rot_axis[0] , rot_axis[1] , rot_axis[2] )
    """
    def __init__( self , w_size ):

        self.__v     = np.array( [ 0.0 , 0.0 , 0.0 ] )
        self.__v_old = np.array( [ 0.0 , 0.0 , 0.0 ] )

        self.__win_size = ( 800 , 600 )

        self.__win_width = 800
        self.__win_height = 600

        self.win_size = w_size

    def get_V( self ):
        return self.__v

    def set_V( self , v ):
        self.__v = v

    V = property( get_V , set_V )

    def get_win_size( self ):
        return ( self.__win_width , self.__win_height )

    def set_win_size( self , w_size ):
        self.__win_width  = w_size[0]
        self.__win_height = w_size[1]

    win_size = property( get_win_size , set_win_size )

    def track_ball_mapping( self , point ):
        """
        Function to be called after a click on the mouse or at beginnig of the rotation, it takes the current coordinates of the pointer.
        """
        self.__v_old[:] = self.__v[:]

        self.__v[0] = ( 2.0 * point[0]   - self.win_size[0] ) / self.win_size[0]
        self.__v[1] = ( self.win_size[1] - 2.0 * point[1] )   / self.win_size[1]

        self.__v[2] = 0.0

        d = np.linalg.norm( self.__v )

        if d > 1.0 :
            self.__v[:] = self.__v[:] / d

        tb_radius = 4.0

        self.__v[:] = self.__v[:] * tb_radius * 0.999

        self.__v[2] = np.sqrt( tb_radius**2 - self.__v[0]**2 - self.__v[1]**2 )

        self.__v[:] = self.__v / np.linalg.norm( self.__v )

    def on_move( self , point ):
        """
        function to be called when the mouse is moved. argument requires the coordinates of the mouse pointer and it returns the axis of rotation and angle.
        """

        self.track_ball_mapping( point )

        direction = self.__v - self.__v_old

        velocity = np.linalg.norm( direction )

        rot_axis = np.cross( self.__v_old , self.__v )
        rot_angle = velocity * 400.0

        rot_axis = rot_axis / np.linalg.norm( rot_axis )

        return ( rot_axis , rot_angle )

    def on_joystick( self , jaxes ):
        """
        Given the axes ( x and y ) of the joystick; it returns the axis and the angle of rotation.
            Example: ::

                ( rot_axis , rot_angle ) = trk.on_joystick( [ x , y ] )
        """
        ws = self.win_size

        jd = 400

        self.track_ball_mapping( ( ws[0]/2 , ws[1]/2 ) )
        return self.on_move( ( ws[0]/2 + jaxes[0]/jd , ws[1]/2 + jaxes[1]/jd ) )


# --------------------------------
# Name: plot_sbcape_loop.py
# Author: Robert M. Frost
# NOAA Global Systems Laboratory
# Created: 26 June 2023
# Purpose: Loop to plot 2m dew point
# and wind barb comparisons at
# different times during forecast runs
# --------------------------------
from UFSutils import read_grib
import matplotlib.pyplot as plt
from matplotlib import rc
import cartopy.crs as ccrs
import cartopy.feature as cpf
import seaborn
import numpy as np
import geopandas as gpd
# --------------------------------
# settings

# date being plot
date = "2023041912" #YYYMMDDHH
# hour forecast was initialized (UTC)
init = 12
# directory where hrrr grib data are located
dgrib_h = f"/scratch2/BMC/fv3lam/Robby.Frost/expt_dirs/{date}_3km_hrrrphys/{date}/postprd/"
# directory where rap grib data are located
dgrib_r = f"/scratch2/BMC/fv3lam/Robby.Frost/expt_dirs/{date}_3km_rapphys/{date}/postprd/"
# natlev or prslev
nat_prs = "natlev"
# message number for dew point
mn_td2m = 1358
# message number for u at 10m
mn_u10 = 1364
# message number for v at 10m
mn_v10 = 1365
# directory for figure to be output
figdir = f"/scratch2/BMC/fv3lam/Robby.Frost/figures/{date}/td2m/"
# --------------------------------
# plotting setup
rc('font',weight='normal',size=12.5)
# rc('text',usetex='True')
rc('figure',facecolor='white')
# --------------------------------
# NWS dew point colorbar

import matplotlib.colors as colors
a = np.array([0,10,20,30,40,45,50,55,60,65,70,75,80])
# Normalize the bin between 0 and 1 (uneven bins are important here)
norm = [(float(i)-min(a))/(max(a)-min(a)) for i in a]
# Color tuple for every bin
C = np.array([[59,34,4],
            [84,48,5],
            [140,82,10],
            [191,129,45],
            [204,168,84],
            [223,194,125],
            [230,217,181],
            [211,235,231],
            [169,219,211],
            [114,184,173],
            [49,140,133],
            [1,102,95],
            [0,60,48],
            [0,41,33]])
# Create a tuple for every color indicating the normalized position on the colormap and the assigned color.
COLORS = []
for i, n in enumerate(norm):
    COLORS.append((n, np.array(C[i])/255.))
# Create the colormap
cmap = colors.LinearSegmentedColormap.from_list("dewpoint", COLORS)
# --------------------------------
# loop over time

for hr in range(0,37):
    print(f"Hour {hr}")

    # read in dew point
    hrrr, td2m_h, lat, lon, valid_date = read_grib(init, hr, dgrib_h, nat_prs, mn_td2m, ret_type=0)
    rap, td2m_r, lat, lon, valid_date = read_grib(init, hr, dgrib_r, nat_prs, mn_td2m, ret_type=0)
    # convert to fahrenheit (superior unit of temperature)
    td2m_h = (td2m_h.values - 273.15) * (9/5) + 32
    td2m_r = (td2m_r.values - 273.15) * (9/5) + 32

    # read in 10m wind
    u10_h = hrrr[mn_u10].values
    v10_h = hrrr[mn_v10].values
    u10_r = rap[mn_u10].values
    v10_r = rap[mn_v10].values

    # convert 10m wind to knots
    u10_h = u10_h * 1.944
    v10_h = v10_h * 1.944
    u10_r = u10_r * 1.944
    v10_r = v10_r * 1.944
    # --------------------------------
    # Plot dew point comparison
    print(f"Creating 1 x 2 Td2m Plot")

    # Define your custom colorbar bounds
    cbar_min = 0
    cbar_max = 80.1
    # levels for sbcape to be plot
    clevs = np.arange(cbar_min, cbar_max, 2)

    # create plot
    fig, ax = plt.subplots(ncols=2, subplot_kw={'projection': ccrs.PlateCarree()},
                        figsize=(16,10), constrained_layout=True)

    # plot HRRR
    c0 = ax[0].contourf(lon, lat, td2m_h, clevs,
                        transform=ccrs.PlateCarree(),
                        cmap=cmap, extend="both")
    # plot RAP
    c1 = ax[1].contourf(lon, lat, td2m_r, clevs,
                        transform=ccrs.PlateCarree(),
                        cmap=cmap, extend="both")

    # mapping
    plt_area = [-101, -94, 30, 37.5] # W, E, S, N
    for i, iax in enumerate(ax):
        iax.coastlines()
        iax.add_feature(cpf.BORDERS)
        iax.add_feature(cpf.STATES)
        iax.set_extent(plt_area)
        # Load the json file with county coordinates
        geoData = gpd.read_file('https://raw.githubusercontent.com/holtzy/The-Python-Graph-Gallery/master/static/data/US-counties.geojson')
        geoData.plot(ax=iax, color="none", lw=0.3, aspect=1)

    # set title
    ax[0].set_title(f"No-GF F0{hr},  Valid {valid_date} UTC")
    ax[1].set_title(f"GF F0{hr},  Valid {valid_date} UTC")

    # Add colorbar
    cbar = fig.colorbar(c1, ax=ax, orientation='horizontal', extend=True, pad=0.03, aspect=50)
    cbar.set_label('2m Dew Point Temperature [$^{\circ}$F]')
    cbar.set_ticks(np.arange(cbar_min, cbar_max, 10))

    # Wind barbs
    spacing=25 #barbspacing (smaller if zoomed in)
    ax[0].barbs(lon[::spacing,::spacing], lat[::spacing,::spacing],
                u10_h[::spacing,::spacing], v10_h[::spacing,::spacing],
                length=6)
    ax[1].barbs(lon[::spacing,::spacing], lat[::spacing,::spacing],
                u10_r[::spacing,::spacing], v10_r[::spacing,::spacing],
                length=6)

    # save and close figure
    figdir_full = f"{figdir}td2m_sidebyside_f{hr}.png"
    print(f"Saving figure to {figdir_full}")
    plt.savefig(figdir_full)
    plt.close()
    print("Finished plotting 1 x 2 Td2m!")
    # --------------------------------
    # Plot dew point comparison
    print("Creating Td2m Difference Plot!")

    # Define your custom colorbar bounds
    cbar_min = -30
    cbar_max = 30.1
    # contour levels
    clevs = np.linspace(cbar_min, cbar_max, 50)
    # color palette
    colors = seaborn.color_palette("seismic", as_cmap=True)

    # create plot
    fig, ax = plt.subplots(subplot_kw={'projection': ccrs.PlateCarree()},
                        figsize=(10,6.5), constrained_layout=True)

    # plot HRRR - RAP
    c0 = ax.contourf(lon, lat, td2m_h - td2m_r,
                    clevs, transform=ccrs.PlateCarree(),
                    cmap=colors, extend="both")

    # mapping
    plt_area = [-101, -94, 33.5, 37.5] # W, E, S, N
    ax.coastlines()
    ax.add_feature(cpf.BORDERS)
    ax.add_feature(cpf.STATES)
    ax.set_extent(plt_area)
    # Load the json file with county coordinates
    geoData = gpd.read_file('https://raw.githubusercontent.com/holtzy/The-Python-Graph-Gallery/master/static/data/US-counties.geojson')
    geoData.plot(ax=ax, color="none", lw=0.3, aspect=1)

    # set title
    ax.set_title(f"HRRR - RAP F0{hr},  Valid {valid_date} UTC")

    # Add colorbar
    cbar = fig.colorbar(c0, ax=ax, orientation='horizontal', extend=True, pad=0.03, aspect=50)
    cbar.set_label('HRRR - RAP 2m Dew Point Temperature [$^{\circ}$F]')
    cbar.set_ticks(np.arange(cbar_min, cbar_max, 5))

    # save and close figure
    figdir_full = f"{figdir}td2m_diff_f{hr}.png"
    print(f"Saving figure to {figdir_full}")
    plt.savefig(figdir_full)
    plt.close()
    print(f"Finished with hour {hr}! \n")


a = []
for i in range(1, 1000):
    if i % 7 == 0 and i % 4 != 0:
        a.append(i)
t = int(input())
for _ in range(t):
    n = int(input())
    for i in a:
        if i >= n:
            print(i)
            break


import os
import shutil
import subprocess
import tempfile
from typing import List

from phrasetree.tree import Tree

from elit.metrics.f1 import F1
from elit.metrics.metric import Metric
from elit.utils.io_util import get_resource, run_cmd
from elit.utils.log_util import cprint

class EvalbBracketingScorer(Metric):
    """
    This class uses the external EVALB software for computing a broad range of metrics
    on parse trees. Here, we use it to compute the Precision, Recall and F1 metrics.
    You can download the source for EVALB from here: <https://nlp.cs.nyu.edu/evalb/>.

    Note that this software is 20 years old. In order to compile it on modern hardware,
    you may need to remove an `include <malloc.h>` statement in `evalb.c` before it
    will compile.

    AllenNLP contains the EVALB software, but you will need to compile it yourself
    before using it because the binary it generates is system dependent. To build it,
    run `make` inside the `allennlp/tools/EVALB` directory.

    Note that this metric reads and writes from disk quite a bit. You probably don't
    want to include it in your training loop; instead, you should calculate this on
    a validation set only.

    # Parameters

    evalb_directory_path : `str`, required.
        The directory containing the EVALB executable.
    evalb_param_filename : `str`, optional (default = `"COLLINS.prm"`)
        The relative name of the EVALB configuration file used when scoring the trees.
        By default, this uses the nk.prm configuration file which comes with LAL-Parser.
        This configuration ignores POS tags, S1 labels and some punctuation labels.
    evalb_num_errors_to_kill : `int`, optional (default = `"10"`)
        The number of errors to tolerate from EVALB before terminating evaluation.
    """

    def __init__(
            self,
            evalb_directory_path: str = None,
            evalb_param_filename: str = "nk.prm",
            evalb_num_errors_to_kill: int = 10,
    ) -> None:
        if not evalb_directory_path:
            evalb_directory_path = get_resource('https://github.com/KhalilMrini/LAL-Parser/archive/master.zip#EVALB/')
        self._evalb_directory_path = evalb_directory_path
        self._evalb_program_path = os.path.join(evalb_directory_path, "evalb")
        self._evalb_param_path = os.path.join(evalb_directory_path, evalb_param_filename)
        self._evalb_num_errors_to_kill = evalb_num_errors_to_kill

        self._header_line = [
            "ID",
            "Len.",
            "Stat.",
            "Recal",
            "Prec.",
            "Bracket",
            "gold",
            "test",
            "Bracket",
            "Words",
            "Tags",
            "Accracy",
        ]

        self._correct_predicted_brackets = 0.0
        self._gold_brackets = 0.0
        self._predicted_brackets = 0.0

    def __call__(self, predicted_trees: List[Tree], gold_trees: List[Tree]) -> None:  # type: ignore
        """
        # Parameters

        predicted_trees : `List[Tree]`
            A list of predicted NLTK Trees to compute score for.
        gold_trees : `List[Tree]`
            A list of gold NLTK Trees to use as a reference.
        """
        if not os.path.exists(self._evalb_program_path):
            cprint(f"EVALB not found at {self._evalb_program_path}.  Attempting to compile it.")
            EvalbBracketingScorer.compile_evalb(self._evalb_directory_path)

            # If EVALB executable still doesn't exist, raise an error.
            if not os.path.exists(self._evalb_program_path):
                compile_command = (
                    f"python -c 'from allennlp.training.metrics import EvalbBracketingScorer; "
                    f'EvalbBracketingScorer.compile_evalb("{self._evalb_directory_path}")\''
                )
                raise RuntimeError(
                    f"EVALB still not found at {self._evalb_program_path}. "
                    "You must compile the EVALB scorer before using it."
                    " Run 'make' in the '{}' directory or run: {}".format(
                        self._evalb_program_path, compile_command
                    )
                )
        tempdir = tempfile.mkdtemp()
        gold_path = os.path.join(tempdir, "gold.txt")
        predicted_path = os.path.join(tempdir, "predicted.txt")
        with open(gold_path, "w") as gold_file:
            for tree in gold_trees:
                gold_file.write(f"{tree.pformat(margin=1000000)}\n")

        with open(predicted_path, "w") as predicted_file:
            for tree in predicted_trees:
                predicted_file.write(f"{tree.pformat(margin=1000000)}\n")

        command = [
            self._evalb_program_path,
            "-p",
            self._evalb_param_path,
            "-e",
            str(self._evalb_num_errors_to_kill),
            gold_path,
            predicted_path,
        ]
        completed_process = run_cmd(' '.join(command))

        _correct_predicted_brackets = 0.0
        _gold_brackets = 0.0
        _predicted_brackets = 0.0

        for line in completed_process.split("\n"):
            stripped = line.strip().split()
            if len(stripped) == 12 and stripped != self._header_line:
                # This line contains results for a single tree.
                numeric_line = [float(x) for x in stripped]
                _correct_predicted_brackets += numeric_line[5]
                _gold_brackets += numeric_line[6]
                _predicted_brackets += numeric_line[7]

        shutil.rmtree(tempdir)

        self._correct_predicted_brackets += _correct_predicted_brackets
        self._gold_brackets += _gold_brackets
        self._predicted_brackets += _predicted_brackets

    def get_metric(self):
        """
        # Returns

        The average precision, recall and f1.
        """

        return F1(self._predicted_brackets, self._gold_brackets, self._correct_predicted_brackets)

    def reset(self):
        self._correct_predicted_brackets = 0.0
        self._gold_brackets = 0.0
        self._predicted_brackets = 0.0

    @staticmethod
    def compile_evalb(evalb_directory_path: str = None):
        os.system("cd {} && make && cd ../../../".format(evalb_directory_path))
        run_cmd('chmod +x ' + os.path.join(evalb_directory_path, "evalb"))

    @staticmethod
    def clean_evalb(evalb_directory_path: str = None):
        return run_cmd("rm {}".format(os.path.join(evalb_directory_path, "evalb")))

    @property
    def score(self):
        return self.get_metric().prf[-1]

    def __repr__(self) -> str:
        return str(self.get_metric())

def main():
    tree1 = Tree.fromstring("(S (NP (D the) (N dog)) (VP (V chased) (NP (D the) (N cat))))")
    tree2 = Tree.fromstring("(S (NP (D the) (N dog)) (VP (V chased) (NP (D the) (N cat))))")
    evalb_scorer = EvalbBracketingScorer()
    evalb_scorer([tree1], [tree2])
    metrics = evalb_scorer.get_metric()
    assert metrics.prf == (1.0, 1.0, 1.0)

    tree1 = Tree.fromstring("(S (VP (D the) (NP dog)) (VP (V chased) (NP (D the) (N cat))))")
    tree2 = Tree.fromstring("(S (NP (D the) (N dog)) (VP (V chased) (NP (D the) (N cat))))")
    evalb_scorer = EvalbBracketingScorer()
    evalb_scorer([tree1], [tree2])
    metrics = evalb_scorer.get_metric()
    assert metrics.prf == (0.75, 0.75, 0.75)

    tree1 = Tree.fromstring("(S (VP (D the) (NP dog)) (VP (V chased) (NP (D the) (N cat))))")
    tree2 = Tree.fromstring("(S (NP (D the) (N dog)) (VP (V chased) (NP (D the) (N cat))))")
    evalb_scorer = EvalbBracketingScorer()
    evalb_scorer([tree1, tree2], [tree2, tree2])
    metrics = evalb_scorer.get_metric()
    assert metrics.prf == (0.875, 0.875, 0.875)

if __name__ == '__main__':
    main()


import os

app_name = "spec_with_splash"

# If set and 'onefile', a onefile project is built instead of onedir one.
build_mode = os.environ.get('_TEST_SPLASH_BUILD_MODE', 'onedir')

# If set and different from '0', collect tkinter via hidden import.
with_tkinter = os.environ.get('_TEST_SPLASH_WITH_TKINTER', '0')

if with_tkinter != '0':
    # Force tkinter collection via hiddenimports; this simulates a program importing tkinter.
    a = Analysis(
        ['../scripts/pyi_interact_pyi_splash.py'],
        hiddenimports=['tkinter'],
    )
else:
    # On some systems tkinter is included automatically; explicitly exclude it.
    a = Analysis(
        ['../scripts/pyi_interact_pyi_splash.py'],
        excludes=['tkinter', '_tkinter'],
    )

splash = Splash(
    '../data/splash/image.png',
    binaries=a.binaries,
    datas=a.datas,
    text_pos=(10, 50),
    text_color='red',
)

pyz = PYZ(a.pure, a.zipped_data)

if build_mode == 'onefile':
    exe = EXE(
        pyz,
        a.scripts,
        a.binaries,
        a.zipfiles,
        a.datas,
        splash,
        splash.binaries,
        exclude_binaries=False,
        name=app_name,
        debug=True,
        console=True,
    )
else:
    exe = EXE(
        pyz,
        a.scripts,
        splash,
        exclude_binaries=True,
        name=app_name,
        debug=True,
        console=True,
    )
    coll = COLLECT(
        exe,
        a.binaries,
        a.zipfiles,
        a.datas,
        splash.binaries,
        name=app_name,
    )