"""
grape1.py

Code for analyzing and visualizing HamSCI Grape1 Personal Space Weather Station
Observations.

Contributions by:
    Nathaniel Frissell W2NAF
    Kristina Collins KD8OXT
    Aidan Montare KB3UMD
    David Kazdan AD8Y
    John Gibbons N8OBJ
    Bob Benedict KD8CGH

    July 2022
"""

import os
import sys
import glob
import string
letters = string.ascii_lowercase

import datetime
import pytz

import numpy as np
import pandas as pd
import xarray as xr

import plotly.express as px
import plotly.graph_objects as go

import matplotlib as mpl
import matplotlib.pyplot as plt

from tqdm.auto import tqdm
tqdm.pandas(dynamic_ncols=True)

from scipy.interpolate import interp1d
from scipy import signal

from . import locator
from . import solar
from . import goes
from . import gen_lib as gl

prm_dict = {}

pkey = 'Freq'
prm_dict[pkey] = {}
prm_dict[pkey]['label'] = 'Doppler Shift [Hz]'
prm_dict[pkey]['vmin']  = -0.5
prm_dict[pkey]['vmax']  =  0.5
prm_dict[pkey]['cmap']  = 'bwr_r'
prm_dict[pkey]['title'] = 'Grape Doppler Shift'

pkey = 'Vpk'
prm_dict[pkey] = {}
prm_dict[pkey]['label'] = 'Peak Voltage [V]'

pkey = 'Power_dB'
prm_dict[pkey] = {}
prm_dict[pkey]['label'] = 'Received Power [dB]'
prm_dict[pkey]['title'] = 'Grape Received Power'

pkey = 'LMT'
prm_dict[pkey] = {}
prm_dict[pkey]['label'] = 'Mean Solar Time'

pkey = 'LMT_Hour'
prm_dict[pkey] = {}
prm_dict[pkey]['label'] = 'Solar Mean Time [Hours]'

pkey = 'LMT_Date'
prm_dict[pkey] = {}
prm_dict[pkey]['label'] = 'Date'

pkey = 'UTC_Hour'
prm_dict[pkey] = {}
prm_dict[pkey]['label'] = 'UTC [Hours]'

pkey = 'UTC_Date'
prm_dict[pkey] = {}
prm_dict[pkey]['label'] = 'Date'

pkey = 'lon'
prm_dict[pkey] = {}
prm_dict[pkey]['label'] = 'Longitude'

pkey = 'lat'
prm_dict[pkey] = {}
prm_dict[pkey]['label'] = 'Latitude'

class DataInventory(object):
    def __init__(self,nodes=None,G=None,freq=None,sTime=None,eTime=None,
                    data_path='data',suffix='.csv'):
        """
        Create an inventory of availble grape1 data in the data_path.
        Inventory will be dataframe df attached to DataInventory object.

        data_path: location of grape1 data
        suffix:    suffix of data files. Defaults to '.csv'
	"""
        # Load filenames and create a dataframe.
        fpaths = glob.glob(os.path.join(data_path,'*'+suffix))
        bnames = [os.path.basename(fpath) for fpath in fpaths]
        df = pd.DataFrame({'Filename':bnames})

        # Parse filenames into useful information and place into dataframe.
        df2              = df['Filename'].str.split('_', expand=True)
        df2.columns      =['Datetime', 'Node', 'G', 'Grid Square', 'FRQ', 'Frequency']
        df2              = df2.drop(columns=['FRQ']) # no information in this columnn
        df2["Frequency"] = df2["Frequency"].str.strip(suffix)

        # # Parse Grid Squares
        # lat,lon = locator.gridsquare2latlon(df2['Grid Square'])
        # df2['Latitude']  = lat
        # df2['Longitude'] = lon

        df               = pd.concat([df2, df], axis = 1) # concatenate dataframes horizontally
        df['Datetime']   = pd.to_datetime(df['Datetime']) # cast to datetime
        df['Node']       = df['Node'].str.strip("N")                   # Ditch the leading N on the node numbers
        df['Node']       = df['Node'].astype(str).astype(int)          # Cast node number to int
        df               = df[~df['Frequency'].str.contains('G1')]     # discarding files with naming errors

        # Convert frequency abbreviations to numbers:
        df.loc[df['Frequency'] == 'WWV5',    'Frequency'] = 5e6
        df.loc[df['Frequency'] == 'WWV10',   'Frequency'] = 10e6
        df.loc[df['Frequency'] == 'WWV2p5',  'Frequency'] = 2.5e6
        df.loc[df['Frequency'] == 'WWV15',   'Frequency'] = 15e6
        df.loc[df['Frequency'] == 'WWV20',   'Frequency'] = 20e6
        df.loc[df['Frequency'] == 'WWV25',   'Frequency'] = 25e6
        df.loc[df['Frequency'] == 'CHU3',    'Frequency'] = 3330e3
        df.loc[df['Frequency'] == 'CHU7',    'Frequency'] = 7850e3
        df.loc[df['Frequency'] == 'CHU14',   'Frequency'] = 14.67e6
        df.loc[df['Frequency'] == 'Unknown', 'Frequency'] = 0

        # Sort by Datetime
        df = df.sort_values(['Datetime','Frequency','Node']).copy()

        # Save dataframe to object
        self.df_unfiltered = df

        df = self.filter(nodes=nodes,G=G,freq=freq,sTime=sTime,eTime=eTime)

    def filter(self,nodes=None,G=None,freq=None,sTime=None,eTime=None):
        """
        Filter data by parameters. The filtered results will be returned
        and stored in self.df. Unfiltered results will always be available
        in self.df_unfiltered.

        Parameters
        ----------
        nodes:  node numbers (int or list of ints)
        G:      type of receiving station (string or list of strings)
		        G – Grape (Low Cost Platform)
		    	1 – Generation 1 (has 1 receiver)
		    	2 – Generation 2 (has 4 receivers)
		        T – Tangerine (High Performance Platform)
		    	1 – Generation 1
		     	2 – Generation 2
		        S – Standard / Commercial (Amateur) Hardware (details in Metadata)
		        U – User Custom Hardware (details in Metadata)
		        N – No radio collectiopn – instrument(s) only – defaults to N0
        freq:   frequency in Hz (float or list of floats)
        sTime:  UTC starting time of observations (datetime object)
        eTime:  UTC ending time of observations (datetime object)

        """

        df = self.df_unfiltered.copy()

        if nodes is not None:
            nodes   = gl.get_iterable(nodes)
            tf      = df['Node'].isin(nodes)
            df      = df[tf].copy()

        if G is not None:
            G       = gl.get_iterable(G)
            tf      = df['G'].isin(G)
            df      = df[tf].copy()

        if freq is not None:
            freq    = gl.get_iterable(freq)
            tf      = df['Frequency'].isin(freq)
            df      = df[tf].copy()

        if sTime is not None:
            # Return files that start a day early,
            # otherwise you may not get all of the files you need.
            sTime  -= datetime.timedelta(days=1)

            tf      = df['Datetime'] >= sTime
            df      = df[tf].copy()

        if eTime is not None:
            # Return files that end a day after eTime,
            # otherwise you may not get all of the files you need.
            eTime  += datetime.timedelta(days=1)

            tf      = df['Datetime'] < eTime
            df      = df[tf].copy()

        self.df = df

        # List of logged nodes during the period of interest, for sorting:
        logged_nodes = df["Node"].unique().tolist()
        logged_nodes.sort()
        self.logged_nodes = logged_nodes

        return df

    def get_nodes(self):
        """
        Return a list of all of the unique node numbers in the inventory.
        """
        nodes = self.df['Node'].unique().tolist()
        return nodes

    def plot_inventory(self,html_out='inventory.html'):
        """
        Generate a plot of the data inventory using plotly.

        html_out: Filename of html-version of data inventory plot.
        """
        # We can create a Data Inventory (Gantt chart) showing when different stations were active.
        invt         = self.df.copy()
        logged_nodes = self.logged_nodes

        invt.set_index('Node')
        invt = invt.drop(columns=['G', 'Grid Square'])
        invt['EndTime'] = invt['Datetime']+ datetime.timedelta(days=1) # create an end
        invt['Filename'].str.strip('FRQ_')

        fig = px.timeline(invt, x_start="Datetime", x_end="EndTime", y="Node", color="Frequency", category_orders={"Node": logged_nodes})
        fig.update_yaxes(type='category')
        fig.update_annotations(text = "Filename", clicktoshow='on')

        fig.update_layout({
            'plot_bgcolor': 'rgba(0, 0, 0, 0)',
            'paper_bgcolor': 'rgba(0, 0, 0, 0)',
            'font_size': mpl.rcParams['font.size']
        })

        fig.show()

        if html_out:
            fig.write_html(html_out, include_plotlyjs="cdn")

class GrapeNodes(object):
    def __init__(self,fpath='nodelist.csv',logged_nodes=None):
        """
        Create an object with information about known Grape1 nodes from a nodelist.csv.

        fpath:        path to nodelist.csv
        logged_nodes: list of nodes we have data for. This can be obtained from DataInventory.logged_nodes.
        """
        # Read in node list:
        nodes_df = pd.read_csv(fpath)
        nodes_df = nodes_df.rename(columns={'Node #': 'Node_Number'})
        nodes_df = nodes_df.set_index('Node_Number')

        self.nodes_df = nodes_df

        if logged_nodes:
            nodes_df = self.update_status(logged_nodes)

    def update_status(self,logged_nodes):
        """
        Updates the nodes_df dataframe with with information regarding which nodes we have data from.

        logged_nodes: list of nodes we have data for. This can be obtained from DataInventory.logged_nodes.
        """
        nodes_df = self.nodes_df

        nodes_df['Status'] = np.where((nodes_df.index.isin(logged_nodes)), "Data logged", "No data logged")
        self.nodes_df = nodes_df
        return nodes_df

    def status_table(self):
        """
        Returns the nodes_df data frame with nodes that we have data for highlighted in green.
        """
        # Highlight rows in green for stations that we have data from.
        nodes_df = self.nodes_df.copy()
        color    = (nodes_df['Status'] == 'Data logged').map({True: 'background-color: palegreen', False: ''})
        nodes_df = nodes_df.style.apply(lambda s: color)
        return nodes_df

    def plot_map(self,color="Status",projection='albers usa',
            width=1200,height=900):
        """
        Use plotly to generate a map of Grape1 nodes.
        """
        nodes_df = self.nodes_df
        # Map nodes:
        fig = px.scatter_geo(nodes_df, "Latitude", "Longitude",
                             color=color, # which column to use to set the color of markers
                             hover_name=nodes_df["Callsign"], # column added to hover information
                             projection = projection,
                             width = width, height = height,
                             )
        fig.show()

class Filter(object):
    def __init__(self,N=6,Tc_min = 3.3333,btype='low',fs=1.):
        """
        Generate a digital filter that can be applied to the data.

        This routine uses the scipy.signal.butter Butterworth filter.

        Parameters
        ----------
        N:      Filter order.
        Tc_min: Cuttoff in minutes. Scalar value for 'low' and 'high'
                filter btypes; 2-element iterable for 'bandpass' and
                'bandstop' filter btypes.
        fs:     Sampling frequency of the digital system in samples/sec.
        """
        Wn     = (1./(np.array(Tc_min)*60.))

        if np.size(Wn) == 2:
            Wn = Wn[::-1]

        #Wn    = (1000, 5000)    # 3 dB Cutoff Frequency in Hz
               # Choose 'bandpass' or 'bandstop'

        b, a = signal.butter(N, Wn, btype, fs=fs)

        self.fs = fs
        self.Wn = Wn
        self.a  = a
        self.b  = b

    def filter_data(self,data):
        """
        Apply the filter using scipy.signal.filtfilt to data.

        data: Vector of data to be filtered.
        """
        try:
            filtered = signal.filtfilt(self.b,self.a,data)
        except Exception as err:
            print('Filter error... returning NaNs.')
            print('   Error: {!s}'.format(err))
            filtered = data*np.nan

        return filtered

    def plotResponse(self):
        """
        Plot the magnitude and phase response of the filter.
        """
        fs = self.fs
        Wn = self.Wn
        a  = self.a
        b  = self.b

        w, h = signal.freqz(b, a,worN=2**16)
        f = (fs/2)*(w/(np.pi))

        plt.figure(figsize=(12,8))
        plt.subplot(211)
        plt.plot(f, 20 * np.log10(abs(h)))
        plt.xscale('log')
        plt.title('Butterworth Filter Frequency Response')
        plt.xlabel('Frequency [Hz]')
        plt.ylabel('Amplitude [dB]')
        plt.grid(which='both', axis='both')
        if np.size(Wn) == 1:
            plt.axvline(Wn, color='green')    # cutoff frequency
        else:
            plt.axvline(Wn[0], color='green') # cutoff frequency
            plt.axvline(Wn[1], color='green') # cutoff frequency

        plt.ylim(-30,0)

        plt.subplot(212)
        plt.plot(f, np.unwrap(np.angle(h)))
        plt.xscale('log')
        plt.title('Butterworth Filter Phase Response')
        plt.xlabel('Frequency [Hz]')
        plt.ylabel('Phase [rad]')
        plt.grid(which='both', axis='both')
        if np.size(Wn) == 1:
            plt.axvline(Wn, color='green')    # cutoff frequency
        else:
            plt.axvline(Wn[0], color='green') # cutoff frequency
            plt.axvline(Wn[1], color='green') # cutoff frequency

        plt.tight_layout()

class Grape1Data(object):
    def __init__(self,node=None,freq=None,sTime=None,eTime=None,data_path='data',
                 lat=None,lon=None,call_sign=None,
                 solar_lat=None,solar_lon=None,
                 inventory=None,grape_nodes=None,
                 data=None,meta=None):

        if data is None and meta is None:
            self.__load_raw(node,freq,sTime,eTime=eTime,data_path=data_path,
                     lat=lat,lon=lon,call_sign=call_sign,solar_lat=solar_lat,solar_lon=solar_lon,
                     inventory=inventory,grape_nodes=grape_nodes)
        else:
            self.data = data
            self.meta = meta

    def __load_raw(self,node,freq,sTime,eTime,data_path,
                 lat,lon,call_sign,solar_lat,solar_lon,inventory,grape_nodes):

        if inventory is None:
            inventory = DataInventory(data_path=data_path)

        # Make temporary copy of dataframe.
        dft   = inventory.df.copy()

        # Select rows with selected frequency
        tf    = dft['Frequency'] == freq
        dft   = dft[tf].copy()

        # Select rows matching node number.
        tf    = dft['Node'] == node
        dft   = dft[tf].copy()

        if sTime is None:
            sTime = min(dft['Datetime'])

        if eTime is None:
            eTime = max(dft['Datetime'])

        # Expand time range of loading files, or you might miss
        # files you need. Strict time limits are applied later in this
        # routine after the data has been loaded.
        sTime_load  = sTime - datetime.timedelta(days=1)
        eTime_load  = eTime + datetime.timedelta(days=1)

        # Select rows matching time range.
        tf    = np.logical_and(dft['Datetime'] >= sTime_load, dft['Datetime'] < eTime_load)
        dft   = dft[tf].copy()
        dft   = dft.sort_values('Datetime')

        # Load data from every data file available for node/frequency/date range.
        df_raw = []
        for rinx,row in tqdm(dft.iterrows(),total=len(dft),dynamic_ncols=True,desc='Loading Raw Data'):
            fname = row['Filename']
            fpath = os.path.join(data_path,fname)

            df_load = pd.read_csv(fpath, comment = '#', parse_dates=[0])
            if len(df_load) == 0: continue

            # Remove the center frequency offset from the frequency column.
            df_load['Freq'] = df_load['Freq']-freq
            df_raw.append(df_load)

        df_raw  = pd.concat(df_raw,ignore_index=True)
        df_raw  = df_raw.sort_values('UTC')

        # Enforce sTime/eTime
        tf      = np.logical_and(df_raw['UTC'] >= sTime, df_raw['UTC'] < eTime)
        df_raw  = df_raw[tf].copy()

        # Generate a label for each Node
        if (call_sign is None) and (grape_nodes is not None):
            call_sign = grape_nodes.nodes_df.loc[node,'Callsign']
        else:
            call_sign = ''

        # Get latitude and longitude
        if (lat is None) and (grape_nodes is not None):
            lat = grape_nodes.nodes_df.loc[node,'Latitude']

        if (lon is None) and (grape_nodes is not None):
            lon = grape_nodes.nodes_df.loc[node,'Longitude']

        if solar_lat is None:
            solar_lat = lat

        if solar_lon is None:
            solar_lon = lon

        # Store data into dictionaries.
        data = {}
        data['raw']          = {}
        data['raw']['df']    = df_raw
        data['raw']['label'] = 'Raw Data'
        self.data = data

        label     = '{:0.1f} MHz N{:07d} {!s}'.format(freq/1e6,node,call_sign)

        meta = {}
        meta['label']       = label
        meta['freq']        = freq
        meta['node']        = node
        meta['call_sign']   = call_sign
        meta['lat']         = lat
        meta['lon']         = lon
        meta['solar_lat']   = solar_lat
        meta['solar_lon']   = solar_lon
        self.meta      = meta

    def process_data(self,profile='standard'):
        tic_0 = datetime.datetime.now()
        print('Processing data using "{!s}" profile...'.format(profile))
        print('')
        if profile == 'standard':
            print('Computing Solar Local Time on raw data...')
            tic = datetime.datetime.now()
            self.calculate_solar_time('raw')
            toc = datetime.datetime.now()
            print('  Solar Time Computation Time: {!s}'.format(toc-tic))

            resample_rate = datetime.timedelta(seconds=1)
            filter_order  = 6
            Tc_min        = 3.3333
            btype         = 'low'

            print('Resampling data with {!s} second cadence...'.format(resample_rate.total_seconds()))
            tic = datetime.datetime.now()
            self.resample_data(resample_rate=resample_rate,
                              data_set_in='raw',data_set_out='resampled')
            toc = datetime.datetime.now()
            print('  Resampling Time: {!s}'.format(toc-tic))

            print('Computing Solar Local Time on resampled...')
            tic = datetime.datetime.now()
            self.calculate_solar_time('resampled')
            toc = datetime.datetime.now()
            print('  Solar Time Computation Time: {!s}'.format(toc-tic))

            # Convert Vpk to Power_dB
            print('dB Conversion')
            tic = datetime.datetime.now()
            self.data['resampled']['df']['Power_dB'] = 20*np.log10( self.data['resampled']['df']['Vpk'])
            toc = datetime.datetime.now()
            print('  dB Conversion Time: {!s}'.format(toc-tic))

            print('Filtering data with {!s} minute low-pass Butterworth filter...'.format(Tc_min))
            tic = datetime.datetime.now()
            self.filter_data(N=filter_order,Tc_min=Tc_min,btype=btype)
            toc = datetime.datetime.now()
            print('  Filtering Time: {!s}'.format(toc-tic))

        elif profile == '5min_mean':
            data_set_in = 'raw'
            data_set    = 'resampled'
            xkeys       = ['LMT','UTC']
            params      = ['Freq','Power_dB']

            resample_rate = datetime.timedelta(minutes=5)
            print('Resampling data with {!s} minute cadence...'.format(resample_rate.total_seconds()/60.))
            tic = datetime.datetime.now()
            self.resample_data(resample_rate=resample_rate,method='mean',
                              data_set_in=data_set_in,data_set_out=data_set)
            toc = datetime.datetime.now()
            print('  Resampling Time: {!s}'.format(toc-tic))
            print()

            print('Computing Solar Local Time on resampled...')
            tic = datetime.datetime.now()
            self.calculate_solar_time(data_set)
            toc = datetime.datetime.now()
            print('  Solar Time Computation Time: {!s}'.format(toc-tic))
            print()

            # Convert Vpk to Power_dB
            print('dB Conversion')
            tic = datetime.datetime.now()
            self.data[data_set]['df']['Power_dB'] = 20*np.log10( self.data[data_set]['df']['Vpk'])
            toc = datetime.datetime.now()
            print('  dB Conversion Time: {!s}'.format(toc-tic))
            print()

            for xkey in xkeys:
                for param in params:
                    print('Processing: {!s} - {!s}'.format(xkey,param))
                    print('Compute Time-Date-Parameter (TDP) Array')
                    tic = datetime.datetime.now()
                    tdp = self.calculate_timeDateParameter_array(data_set,param,xkey=xkey)
                    toc = datetime.datetime.now()
                    print('  Time-Date-Parameter Time: {!s}'.format(toc-tic))
                    print()

                    tdp_key = self.get_tdp_key(param,xkey)
                    self.data[data_set][tdp_key] = tdp

        toc_0 = datetime.datetime.now()
        print('')
        print('Total Processing Time: {!s}'.format(toc_0-tic_0))

    def resample_data(self,resample_rate,on='UTC',method='linear_interp',
                          data_set_in='raw',data_set_out='resampled'):

        df   = self.data[data_set_in]['df'].copy()

        if len(df) == 0:
            rs_df = df.copy()
        else:
            # Create the list of datetimes that we want to resample to.
            # Find the start and end times of the array.
            sTime = df['UTC'].min()
            eTime = df['UTC'].max()

            tzinfo= sTime.tzinfo

            # Break
            sYr  = sTime.year
            sMon = sTime.month
            sDy  = sTime.day
            sHr  = sTime.hour
            sMin = sTime.minute
            sSec = sTime.second
            resample_sTime = datetime.datetime(sYr,sMon,sDy,sHr,sMin,sSec,tzinfo=tzinfo)

            eYr  = eTime.year
            eMon = eTime.month
            eDy  = eTime.day
            eHr  = eTime.hour
            eMin = eTime.minute
            eSec = eTime.second
            resample_eTime = datetime.datetime(eYr,eMon,eDy,eHr,eMin,eSec,tzinfo=tzinfo)

            # Remove LMT column if it exists because it cannot be resampled.
            if 'LMT' in df.keys():
                df = df.drop('LMT',axis=1)

            cols        = df.keys()
            df          = df.set_index(on) # Need to make UTC column index for interpolation to work.
            df          = df.drop_duplicates()

            df          = df[~df.index.duplicated(keep='first')] # Make sure there are no duplicated indices.

            rs_df       = df.resample(resample_rate,origin=resample_sTime)
            if method == 'mean':
                rs_df = rs_df.mean()
            else:
                rs_df = rs_df.interpolate(method='linear')

            rs_df       = rs_df.copy()
            rs_df[on]   = rs_df.index
            rs_df.index = np.arange(len(rs_df))

            # Put Columns back in original order.
            rs_df       = rs_df[cols].copy()

        tmp          = {}
        tmp['df']    = rs_df
        tmp['label'] = 'Resampled Data (dt = {!s} s)'.format(resample_rate.total_seconds())
        tmp['Ts']    = resample_rate.total_seconds()
        self.data[data_set_out] = tmp

    def filter_data(self,N,Tc_min,btype,
                    data_set_in='resampled',data_set_out='filtered',
                    params=['Freq','Vpk','Power_dB']):

        df     = self.data[data_set_in]['df'].copy()

        # Get sample rate of data set.
        Ts     = self.data[data_set_in].get('Ts')
        if Ts is None:
            Ts_arr = np.unique(np.diff(df['UTC']))
            if len(Ts_arr) != 1:
                raise Exception("{!s} is not evenly sampled. Cannot apply filter.".format(data_set_in))
            Ts   = (Ts_arr[0]).total_seconds()

        # Convert sample rate to sampling frequency.
        fs   = 1./Ts

        filt = Filter(N=N,Tc_min=Tc_min,btype=btype,fs=fs)

        for param in params:
            if param not in df.keys():
                continue
            df[param] = filt.filter_data(df[param])

        tmp          = {}
        tmp['df']    = df
        tmp['label'] = 'Butterworth Filtered Data\n(N={!s}, Tc={!s} min, Type: {!s})'.format(N, Tc_min, btype)
        self.data[data_set_out] = tmp

    def calculate_solar_time(self,data_set,solar_lon=None):
        if solar_lon is None:
            solar_lon = self.meta.get('solar_lon')
        else:
            self.meta['solar_lon'] = solar_lon

        df = self.data.get(data_set)['df']
        df['LMT'] = df['UTC'].progress_apply(solar.solar_time,lon=solar_lon)

        # Set columns so UTC and LMT lead.
        keys = list(df.keys())
        keys.remove('UTC')
        keys.remove('LMT')
        keys = ['UTC','LMT'] + keys
        df   = df[keys]
        self.data[data_set]['df'] = df

    def calculate_timeDateParameter_array(self,data_set,param='Freq',xkey='LMT'):
        df          = self.data[data_set]['df']
        time_vec    = df[xkey]

        # Calculate bin size.
        dt_hr       = (time_vec.diff().mean().total_seconds())/3600.
        hour_bins   = np.arange(0,24+dt_hr,dt_hr)

        print('Splitting {!s} into dates...'.format(xkey))
        dates       = time_vec.progress_apply(gl.datetime2date)
        date_bins   = list(set(dates))
        date_bins.sort()
        date_bins   = np.array(date_bins)

        print('Splitting {!s} into decimal hours...'.format(xkey))
        hours       = time_vec.progress_apply(gl.decimal_hours)

        # Rearrange Data into a 2D Array
        print('Filling timeDateParameter Array...')
        tdp_arr = np.zeros( (len(date_bins), len(hour_bins)) )*np.nan
        for rinx, row in tqdm(df.iterrows(),total=len(df),dynamic_ncols=True):
            val         = row[param]
            date_inx    = np.where(dates.loc[rinx] == date_bins)[0][0]
            t_bin_inx   = np.digitize(hours.loc[rinx],hour_bins)

            tdp_arr[date_inx,t_bin_inx] = val

        # Put array into xarray.
        xr_xkey = '{!s}_Date'.format(xkey)
        xr_ykey = '{!s}_Hour'.format(xkey)

        crds    = {}
        crds[xr_xkey]	= date_bins
        crds[xr_ykey]   = hour_bins
        tdp_da          = xr.DataArray(tdp_arr,crds,dims=[xr_xkey,xr_ykey])

        tdp_key         = self.get_tdp_key(param,xkey)

        # Save back to object
        self.data[data_set][tdp_key] = tdp_da

        return tdp_da

    def get_tdp_key(self,param,xkey):
        tdp_key = 'tdp_{!s}_{!s}'.format(param,xkey)
        return tdp_key

    def show_datasets(self):
        keys        = []
        datasets    = []
        for key,dct in self.data.items():
            keys.append(key)

            tmp  = {}
            tmp['label'] = dct.get('label')

            datasets.append(tmp)

        datasets = pd.DataFrame(datasets,index=keys)
        return datasets

    def plot_timeSeries(self,data_sets=['raw'],
                        sTime=None,eTime=None,
                        xkey='UTC',params=['Freq','Vpk','Power_dB'],
                        ylims = {},
                        plot_kws = [{'ls':'','marker':'.'},{}],
                        overlayTerminator=True,
                        solar_lat=None,solar_lon=None,
                        fig_width=15,panel_height=6):

        data_sets = gl.get_iterable(data_sets)

        df      = self.data[data_sets[0]]['df']
        if sTime is None:
            sTime = min(df[xkey])

        if eTime is None:
            eTime = max(df[xkey])

        # Make sure we have the requested parameters.
        params = gl.get_iterable(params)
        params_good = []
        for param in params:
            if param in df.keys():
                params_good.append(param)
        params = params_good

        # Start plotting
        ncols   = 1
        nrows   = len(params)
        figsize = (fig_width, nrows*panel_height)

        fig = plt.figure(figsize=figsize)
        axs = []
        for plt_inx,param in enumerate(params):
            ax  = fig.add_subplot(nrows,ncols,plt_inx+1)
            axs.append(ax)

            if overlayTerminator:
                solar_lat = self.meta.get('solar_lat',solar_lat)
                solar_lon = self.meta.get('solar_lon',solar_lon)
                if (solar_lat is not None) and (solar_lon is not None):
                    solar.add_terminator(sTime,eTime,solar_lat,solar_lon,ax,xkey=xkey)
                else:
                    print('Error: Need to provide solar_lat and solar_lon to overlay solar terminator.')

            ax.set_xlim(sTime,eTime)
            if plt_inx != nrows-1:
                ax.set_xticklabels('')
            else:
                fig.autofmt_xdate()
                xprmd  = prm_dict.get(xkey,{})
                xlabel = xprmd.get('label',xkey)
                ax.set_xlabel(xlabel)

            yprmd  = prm_dict.get(param,{})
            ylabel = yprmd.get('label',yprmd)
            ax.set_ylabel(ylabel)
            ylim = ylims.get(param,None)
            if ylim is not None:
                ax.set_ylim(ylim)

            if plt_inx == 0:
                ax.set_title(self.meta.get('label',''))
            ax.set_title('({!s})'.format(letters[plt_inx]),loc='left')

        for ds_inx,data_set in enumerate(data_sets):
            for plt_inx,param in enumerate(params):
                ax = axs[plt_inx]

                df  = self.data[data_set]['df']
                xx  = df[xkey]
                yy  = df[param]

                label   = self.data[data_set].get('label','')
                plot_kw = plot_kws[ds_inx]
                ax.plot(xx,yy,label=label,**plot_kw)

        for plt_inx,param in enumerate(params):
            ax = axs[plt_inx]
            ax.legend(loc='upper right',fontsize='small')

        fig.tight_layout()

        return {'fig':fig}

    def plot_timeDateParameter(self,data_set,params=['Freq','Power_dB'],xkey='LMT',
            fig_width=15,panel_height=6):

        # Start plotting
        ncols   = 1
        nrows   = len(params)
        figsize = (fig_width, nrows*panel_height)

        fig = plt.figure(figsize=figsize)
        axs = []
        for plt_inx,param in enumerate(params):
            tdp_key = self.get_tdp_key(param,xkey)
            tdp = self.data[data_set][tdp_key]

            ax  = fig.add_subplot(nrows,ncols,plt_inx+1)
            axs.append(ax)

            xr_xkey = '{!s}_Date'.format(xkey)
            xprmd   = prm_dict.get(xr_xkey)
            xlabel  = xprmd.get('label')

            xr_ykey = '{!s}_Hour'.format(xkey)
            yprmd   = prm_dict.get(xr_ykey)
            ylabel  = yprmd.get('label')

            prmd = prm_dict.get(param)
            vmin = prmd.get('vmin')
            vmax = prmd.get('vmax')
            cmap = prmd.get('cmap')
            plbl = prmd.get('label')

            cbar_kwargs = {}
            cbar_kwargs['label'] = plbl

            ret  = tdp.plot.pcolormesh(xr_xkey,xr_ykey,vmin=vmin,vmax=vmax,cmap=cmap,
                    cbar_kwargs=cbar_kwargs,ax=ax)
            ax.set_ylabel(ylabel)

            if plt_inx != nrows-1:
#                ax.set_xticklabels('')
                ax.set_xlabel('')
            else:
                ax.set_xlabel(xlabel)

            if plt_inx == 0:
                ax.set_title(self.meta.get('label',''))
            ax.set_title('({!s})'.format(letters[plt_inx]),loc='left')

            fig.tight_layout()

        return {'fig':fig}

class GrapeMultiplot(object):
    def __init__(self,gds):
        """
        gds: List of Grape1Data objects.
        """

        self.gds = gds

    def multiplot(self,data_set,params=['Freq','Power_dB'],
                    xkey='UTC',
                    sTime=None,eTime=None,
                    ylims=None,
                    color_dct=None,legend=True,
                    solar_lat=None,solar_lon=None,
                    events=None, event_fontdict = {'size':20,'weight':'bold'},
                    plot_GOES=False,GOES_sat_nr=17,GOES_data_dir='data/goes',
                    fig_width=22,panel_height=8):
        """
        Plot a time series with traces from multiple instruments overlaid on the same plot.

        Parameters
        ----------
        data_set:   Name of data set to plot (string)
        params:     List of parameter keys to plot. A new subplot will be created for each param.
        xkey:       X-axis parameter.
                        'UTC' for Coordinated Universal Time
                        'LMT' for Local Mean Time (solar time)
        sTime:      Start time of plot (UTC or LMT datetime object)
        eTime:      End time of plot (UTC or LMT datetime object)
        ylims:      None or dictionary with ylim for each parameter.
                    Example:
                      {'Freq':(-5,5)}
        color_dct:  Dictionary indicating how to color the traces.
                    Example:
                      {'ckey':'lon'} will color the traces according to the lon parameter
                            using the default colormap (viridis) and vmax/min parameters.

                      {'ckey':'lat','cmap':'jet','vmin':-90,'vmax':90} will color the traces
                            according to the lat parameter using the jet colormap with
                            vmin=-90 and vmax=90.
        legend:     Plot legend with name of each receiving station.
        solar_lat:  Latitude for computing sunrise/sunset times.
        solar_lon:  Longitude for computing sunrise/sunset times.

        """

        data = self.gds

        # Sort data objects by specified parameter and set color.
        if color_dct is not None:
            ckey = color_dct.get('ckey')

            # Sort data by ckey.
            vals = [x.meta[ckey] for x in data]
            srt  = np.argsort(vals)[::-1]
            data = [data[x] for x in srt]

            cmap = color_dct.get('cmap','viridis')
            if len(vals) > 0:
                vmin = color_dct.get('vmin',np.min(vals))
                vmax = color_dct.get('vmax',np.max(vals))
            else:
                vmin = -10.
                vmax =  10.

            if vmin == vmax:
                vmin = vmin - 10.
                vmax = vmax + 10.

            norm = mpl.colors.Normalize(vmin=vmin, vmax=vmax)
            mpbl = mpl.cm.ScalarMappable(norm=norm, cmap=cmap)

        # Determine sTime and eTime if not specified.
        if sTime is None or eTime is None:
            xx_xtrm   = [] # Keep track of time extrema.
            for gd in data:
                df_data = gd.data[data_set]['df']
                if len(df_data) == 0:
                    continue

                xx      = df_data[xkey]
                xx_xtrm.append(np.min(xx))
                xx_xtrm.append(np.max(xx))

            if sTime is None:
                sTime = min(xx_xtrm)

            if eTime is None:
                eTime = max(xx_xtrm)

        ncols = 1
        nrows = len(params)
        if plot_GOES:
            nrows += 1

        figsize = (fig_width, nrows*panel_height)
        fig     = plt.figure(figsize=figsize)

        axs = []
        plt_inx = 0
        if plot_GOES:
            plt_inx += 1
            ax   = fig.add_subplot(nrows, ncols, plt_inx)
            ax.set_title('({!s})'.format(letters[plt_inx]),loc='left')
            axs.append(ax)

            goes_data = goes.read_goes(sTime,eTime,sat_nr=GOES_sat_nr,data_dir=GOES_data_dir)
            goes.goes_plot(goes_data,sTime,eTime,ax=ax)
            ax.set_title('NOAA GOES-{:02d}'.format(GOES_sat_nr),loc='right')

            if plt_inx != nrows-1:
                ax.set_xticklabels('')

            if events is not None:
                trans       = mpl.transforms.blended_transform_factory(ax.transData,ax.transAxes)
                for event in events:
                    evt_dtime   = event.get('datetime')
                    evt_label   = event.get('label')
                    evt_color   = event.get('color','0.4')

                    ax.axvline(evt_dtime,lw=1,ls='--',color=evt_color)
                    if evt_label is not None:
                        ax.text(evt_dtime,0.01,evt_label,transform=trans,
                                rotation=90,fontdict=event_fontdict,color=evt_color,
                                va='bottom',ha='right')

        params = gl.get_iterable(params)
        for prm_inx,(param) in enumerate(params):
            plt_inx += 1
            ax   = fig.add_subplot(nrows, ncols, plt_inx)
            axs.append(ax)

            abs_maxes = [] # Keep track of absolute value maxima
            for gd in data:
                df_data = gd.data[data_set]['df']
                if len(df_data) == 0:
                    continue

                label   = gd.meta['label']

                plt_kw  = {}

                xx      = df_data[xkey]
                yy      = df_data[param]

                if not np.all(np.isfinite(yy)):
                    continue

                abs_maxes.append(np.nanmax(np.abs(yy)))
                if color_dct is not None:
                    val     = gd.meta[ckey]
                    plt_kw['color'] = mpbl.cmap(mpbl.norm(val))

                ax.plot(xx,yy,label=label,**plt_kw)

            ax.set_title('({!s})'.format(letters[plt_inx]),loc='left')

            prmd    = prm_dict.get(param,{})
            rtitle  = prmd.get('title')
            if rtitle:
                ax.set_title(rtitle,loc='right')

            ylbl    = prmd.get('label',param)
            ax.set_ylabel(ylbl)

            legend_bbox_to_anchor = (1.04,1)
            if color_dct is not None:
                cb_prmd = prm_dict.get(ckey,{})
                clabel  = cb_prmd.get('label',ckey)
                fig.colorbar(mpbl,ax=ax,label=clabel)
                legend_bbox_to_anchor = (1.2,1)

            if legend:
                ax.legend(bbox_to_anchor=legend_bbox_to_anchor, loc="upper left", borderaxespad=0)

            if len(xx_xtrm) == 0:
                ax.text(0.5,0.5,'No Data',ha='center',transform=ax.transAxes,fontsize=36)
                continue

            ax.set_xlim(sTime,eTime)

            if param == 'Freq':
                abs_max = np.max(abs_maxes)
                ax.set_ylim(-abs_max*1.05,abs_max*1.05)

            # Set ylim from ylims keyword.
            if ylims is not None:
                ylim = ylims.get(param)
                if ylim is not None:
                    ax.set_ylim(ylim)

            if plt_inx != nrows:
                ax.set_xticklabels('')
            else:
                xprmd   = prm_dict.get(xkey,{})
                xlbl    = xprmd.get('label',xkey)
                ax.set_xlabel(xlbl)

            if events is not None:
                trans       = mpl.transforms.blended_transform_factory(ax.transData,ax.transAxes)
                for event in events:
                    evt_dtime   = event.get('datetime')
                    evt_label   = event.get('label')
                    evt_color   = event.get('color','0.4')

                    ax.axvline(evt_dtime,lw=1,ls='--',color=evt_color)
                    if evt_label is not None:
                        ax.text(evt_dtime,0.01,evt_label,transform=trans,
                                rotation=90,fontdict=event_fontdict,color=evt_color,
                                va='bottom',ha='right')

            # Add solar terminator
            if (solar_lat is not None) and (solar_lon is not None):
                solar.add_terminator(sTime,eTime,solar_lat,solar_lon,ax,xkey=xkey)

        date_str = sTime.strftime('%Y %b %d %H:%M UT') + ' - ' + eTime.strftime('%Y %b %d %H:%M UT')
        title = []
#        title.append('HamSCI Grape PSWS Observations')
        title.append(date_str)
        axs[0].text(0.5,1.1,'\n'.join(title),ha='center',fontsize=24,
                fontweight='bold',transform=axs[0].transAxes)

        fig.tight_layout()
        if plot_GOES:
            gl.adjust_axes(axs[0],axs[-1])

        return {'fig':fig,'axs':axs}


class DLLNode(object):

    def __init__(self, value, nxt, prev):
        self.value = value
        self.prev = prev
        self.nxt = nxt

    def __repr__(self):
        pval = self.prev and self.prev.value or None
        nval = self.nxt and self.nxt.value or None
        return f'[{repr(pval)}, {self.value}, {repr(nval)}]'

class DLList(object):

    def __init__(self):
        self.head = None
        self.end = None

    def push(self, obj):

        # Apends a new value to the end of the list.

        new_elem = DLLNode(obj, None, None)

        if self.head == None and self.end == None:
            self.head = new_elem
            self.end = self.head

        else:
            self.end.nxt = new_elem
            new_elem.prev = self.end
            self.end = new_elem

    def count(self):

        # Returns the number of items in the list.

        if not self.head:
            return 0

        else:
            x = 1
            n = self.head
            while n.nxt != None:
                x += 1
                n = n.nxt
            else:
                return x

    def traverse_list(self):

        # Return the contents of the list.

        if self.head == None:
            print('List is empty []')

        else:
            n = self.head
            while n != None:
                print(n)
                n = n.nxt

    def first(self):

        # Returns a reference for first item in list but does not remove it.

        if not self.head:
            return None

        else:
            return self.head.value

    def last(self):

        # Return last item in the list.

        if not self.head:
            return None

        else:
            return self.end.value

    def pop(self):

        # Removes the last item and returns it.

        if not self.head:
            return None

        elif self.head == self.end:
            n = self.end
            self.end = None
            self.head = None
            return n.value

        else:
            n = self.end
            new_end = self.end.prev
            new_end.nxt = None
            self.end = new_end
            return n.value

    def popleft(self):

        # Removes the first item and returns it.

        if not self.head:
            return None

        elif self.head == self.end:
            n = self.head
            self.end = None
            self.head = None
            return n.value

        else:
            n = self.head
            new_head = self.head.nxt
            new_head.prev = None
            self.head = new_head
            return n.value

    def remove(self, obj):

        # Finds a matching item and removes it from the list.

        if not self.head:
            print('List is empty[]')
            return False

        elif self.head.value == obj:
            self.head = self.head.nxt
            self.head.prev = None
            return True

        else:
            n = self.head
            while n.nxt != None:
                if n.nxt.value == obj:
                    n.nxt = n.nxt.nxt
                    if n.nxt == None:
                        pass
                    else:
                        n.nxt.prev = n
                    return True
                else:
                    print('Element not found in list[]')
                    return False

    def get(self, index):

        # Returns reference of item at given index.

        if not self.head:
            print('List is empty[]')

        elif index == 0:
            return self.head.value

        else:
            n = self.head
            node_assigned_index = 0
            while node_assigned_index < index:
                if not n.nxt:
                    return None
                else:
                    node_assigned_index += 1
                    n = n.nxt
            else:
                return n.value

    def dump(self):

        # Debugging func that dumps content of the list.

        if not self.head:
            print('List is empty[]')

        elif self.head and not self.head.nxt:
            return f'[{self.head.value}]'

        else:
            dump = f'['
            n = self.head
            while n.nxt != None:
                dump += n.value + ', '
                n = n.nxt
            else:
                dump += n.value + ']'

            return dump

if __name__ == '__main__':

    pass


from pathlib import Path
from autodocx.nodes import *
import docx
from autodocx.pygments import *
from docx.shared import Pt, Inches
from pygments import highlight
from pygments.lexers import PythonLexer
from docx.enum.text import WD_ALIGN_PARAGRAPH
from docx.enum.text import WD_BREAK
from docx.oxml.ns import qn
from docx.oxml import OxmlElement
from autodocx.formats import list_number
from autodocx.docedequation import *
from autodocx.tokenize import *

def get_sub_toc(file, l, sec_num):
    with open(file, encoding='utf-8') as file:
        file_cached = []
        for i in file:
            file_cached.append(i)
    file_cached = "".join(file_cached).replace("\\", "")
    file_cached = file_cached.split("\n")
    # for i in range(len(file_cached)):
    #    file_cached[i] = file_cached[i].replace("\n", " ")
    z = 0
    y = 0
    x = 0
    for i in file_cached:
        if i[:6] == "### ":
            x += 1
            l.append(f"\t{i[6:]}\t{sec_num}{z}.{y}.{x}")
        elif i[:5] == "## ":
            x = 0
            y += 1
            l.append(f"\t{i[5:]}\t{sec_num}{z}.{y}")
        elif i[:4] == "# ":
            x = 0
            y = 0
            z += 1
            l.append(f"\t{i[4:]}\t{sec_num}{z}")
    return l

def setcol(doc, num):
    section = doc.add_section(0)
    sectPr = section._sectPr
    cols = sectPr.xpath('./w:cols')[0]
    cols.set(qn('w:num'), str(num))

def add_pagenum(doc, section):
    doc.is_linked_to_previous = False
    paragraph = doc.paragraphs[-1]
    paragraph.text = f"{section}\t"
    paragraph.style = "header top"
    paragraph.paragraph_format.tab_stops.add_tab_stop(Inches(4.8), 2)
    run = paragraph.add_run()
    fldChar = OxmlElement('w:fldChar')
    fldChar.set(qn('w:fldCharType'), 'begin')
    instrText = OxmlElement('w:instrText')
    instrText.set(qn('xml:space'), 'preserve')
    instrText.text = 'PAGE \\*Arabic \\* MERGEFORMAT'
    fldChar2 = OxmlElement('w:fldChar')
    fldChar2.set(qn('w:fldCharType'), 'separate')
    fldChar3 = OxmlElement('w:t')
    fldChar3.text = ""
    fldChar2.append(fldChar3)

    fldChar4 = OxmlElement('w:fldChar')
    fldChar4.set(qn('w:fldCharType'), 'end')

    r_element = run._r
    r_element.append(fldChar)
    r_element.append(instrText)
    r_element.append(fldChar2)
    r_element.append(fldChar4)

def add_toc_section(doc):
    section = doc.sections[-1]
    add_pagenum(section.header, "Table of Contents")
    #sectPr = section._sectPr
    #cols = sectPr.xpath('./w:cols')[0]
    # cols.set(qn('w:num'),'2')

    paragraph = doc.paragraphs[-1]
    paragraph.text = "Table of Contents"
    paragraph.style = "Heading TOC"
    run = paragraph.add_run()
    fldChar = OxmlElement('w:fldChar')  # creates a new element
    fldChar.set(qn('w:fldCharType'), 'begin')  # sets attribute on element
    instrText = OxmlElement('w:instrText')
    instrText.set(qn('xml:space'), 'preserve')  # sets attribute on element
    # change 1-3 depending on heading levels you need
    instrText.text = 'TOC \\o "1-3" \\h \\z \\u'
    fldChar2 = OxmlElement('w:fldChar')
    fldChar2.set(qn('w:fldCharType'), 'separate')
    fldChar3 = OxmlElement('w:t')
    fldChar3.text = ""
    fldChar2.append(fldChar3)

    fldChar4 = OxmlElement('w:fldChar')
    fldChar4.set(qn('w:fldCharType'), 'end')

    r_element = run._r
    r_element.append(fldChar)
    r_element.append(instrText)
    r_element.append(fldChar2)
    r_element.append(fldChar4)

def add_index_section(doc):
    section = doc.add_section(4)
    sectPr = section._sectPr
    cols = sectPr.xpath('./w:cols')[0]
    cols.set(qn('w:num'), '2')

    add_pagenum(section.header, "Index")

    doc.add_paragraph("Index", 'Heading 1')
    section = doc.add_section(0)
    sectPr = section._sectPr
    cols = sectPr.xpath('./w:cols')[0]
    cols.set(qn('w:num'), '2')

    paragraph = doc.add_paragraph()
    run = paragraph.add_run()
    fldChar = OxmlElement('w:fldChar')  # creates a new element
    fldChar.set(qn('w:fldCharType'), 'begin')  # sets attribute on element
    instrText = OxmlElement('w:instrText')
    instrText.set(qn('xml:space'), 'preserve')  # sets attribute on element
    instrText.text = 'INDEX \\h "A" \\c "3" \\z "1033"'
    fldChar2 = OxmlElement('w:fldChar')
    fldChar2.set(qn('w:fldCharType'), 'separate')
    fldChar3 = OxmlElement('w:t')
    fldChar3.text = ""
    fldChar2.append(fldChar3)

    fldChar4 = OxmlElement('w:fldChar')
    fldChar4.set(qn('w:fldCharType'), 'end')

    r_element = run._r
    r_element.append(fldChar)
    r_element.append(instrText)
    r_element.append(fldChar2)
    r_element.append(fldChar4)

def MarkIndexEntry(entry, paragraph):
    run = paragraph.add_run()
    r = run._r
    fldChar = OxmlElement('w:fldChar')
    fldChar.set(qn('w:fldCharType'), 'begin')
    r.append(fldChar)

    run = paragraph.add_run()
    r = run._r
    instrText = OxmlElement('w:instrText')
    instrText.set(qn('xml:space'), 'preserve')
    instrText.text = ' XE "%s" ' % (entry)
    r.append(instrText)

    run = paragraph.add_run()
    r = run._r
    fldChar = OxmlElement('w:fldChar')
    fldChar.set(qn('w:fldCharType'), 'end')
    r.append(fldChar)

def match_star(tokens, With):
    matched_nodes = []
    consumed = 0
    parser = With

    while consumed < tokens.length():
        node = parser(tokens.offset(consumed))
        if type(node) == nullNode:
            break
        matched_nodes.append(node)
        consumed += node.consumed
    return matched_nodes, consumed

def match_stars(tokens, Withs):
    matched_nodes = []
    consumed = 0
    parsers = Withs

    while consumed < tokens.length():
        node = match_first(tokens.offset(consumed), parsers)
        if type(node) == nullNode:
            break
        matched_nodes.append(node)
        consumed += node.consumed
    return matched_nodes, consumed

def match_first(tokens, matches):
    for j in matches:
        node = j(tokens)
        if type(node) != nullNode:
            return node
    return nullNode()

class parser():
    def Parse_Text(tokens):
        if tokens.peek(["TEXT"]):
            return Node("TEXT", tokens.grab(0).value, 1)
        return nullNode()

    def single_newline(tokens):
        node = nullNode()
        if tokens.peek(["NEWLINE"]):
            node = Node("TEXT", " ", 1)
        if tokens.peek(["NEWLINE", "NEWLINE"]):
            return nullNode()
        return node

    def Parse_Bold(tokens):
        if tokens.peek_or([["STAR", "STAR", "TEXT", "STAR", "STAR"], ["UNDERSCORE", "UNDERSCORE", "TEXT", "UNDERSCORE", "UNDERSCORE"]]):
            return Node("BOLD", tokens.grab(2).value, 5)
        return nullNode()

    def Parse_Emph(tokens):
        if tokens.peek_or([["STAR", "TEXT", "STAR"], ["UNDERSCORE", "TEXT", "UNDERSCORE"]]):
            return Node("EMPH", tokens.grab(1).value, 3)
        return nullNode()

    def Parse_Tag(tokens):
        if tokens.peek(["TAGO", "TEXT", "TAGC"]):
            return Node("TAG", tokens.grab(1).value, 3)
        return nullNode()

    def Parse_Tags(tokens):
        node = match_first(tokens, [parser.single_newline, parser.Parse_Tag])
        if type(node) == nullNode:
            return nullNode()
        if not tokens.peek_at(node.consumed, ['NEWLINE', 'NEWLINE']):
            return nullNode()
        node.consumed += 2
        return node

    def H1_Parser(tokens):
        if tokens.peek(["HASH", "TEXT"]):
            return HeadNode("HEAD1", tokens.grab(1).value, 2)
        return nullNode()

    def H2_Parser(tokens):
        if tokens.peek(["HASH", "HASH", "TEXT"]):
            return HeadNode("HEAD2", tokens.grab(2).value, 3)
        return nullNode()

    def H3_Parser(tokens):
        if tokens.peek(["HASH", "HASH", "HASH", "TEXT"]):
            return HeadNode("HEAD3", tokens.grab(3).value, 4)
        return nullNode()

    def Header_Parser(tokens):
        node = match_first(
            tokens, [parser.H1_Parser, parser.H2_Parser, parser.H3_Parser])
        if type(node) == nullNode:
            return nullNode()
        if not tokens.peek_at(node.consumed, ['NEWLINE', 'NEWLINE']):
            return nullNode()
        node.consumed += 2
        return node

    def L1_Parser(tokens):
        if tokens.peek(["STAR"]):
            return Node("LIST1", "", 1)
        return nullNode()

    def L2_Parser(tokens):
        if tokens.peek(["PLUS"]):
            return Node("LIST2", "", 1)
        return nullNode()

    def L3_Parser(tokens):
        if tokens.peek(["MINUS"]):
            return Node("LIST3", "", 1)
        return nullNode()

    def Item_Parser(tokens):
        return match_first(tokens, [parser.Sentence_Parser, parser.L1_Parser, parser.L2_Parser, parser.L3_Parser])

    def List_Parser(tokens):
        nodes, consumed = match_star(tokens, parser.Item_Parser)
        if nodes == []:
            return nullNode()
        if not tokens.peek_at(consumed, ['NEWLINE', 'NEWLINE']):
            return nullNode()
        consumed += 2
        return ListNode(nodes, consumed)

    def Code_Start_Parser(tokens):
        if tokens.peek(["GRAVE", "GRAVE", "GRAVE"]):
            return Node("CODE", "", 3)
        return nullNode()

    def Code_Parser(tokens):
        return match_first(tokens, [parser.Code_Start_Parser, parser.single_newline, parser.Parse_Text])

    def Code_Parser_Multiline(tokens):
        nodes, consumed = match_star(tokens, parser.Code_Parser)
        if len(nodes) < 2:
            return nullNode()
        if not tokens.peek_at(consumed, ['NEWLINE', 'NEWLINE']):
            return nullNode()
        consumed += 2
        return CodeNode(nodes, consumed)

    def Sentence_Parser(tokens):
        return match_first(tokens, [parser.Parse_Text, parser.Parse_Emph, parser.Parse_Bold, parser.single_newline])

    def Sentences_NL_Parser(tokens):
        nodes, consumed = match_star(tokens, parser.Sentence_Parser)
        if nodes == []:
            return nullNode()
        if not tokens.peek_at(consumed, ['NEWLINE', 'NEWLINE']):
            return nullNode()
        consumed += 2
        return ParagraphNode(nodes, consumed)

    def Sentences_EOF_Parser(tokens):
        nodes, consumed = match_star(tokens, parser.Sentence_Parser)
        if nodes == []:
            return nullNode()
        if tokens.peek_at(consumed, ['EOF']):
            consumed += 1
        elif tokens.peek_at(consumed, ['NEWLINE', 'EOF']):
            consumed += 2
        else:
            return nullNode()
        return ParagraphNode(nodes, consumed)

    def Parse_Paragraph(tokens):
        return match_first(tokens, [parser.Parse_Tags, parser.Sentences_NL_Parser, parser.Sentences_EOF_Parser, parser.Code_Parser_Multiline, parser.Header_Parser, parser.List_Parser])

    def Parse_Body(tokens):
        nodes, consumed = match_star(tokens, parser.Parse_Paragraph)
        if nodes == []:
            return nullNode()
        return BodyNode(nodes, consumed)

def parse(tokens, file):
    body = parser.Parse_Body(tokens)
    if body.consumed != -1 + tokens.length():
        print(f"error in {file.name} at {tokens.grab(body.consumed).at}")
    return body

def add_to_doc(doc, parsed, file, main):
    if type(parsed) is not nullNode:
        for i in parsed.paragraphs:
            if type(i) is HeadNode or type(i) is Node or type(i) is nullNode:
                if main:
                    if i.type == "HEAD1":
                        doc.add_paragraph(i.value, style="Heading 3")
                    elif i.type == "HEAD2":
                        doc.add_paragraph(i.value, style="Heading 4")
                    elif i.type == "HEAD3":
                        doc.add_paragraph(i.value, style="Heading 5")
                else:
                    if i.type == "HEAD1":
                        doc.add_paragraph(i.value, style="Heading 1")
                    elif i.type == "HEAD2":
                        doc.add_paragraph(i.value, style="Heading 2")
                    elif i.type == "HEAD3":
                        doc.add_paragraph(i.value, style="Heading 3")
                if i.type == "TAG":
                    if i.value == "3COL":
                        setcol(doc, 3)
                    elif i.value == "1COL":
                        setcol(doc, 1)
                    elif i.value == "2COL":
                        setcol(doc, 2)
            elif type(i) is ListNode:
                p = None
                for j in i.sentences:
                    if j.type == "LIST1":
                        p = doc.add_paragraph(style="List Bullet")
                    elif j.type == "LIST2":
                        p = doc.add_paragraph(style="List Bullet 2")
                    elif j.type == "LIST3":
                        p = doc.add_paragraph(style="List Bullet 3")
                    elif j.value != "" and j.value != " " and j.type == "TEXT":
                        if p is None:
                            p = doc.add_paragraph(style="List Bullet")
                        p.add_run(f"{j.value} ")
                    elif j.value != "" and j.value != " " and j.type == "EMPH":
                        r = p.add_run(f"{j.value} ")
                        r.italic = True
                    elif j.value != "" and j.value != " " and j.type == "BOLD":
                        r = p.add_run(f"{j.value} ")
                        r.bold = True
            elif type(i) is CodeNode:
                code = ""
                for j in i.sentences:
                    if j.type == "TEXT":
                        if j.value != " ":
                            code += (j.value + "\n")
                fmt = Word()
                highlight(code.strip("\n"), PythonLexer(), fmt, outfile=doc)
            else:
                p = doc.add_paragraph(style="Normal")
                for j in i.sentences:
                    if j.type == "TEXT":
                        p.add_run(f"{j.value}")
                    elif j.value != "" and j.value != " " and j.type == "EMPH":
                        r = p.add_run(f"{j.value}")
                        r.italic = True
                    elif j.value != "" and j.value != " " and j.type == "BOLD":
                        r = p.add_run(f"{j.value}")
                        r.bold = True


"""
@author OwarMag1c
@desc 豆瓣网站爬取解析模块
@date 2022/3/12
说明: 爬取豆瓣特定页数据，并进行解析
使用: 主要接口crawl_douban(web_config)，返回爬取并解析后的电影数据
"""

from distutils.file_util import move_file
import requests
from bs4 import BeautifulSoup
import pandas as pd
from fake_useragent import UserAgent

import sys
import os
now_path = os.getcwd()
util_path = now_path + r"/../util"
sys.path.append(util_path)
from tag_recommend_system_util import get_cur_info
from tag_recommend_system_util import append_project_path
append_project_path()

ua = UserAgent()
headers = {
  'User-Agent': ua.random,
}

def crawl_douban(web_config: list):
  """爬取豆瓣top250的电影content，并且解析，返回所有电影解析后的信息"""
  print(get_cur_info() + "crawl douban content!")
  url = web_config['url']
  film_list = []
  # 遍历top250的电影
  for index in range(0,10):
    next_url = url + "?start=" + str(index * 25)
    response = requests.get(next_url, headers=headers, timeout=10)
    soup = BeautifulSoup(response.text, 'lxml')
    parse_duoban(soup, film_list)
  return film_list

def parse_duoban(soup, film_list: list):
  """解析豆瓣的content，结果加入movie_list"""
  div_list = soup.find_all('div', class_='info')
  for each in div_list:
    title = each.find('div', class_='hd').a.span.text.strip()
    each_str = str(each.find('div', class_='hd'))
    url_start_index = each_str.find('href=')
    url_end_index = each_str[:len(each_str)].find(r'/">')
    url = each_str[url_start_index + len(r'href="'):url_end_index + 1].strip()
    rating = each.find('span', class_='rating_num').text.strip()
    try:
        quote = each.find('span', class_='inq').text.strip()
    except:
        quote = ""
    info = each.find('div', class_='bd').p.text.strip()
    info = info.replace("\n", " ").replace("\xa0", " ")
    info = ' '.join(info.split())
    dates = ""
    for c in info:
      if(c.isdigit() == True):
        dates += c
    info_set = info.split(' ' + dates + ' / ')
    try:
      directors_and_actors = info_set[0]
      info_set = info_set[1].split(' / ')
      areas = info_set[0]
      tags = info_set[1].split(' ')
    except:
      directors_and_actors = ""
      areas = ""
      tags = ""
    film_info = {'title':title, 'rating': rating, 'dates': dates, 'tags': tags, 'areas': areas, 'directors_and_actors': directors_and_actors, 'quote': quote}
    film_list.append(film_info)

    # file = open('douban_info', 'a')
    # file.write(str(film_info) + '\n')
    # file.close()


import os
import shutil
from time import sleep
import discord
from discord import SyncWebhook
from aiogram.contrib.fsm_storage.memory import MemoryStorage
from aiogram.dispatcher import FSMContext
from aiogram.dispatcher.filters.state import StatesGroup, State
from aiogram import Bot, Dispatcher, executor, types
from asyncio import sleep
from time import sleep
import threading
from db_import import db, GROUPS, GUILDS, LINKED, TG_LINKED, WEBHOOKS
from tokens import TG_TOKEN

if not os.path.isdir("tempdata"):
    os.mkdir("tempdata")

access_error = 'Для данного действия необходимо обладать\nправами администратора телеграм-группы!'
bot = Bot(token=TG_TOKEN)
dp = Dispatcher(bot, storage=MemoryStorage())

# class UserState(StatesGroup):
#     unlink_discord = State()
#     forecast = State()

@dp.message_handler(commands=['help', 'start', 'info', 'hey'])
async def send_welcome(message: types.Message):
    await message.reply("*__Вас приветствует RotexBot\!__*\n\n"
                        "Я разработан специально для организации двусторонней\n"
                        "связи между Telegram чатами и Discord каналами\n\n"
                        "*Команды, которые я поддерживаю:*\n"
                        "/help \- помощь\n"
                        "/id \- узнать id данной телеграм группы\n"
                        "/discord \- привязать к чату дискорд канал\n"
                        "/undiscord \- отвязать дискорд канал от чата",
                        parse_mode='MarkdownV2')

@dp.message_handler(commands=['undiscord', 'андискорд', 'unlink', 'отвязать'])
async def unlink_discord(message: types.Message):
    user = await bot.get_chat_member(message.chat.id, message.from_user.id)
    if not user.is_chat_admin():
        respond = await message.reply(access_error)
        await sleep(5)
        await respond.delete()
        return

    if db.get(GROUPS, str(message.chat.id)).LINKED_CHANNEL_ID:
        buttons = [
            [
                types.InlineKeyboardButton(text="отвязать", callback_data="отвязать"),
                types.InlineKeyboardButton(text="отменить", callback_data="отменить"),
            ],
        ]
        keyboard = types.InlineKeyboardMarkup(inline_keyboard=buttons)
        await message.answer('Вы уверены, что хотите\nотвязать дискорд сервер?', reply_markup=keyboard)
    else:
        await message.reply("Данная телеграм-группа не связана\nни с одним дискордм сервером!")

@dp.callback_query_handler()
async def callback_func(callback: types.CallbackQuery):
    chat_id = str(callback.message.chat.id)
    user = await bot.get_chat_member(int(chat_id), callback.from_user.id)
    if callback.data == 'отвязать':
        if user.is_chat_admin():
            for o in db.query(GUILDS).all():
                if o.TG_LINKED_CHANNELS_ID and db.get(TG_LINKED, chat_id).LINKED_CHANNEL_ID in o.TG_LINKED_CHANNELS_ID:
                    guild = o
                    break
            group = db.get(GROUPS, chat_id)
            linked_group = db.get(TG_LINKED, chat_id)
            linked_channel = db.get(TG_LINKED, linked_group.LINKED_CHANNEL_ID)
            webhook = db.get(WEBHOOKS, linked_channel.CHANNEL_ID)
            group.LINKED_CHANNEL_ID = None
            if ':' in guild.TG_LINKED_CHANNELS_ID:
                groups = guild.TG_LINKED_CHANNELS_ID.split(":")
                groups.remove(linked_channel.CHANNEL_ID)
                guild.TG_LINKED_CHANNELS_ID = ":".join(groups)
            else:
                guild.TG_LINKED_CHANNELS_ID = None
            db.delete(linked_group)
            db.delete(webhook)
            db.delete(linked_channel)
            db.add_all([group, guild])
            db.commit()
            await callback.message.answer('Отвязка завершена успешно!')
            await callback.message.delete()
        else:
            respond = await callback.message.reply(access_error)
            await sleep(5)
            await respond.delete()
    elif callback.data == 'отменить':
        if user.is_chat_admin():
            await callback.message.delete()
        else:
            respond = await callback.message.reply(access_error)
            await sleep(5)
            await respond.delete()

@dp.message_handler(commands=['discord', 'дискорд', 'link_discord', 'привязать_дискорд'])
async def link_discord(message: types.Message):
    user = await bot.get_chat_member(message.chat.id, message.from_user.id)
    if not user.is_chat_admin():
        respond = await message.reply(access_error)
        await sleep(15)
        await respond.delete()
        return
    if db.get(GROUPS, str(message.chat.id)).LINKED_CHANNEL_ID:
        await message.reply(f'Данная телеграм группа уже связана\n'
                            f'с дискорд каналом, id которого:\n'
                            f'{db.get(GROUPS, str(message.chat.id)).LINKED_CHANNEL_ID}\n\n'
                            f'Если хотите отвязать дискорд канал,\n'
                            f'воспользуйтесь командой: /undiscord')
        return
    if ' ' in message.text:
        ds_guild_id = message.text.split()[1]
        tg_group_id = str(message.chat.id)
    else:
        await message.reply('Чтобы привязать дискорд канал к телеграм гурппе,\n'
                            'нужно установить дискорд бота Rotex\n'
                            'на целевой дискорд сервер.\n'
                            'Ссылка для приглашения дискорд бота:'
                            'https://discord.com/api/oauth2/authorize?client_id=1088509697524519115&permissions=8&scope=bot%20applications.commands\n\n'
                            'После этого, с дискорд сервера необходимо отправить\n'
                            'приглашение на создание общего чата при помощи команды\n'
                            '/link_telegram , использованной в дискорд сервере.\n\n'
                            'После того, как приглашение создать общий чат придёт\n'
                            'в эту телеграм-группу, необходимо узнать id\n'
                            'дискорд сервера,где находится канал, а затем\n'
                            'использовать команду /discord [id дисокрд сервера]\n'
                            'Чтобы завершить привязку.\n'
                            'Пример использования команды:\n\n'
                            '/discord 10832845374377255078\n\n'
                            '(узнать id дискорд сервера можно с помощью команды /id\n'
                            'использованной в желаемом дискорд сервере)')
        return

    if not db.get(GUILDS, ds_guild_id):
        await message.reply('ОШИБКА!\n'
                            'Пожалуйста, проверьте правильность введённых данных!\n\n'
                            'P.S. для работы общего telegram-discord чата\n'
                            'На дискорд сервер также необходимо пригласить бота Rotex.\n'
                            'Ссылка для приглашения дискорд-бота:\n'
                            'https://discord.com/api/oauth2/authorize?client_id=1088509697524519115&permissions=8&scope=bot%20applications.commands')
        return
    if db.get(GROUPS, tg_group_id).LINKED_CHANNEL_ID:
        await message.reply('ОШИБКА!\n'
                            'Эта группа уже связанна с дискорд сервером!\n\n'
                            'Отвязать дискорд сервер:\n'
                            '/undiscord')
    if not db.get(GUILDS, ds_guild_id).TG_LINK_WATING_GROUP_ID and not db.get(GUILDS,
                                                                              ds_guild_id).TG_LINK_WATING_CHANNEL_ID:
        await message.reply(f"На данный момент у вас нет вхожящих\n"
                            f"запросов на создание общего дискорд-телеграм чата.\n"
                            f"Если желаете установить двустороннее соединение\n"
                            f"с дискорд сервером, попросите его администратора\n"
                            f"отправить вам запрос на создание общего\n"
                            f"дискорд-телеграм чата.\n\n"
                            f"Id данной телеграм группы: {message.chat.id}")
        return
    else:
        try:
            guild_object = db.get(GUILDS, ds_guild_id)
            group_object = db.get(GROUPS, tg_group_id)
            ds_channel_id = guild_object.TG_LINK_WATING_CHANNEL_ID

            if guild_object.TG_LINKED_CHANNELS_ID:
                channels_list = guild_object.TG_LINKED_CHANNELS_ID.split(":")
                if ds_channel_id not in channels_list:
                    channels_list.append(ds_channel_id)
                guild_object.TG_LINKED_CHANNELS_ID = ":".join(channels_list)
            else:
                guild_object.TG_LINKED_CHANNELS_ID = ds_channel_id
            group_object.LINKED_CHANNEL_ID = ds_channel_id
            db.add(group_object)
            db.add(guild_object)
            db.commit()
            if not db.get(TG_LINKED, tg_group_id) and not db.get(TG_LINKED, ds_channel_id):
                channel_link = TG_LINKED(CHANNEL_ID=ds_channel_id, LINKED_CHANNEL_ID=tg_group_id)
                group_link = TG_LINKED(CHANNEL_ID=tg_group_id, LINKED_CHANNEL_ID=ds_channel_id)
                db.add(channel_link)
                db.add(group_link)
                db.commit()
            group_object.LINK_WAITING_GUILD_ID = None
            guild_object.TG_LINK_WATING_GROUP_ID = None
            guild_object.TG_LINK_WATING_CHANNEL_ID = None
            db.add(guild_object)
            db.add(group_object)
            db.commit()
            await message.reply("Привязка завершена успешно!\n"
                                "(если хотите отвязать дискорд канал,\n"
                                "воспользуйтесь /undiscord )")
        except Exception as e:
            print(e)

@dp.message_handler(commands=['id', 'айди', 'ID', 'Id', 'iD'])
async def group_id(message: types.Message):
    await message.reply(
        f"Айди данной телеграмм группы:\n\n{message.chat.id}\n\nВнимание! Минус перед цифрами (если он есть)\nтак же является частью id!")

@dp.message_handler(content_types=['new_chat_members'])
async def join_group(message: types.Message):
    bot_object = await bot.get_me()
    chat_id = str(message.chat.id)
    for chat_member in message.new_chat_members:
        if chat_member.id == bot_object.id and not db.get(GROUPS, chat_id):
            GROUP = GROUPS(GROUP_ID=chat_id)
            db.add(GROUP)
            db.commit()
    print(f"Телеграм бот стал членом группы с айди:   {chat_id}")

@dp.message_handler(content_types=['left_chat_member'])
async def leave_group(message: types.Message):
    bot_object = await bot.get_me()
    chat_id = str(message.chat.id)
    if message.left_chat_member.id == bot_object.id and db.get(GROUPS, chat_id):
        db.delete(db.get(GROUPS, chat_id))

        if db.get(TG_LINKED, chat_id):
            channel_id = db.get(TG_LINKED, chat_id).LINKED_CHANNEL_ID
            db.delete(db.get(TG_LINKED, chat_id))
            db.delete(db.get(TG_LINKED, channel_id))
        if db.get(WEBHOOKS, channel_id):
            db.delete(db.get(WEBHOOKS, channel_id))
        for guild_object in db.query(GUILDS).all():
            if guild_object.TG_LINKED_CHANNELS_ID and channel_id in guild_object.TG_LINKED_CHANNELS_ID.split(':'):
                if ':' in guild_object.TG_LINKED_CHANNELS_ID:
                    channels_list = guild_object.TG_LINKED_CHANNELS_ID.split(":")
                    channels_list.remove(channel_id)
                    guild_object.TG_LINKED_CHANNELS_ID = ":".join(channels_list)
                else:
                    guild_object.TG_LINKED_CHANNELS_ID = None
                    db.add(guild_object)
                break
        db.commit()
    print(f"Телеграм бот покинул группу с айди:   {chat_id}")

@dp.message_handler(content_types=types.ContentTypes.ANY)
async def resend(message: types.Message):
    if not db.get(GROUPS, str(message.chat.id)):
        return
    if db.get(GROUPS, str(message.chat.id)).LINKED_CHANNEL_ID:
        try:
            text = None
            files = []
            file_size_error = 'Вложение не было доставлено в дискорд канал из за превышения допустимого размера(25Mb)'

            if message.text:
                text = message.text
            if message.voice:
                if message.voice.file_size >= 26214400:
                    respond = await message.reply(file_size_error)
                    await sleep(15)
                    await respond.delete()
                else:
                    file = await bot.get_file(message.voice.file_id)
                    await bot.download_file(file.file_path, f'tempdata/voice_message.mp3')
                    files.append(discord.File('tempdata/voice_message.mp3'))
            elif message.audio:
                if message.audio.file_size >= 26214400:
                    respond = await message.reply(file_size_error)
                    await sleep(15)
                    await respond.delete()
                else:
                    file = await bot.get_file(message.audio.file_id)
                    await bot.download_file(file.file_path, f'tempdata/{message.audio.title}.mp3')
                    files.append(discord.File(f'tempdata/{message.audio.title}.mp3'))
                text = message.caption
            elif message.photo:
                if message.photo[-1]['file_size'] >= 26214400:
                    respond = await message.reply(file_size_error)
                    await sleep(15)
                    await respond.delete()
                else:
                    file = await bot.get_file(message.photo[-1]["file_id"])
                    await bot.download_file(file.file_path,
                                            f'tempdata/{message.photo[-1].file_unique_id}.{file.file_path.split(".")[-1]}')
                    files.append(
                        discord.File(f'tempdata/{message.photo[-1].file_unique_id}.{file.file_path.split(".")[-1]}'))
                text = message.caption
            elif message.video:
                if message.video.file_size >= 26214400:
                    respond = await message.reply(file_size_error)
                    await sleep(15)
                    await respond.delete()
                else:
                    file = await bot.get_file(message.video.file_id)
                    await bot.download_file(file.file_path, f'tempdata/{message.video.file_name}')
                    files.append(discord.File(f'tempdata/{message.video.file_name}'))
                text = message.caption
            elif message.video_note:
                if message.video_note.file_size >= 26214400:
                    respond = await message.reply(file_size_error)
                    await sleep(15)
                    await respond.delete()
                else:
                    file = await bot.get_file(message.video_note.file_id)
                    await bot.download_file(file.file_path,
                                            f'tempdata/videomessage_from_{message.from_user.first_name}.{file.file_path.split(".")[-1]}')
                    files.append(discord.File(
                        f'tempdata/videomessage_from_{message.from_user.first_name}.{file.file_path.split(".")[-1]}'))
            elif message.animation:
                if message.animation.file_size >= 26214400:
                    respond = await message.reply(file_size_error)
                    await sleep(15)
                    await respond.delete()
                else:
                    file = await bot.get_file(message.animation.file_id)
                    if not file.file_path.endswith('tgs'):
                        await bot.download_file(file.file_path, f'tempdata/{message.animation.file_name}')
                        files.append(discord.File(f'tempdata/{message.animation.file_name}'))
                text = message.caption
            elif message.document:
                if message.document.file_size >= 26214400:
                    respond = await message.reply(file_size_error)
                    await sleep(15)
                    await respond.delete()
                else:
                    file = await bot.get_file(message.document.file_id)
                    await bot.download_file(file.file_path, f'tempdata/{message.document.file_name}')
                    files.append(discord.File(f'tempdata/{message.document.file_name}'))
                text = message.caption
            elif message.sticker:
                if message.sticker.file_size >= 26214400:
                    respond = await message.reply(file_size_error)
                    await sleep(15)
                    await respond.delete()
                else:
                    file = await bot.get_file(message.sticker.file_id)
                    if file.file_path.endswith('tgs'):
                        pass
                    else:
                        await bot.download_file(file.file_path,
                                                f'tempdata/{message.sticker.file_unique_id}.{file.file_path.split(".")[-1]}')
                        files.append(
                            discord.File(f'tempdata/{message.sticker.file_unique_id}.{file.file_path.split(".")[-1]}'))
                text = message.caption
            try:
                if text or files:
                    webhook = SyncWebhook.from_url(
                        db.get(WEBHOOKS, (db.get(GROUPS, str(message.chat.id)).LINKED_CHANNEL_ID)).WEBHOOK_URL)
                    webhook.send(content=text,
                                 username=f'{message.from_user.first_name}〔TG〕',
                                 avatar_url="https://logos-download.com/wp-content/uploads/2016/07/Telegram_2.x_version_2014_Logo.png",
                                 files=files)
            except Exception as e:
                await message.answer(f"Если вы видите данное сообщение,\n"
                                     f"значит произошла программная ошибка,\n"
                                     f"из за которой сообщение не было доставлено\n"
                                     f"в дискорд канал, связанный с данной группой\.\n\n"
                                     f"Приносим свои извинения за произошедшее, отчёт\n"
                                     f"о сбое уже оправлен разработчикам для\n"
                                     f"вяснения причин и устранения неполадок\."
                                     f"Если данная ошибка стала появляться регулярно,\n"
                                     f"воспользуйтесь /undiscord , а затем перенастройте связь\n"
                                     f"с дискород-сервером с помощью /discord"
                                     f"Код ошибки:\n"
                                     f"```{e}```", parse_mode="'MarkdownV2")
                print(f"АХТУНГ!!! АХТУНГ!!! АХТУНГ!!!\n"
                      f"сбой в работе ds-tg канала (ошибка вебхука). Код ошибки:\n"
                      f"{e}")
        except Exception as e:
            await message.answer(f"Если вы видите данное сообщение,\n"
                                 f"значит произошла программная ошибка,\n"
                                 f"из за которой сообщение не было доставлено\n"
                                 f"в дискорд канал, связанный с данной группой\.\n\n"
                                 f"Приносим свои извинения за произошедшее, отчёт\n"
                                 f"о сбое уже оправлен разработчикам для\n"
                                 f"вяснения причин и устранения неполадок\."
                                 f"Если данная ошибка стала появляться регулярно,\n"
                                 f"воспользуйтесь /undiscord , а затем перенастройте связь\n"
                                 f"с дискород-сервером с помощью /discord"
                                 f"Код ошибки:\n"
                                 f"```{e}```", parse_mode="'MarkdownV2")
            print(f"АХТУНГ!!! АХТУНГ!!! АХТУНГ!!!\n"
                  f"сбой в работе ds-tg канала. Код ошибки:\n"
                  f"{e}")

def ready():
    print('Telegram-bot started successfully!')

def clearing():
    while True:
        sleep(60)
        try:
            shutil.rmtree('tempdata')
            os.mkdir("tempdata")
        except:
            continue

t = threading.Thread(target=clearing, daemon=True)
t.start()
executor.start_polling(dp, on_startup=ready())


import time

import numpy as np
import numpy.fft as fft
import pandas as pd

import scipy.signal as signal
import scipy.io.wavfile as wav

import simpleaudio as sa
import sounddevice as sd

import matplotlib.pyplot as plt

def get_chirp(signal_length=1.0, impulse_length=0.3, tukey_length=0.05, freq=(200, 4000), sample_rate=48000, method="linear"):
    signal_wave = np.zeros(int(sample_rate * signal_length), np.float32)

    chirp_wave = np.arange(0, impulse_length * sample_rate, dtype=np.float32)
    chirp_wave = chirp_wave / np.max(chirp_wave)
    chirp_wave = signal.chirp(chirp_wave, freq[0], 1, freq[1], phi=-90, method=method)
    #chirp_wave = signal.tukey(len(chirp_wave), alpha=1) * chirp_wave
    chirp_wave = signal.tukey(len(chirp_wave), alpha=(tukey_length * 2)/impulse_length) * chirp_wave

    signal_wave[0:len(chirp_wave)] = chirp_wave

    return signal_wave

def mean_fft(wave, window_size = 4800):

    fft_signal = np.full(window_size, 0, dtype=np.complex)
    counter = 0

    for start in range(0, len(wave) - window_size, int(window_size / 2)):
        temp = fft.fft(signal.tukey(window_size, alpha=0.1) * wave[start:start+window_size])
        fft_signal = fft_signal + temp
        counter += 1

    fft_signal = fft_signal / counter

    return fft_signal

if __name__ == "__main__":
    methods = [
        "linear",
        #"quadratic",
        #"logarithmic",
        #"hyperbolic"
    ]
    lengths = [0.5]

    for length in lengths:
        for method in methods:
            wave = get_chirp(1.5, length, method=method)

            # same
            _, rec1 = wav.read("ISAS Labor/2/2/linear_0.5s_rec.wav")
            _, rec2 = wav.read("ISAS Labor/2/3/linear_0.5s_rec.wav")
            #_, rec3 = wav.read("ISAS Labor/3/3/linear_0.5s_rec.wav")

            # different place
            #_, rec1 = wav.read("ISAS Labor/2/2/linear_0.5s_rec.wav")
            #_, rec2 = wav.read("ISAS Labor/3/1/linear_0.5s_rec.wav")
            #_, rec3 = wav.read("ISAS Labor/3/2/linear_0.5s_rec.wav")

            # different room
            #_, rec1 = wav.read("ISAS Labor/2/2/linear_0.5s_rec.wav")
            #_, rec2 = wav.read("ISAS Raum Daniel Up/2/linear_0.5s_rec.wav")
            #_, rec3 = wav.read("ISAS Flur Gross/1/1/linear_0.5s_rec.wav")

            rec1 = np.array(rec1 / 32767.0, dtype=np.float)
            rec2 = np.array(rec2 / 32767.0, dtype=np.float)
            #rec3 = np.array(rec3 / 32767.0, dtype=np.float)

            audio = fft.ifft(fft.fft(rec2) / fft.fft(wave))
            audio = np.int16(np.absolute(audio)/np.max(np.absolute(audio)) * 32767)
            wav.write("response.wav".format(method, length), 48000, audio)

            plt.plot(np.arange(len(wave)) / 48000.0, wave, alpha=0.75, label="Reference Signal")
            plt.plot(np.arange(len(rec1)) / 48000.0, rec1, alpha=0.75, label="Recorded Signal")
            #plt.plot(np.arange(len(rec2)), rec2, alpha=0.33)
            plt.legend(loc="lower right")
            plt.xlabel("Time (s)")
            plt.ylabel("Signal")
            plt.savefig("signal-{}-{}s.png".format(method, length), dpi=300)
            plt.close()

            fft_wave = mean_fft(wave)
            fft_rec1 = mean_fft(rec1)
            fft_rec2 = mean_fft(rec2)
            #fft_rec3 = mean_fft(rec3)

            #fft_wave = fft.fft(wave)
            #fft_rec1 = fft.fft(rec1)
            #fft_rec2 = fft.fft(rec2)

            fft_response1 = fft_rec1 / fft_wave
            fft_response2 = fft_rec2 / fft_wave
            #fft_response3 = fft_rec3 / fft_wave

            plt.plot((np.arange(len(fft_wave)) - len(fft_wave) / 2.0) * 10, np.absolute(fft_wave), color="blue", alpha=0.33)
            plt.plot((np.arange(len(fft_rec1)) - len(fft_rec1) / 2.0) * 10, np.absolute(fft_rec1), color="orange", alpha=0.33)
            plt.plot((np.arange(len(fft_rec2)) - len(fft_rec2) / 2.0) * 10, np.absolute(fft_rec2), color="green", alpha=0.33)
            plt.savefig("freq-{}-{}s.png".format(method, length))
            #plt.show()
            plt.close()

            plt.scatter((np.arange(len(fft_response1)) - len(fft_response1)/2.0) * 10, np.angle(fft_response1), color="blue", alpha=0.75, s=1)
            plt.scatter((np.arange(len(fft_response2)) - len(fft_response2)/2.0) * 10, np.angle(fft_response2), color="orange", alpha=0.75, s=1)
            #plt.scatter(np.arange(len(fft_recorded)) - len(fft_wave)/2.0, np.angle(fft_recorded) - np.angle(fft_wave), color="red", alpha=0.5)
            plt.xlabel("Frequency (Hz)")
            plt.ylabel("Phase")
            plt.savefig("phase-{}-{}s.png".format(method, length), dpi=300)
            #plt.show()
            plt.close()

            y11 = np.ma.masked_where(np.absolute(fft_response1) <= np.percentile(np.absolute(fft_response1), 99), np.absolute(fft_response1))
            y12 = np.ma.masked_where(np.absolute(fft_response1) >= np.percentile(np.absolute(fft_response1), 99), np.absolute(fft_response1))

            y21 = np.ma.masked_where(np.absolute(fft_response2) <= np.percentile(np.absolute(fft_response2), 99), np.absolute(fft_response2))
            y22 = np.ma.masked_where(np.absolute(fft_response2) >= np.percentile(np.absolute(fft_response2), 99), np.absolute(fft_response2))

            #y31 = np.ma.masked_where(np.absolute(fft_response3) <= np.percentile(np.absolute(fft_response3), 90), np.absolute(fft_response3))
            #y32 = np.ma.masked_where(np.absolute(fft_response3) >= np.percentile(np.absolute(fft_response3), 90), np.absolute(fft_response3))

            x = (np.arange(len(fft_response1)) - len(fft_response1) / 2.0) * 10

            plt.figure(figsize=(12,5))
            plt.plot(x, np.absolute(fft_response1), alpha=0.35, linewidth=2)
            plt.plot(x, np.absolute(fft_response2), alpha=0.35, linewidth=2)
            plt.scatter(x, y11, alpha=1, s=10, label="Highest 1% Reference Signal")
            plt.scatter(x, y21, alpha=1, s=10, label="Highest 1% Same Position")
            #plt.plot(x, y31, alpha=0.66)
            #plt.plot(x, np.real(fft_response1), label="real", color="red", alpha=0.75)
            #plt.plot(x, np.imag(fft_response1), label="imaginary", color="green", alpha=0.75)

            #plt.plot(x, np.absolute(fft_response1), alpha=0.75)
            #plt.plot(x, np.absolute(fft_response2), alpha=0.75)
            #plt.plot((np.arange(len(fft_response12)) - len(fft_response12) / 2.0) * 10, np.absolute(fft_response12), alpha=0.5)
            #plt.plot((np.arange(len(fft_response21)) - len(fft_response21) / 2.0) * 10, np.absolute(fft_response21), alpha=0.5)
            #plt.plot((np.arange(len(fft_response22)) - len(fft_response22) / 2.0) * 10, np.absolute(fft_response22), alpha=0.5)
            #dots = np.where(np.absolute(fft_response1) > np.percentile(np.absolute(fft_response1), 99))[0] - len(fft_response1) / 2
            #plt.scatter(dots, np.full(len(dots), 300), color="black", alpha=0.5)
            #dots = np.where(np.absolute(fft_response2) > np.percentile(np.absolute(fft_response2), 99))[0] - len(fft_response2) / 2
            #plt.scatter(dots, np.full(len(dots), 300), color="red", alpha=0.5)
            #plt.legend(loc="upper left")
            plt.xlim((1500, 17000))
            plt.ylim((0, 500))
            plt.xlabel("Frequency (Hz)")
            plt.ylabel("Magnitude")
            plt.savefig("response-{}-{}s.png".format(method, length), dpi=300)
            #plt.show()
            plt.close()


from typing import List
from ._types import EvalDecentralisationPayload, EvalDecentralisationResponse, StakeOptimizePayload, StakeOptimizeResponse, StakingTarget

from ._evaluation import eval_decentralization
from ._optimizer import stake_optimize

from flask import Flask, request

app = Flask(__name__)

import json

@app.route("/api/dev", methods=["GET"])
def dev():
    """
    payload = EvalDecentralisationPayload(
        stakes=request.args.get("stakes"),
        total_supply=request.args.get("total_supply")
    )
    """

    return json.dumps({"response": "ok"})  # resp.to_json()

@app.route("/api/eval", methods=["POST"])
def eval_decentralisation():
    """
    payload = EvalDecentralisationPayload(
        stakes=request.args.get("stakes"),
        total_supply=request.args.get("total_supply")
    )
    """
    payload: EvalDecentralisationPayload = EvalDecentralisationPayload.from_dict(
        request.get_json())

    resp = EvalDecentralisationResponse(
        eval_decentralization(payload.stakes)  # , payload.total_supply)
    )

    return resp.to_json()  # resp.to_json()

@app.route("/api/optimize", methods=["POST"])
def optimize():
    payload: StakeOptimizePayload = StakeOptimizePayload.from_dict(
        request.get_json()
    )

    # compute weights
    weights = stake_optimize(payload.amount_to_stake, payload.validator_set)

    # compute new stakes
    to_stake_per_val = [
        weights[i] * payload.amount_to_stake for i in range(len(payload.validator_set))]
    new_stakes = [
        payload.validator_set[i].curr_stake + to_stake_per_val[i] for i in range(len(payload.validator_set))
    ]

    # compute our new decentralisation ratio
    decentralisation_ratio = eval_decentralization(new_stakes)

    # recompose for response
    targets: List[StakingTarget] = [
        StakingTarget(
            moniker=payload.validator_set[i].moniker,
            address=payload.validator_set[i].address,
            amount_to_stake=weights[i] * payload.amount_to_stake
        ) for i in range(len(payload.validator_set))
    ]

    # prune validators with 0 to stake
    targets = list(filter(lambda x: x.amount_to_stake > 0, targets))
    # prune validators with < 0.0000001 to stake
    targets = list(filter(lambda x: x.amount_to_stake > 0.000001, targets))

    # create response obj and return
    resp = StakeOptimizeResponse(
        expected_yield=0.0,
        decentralisation_ratio=decentralisation_ratio,
        targets=targets
    )
    return resp.to_json()


#!/usr/bin/env python
from __future__ import print_function

import math
import random
from collections import deque
from lib.plot_histogram import PlotHistogramRT
import sys

import numpy as np

class C51Agent:
    def __init__(self, state_size, action_size, num_atoms):

        # get size of state and action
        self.state_size = state_size
        self.action_size = action_size

        # these is hyper parameters for the DQN
        self.gamma = 0.99
        self.learning_rate = 0.0001
        self.epsilon = 0.1
        self.initial_epsilon = 1.0
        self.final_epsilon = 0.0001
        self.batch_size = 32
        # self.observe = 2000
        self.observe = 2000
        self.explore = 50000
        self.frame_per_action = 4
        self.update_target_freq = 3000
        self.timestep_per_train = 500  # Number of timesteps between training interval

        # Initialize Atoms
        self.num_atoms = num_atoms  # 51 for C51
        self.v_max = 30  # Max possible score for Defend the center is 26 - 0.1*26 = 23.4
        self.v_min = -20  # -0.1*26 - 1 = -3.6
        self.delta_z = (self.v_max - self.v_min) / float(self.num_atoms - 1)
        self.z = [self.v_min + i * self.delta_z for i in range(self.num_atoms)]

        # Create replay memory using deque
        self.memory = deque()
        self.max_memory = 50000  # number of previous transitions to remember

        # Models for value distribution
        self.model = None
        self.target_model = None

        # Performance Statistics
        self.ep_cum_reward = 0
        self.stats_window_size = 5  # window size for computing rolling statistics
        self.mavg_life = []  # Moving Average of Survival Time
        self.var_life = []  # Variance of Survival Time
        self.mavg_score = []  # Moving Average of Survival Time
        self.var_score = []  # Variance of Survival Time
        self.real_time_plotter = PlotHistogramRT(action_size, num_atoms)

    def update_target_model(self):
        """
        After some time interval update the target model to be same with model
        """
        self.target_model.set_weights(self.model.get_weights())

    def get_action(self, state):
        """
        Get action from model using epsilon-greedy policy
        """
        if np.random.rand() <= self.epsilon:
            # print("----------Random Action----------")
            action_idx = random.randrange(self.action_size)
        else:
            action_idx = self.get_optimal_action(state)

        return action_idx

    def get_optimal_action(self, state):
        """Get optimal action for a state
        """
        z = self.model.predict(state)  # Return a list [1x51, 1x51, 1x51]
        z_concat = np.vstack(z)

        # plotea la distribucion
        # self.real_time_plotter.update(np.array(self.z), z_concat)

        # esta multiplicacion esta en el articulo
        # es la probabilidad por el valor
        # en este caso regresa algo asi [1, 2, 3]
        q = np.sum(np.multiply(z_concat, np.array(self.z)), axis=1)

        # Pick action with the biggest Q value
        action_idx = np.argmax(q)

        return action_idx

    def shape_reward(self, r_t, misc, prev_misc, t):

        # Check any kill count
        # if (misc[0] > prev_misc[0]):
        #     r_t = r_t + 1
        #
        # if (misc[1] < prev_misc[1]):  # Use ammo
        #     r_t = r_t - 0.1
        #
        if misc['ale.lives'] < prev_misc['ale.lives']:  # Loss HEALTH
            r_t = r_t - 1
        self.ep_cum_reward += r_t
        return r_t

    # save sample <s,a,r,s'> to the replay memory
    def replay_memory(self, s_t, action_idx, r_t, s_t1, is_terminated, t):
        self.memory.append((s_t, action_idx, r_t, s_t1, is_terminated))
        if self.epsilon > self.final_epsilon and t > self.observe:
            self.epsilon -= (self.initial_epsilon - self.final_epsilon) / self.explore

        if len(self.memory) > self.max_memory:
            self.memory.popleft()

        # Update the target model to be same with model
        if t % self.update_target_freq == 0:
            self.update_target_model()

    # pick samples randomly from replay memory (with batch_size)
    def train_replay(self):

        num_samples = min(self.batch_size * self.timestep_per_train, len(self.memory))
        replay_samples = random.sample(self.memory, num_samples)

        # print ("=num_samples")
        # print (num_samples)
        # print("=memory shape")
        # print (self.memory[0][0].shape)

        state_inputs = np.zeros(((num_samples,) + self.state_size))
        next_states = np.zeros(((num_samples,) + self.state_size))
        m_prob = [np.zeros((num_samples, self.num_atoms)) for i in range(self.action_size)]
        action, reward, done = [], [], []

        # print("=state inputs")
        # print (state_inputs.shape)
        # print("=next states")
        # print (next_states.shape)
        # print("=m prob")
        # print (m_prob[0].shape)

        for i in range(num_samples):
            state_inputs[i, :, :, :] = replay_samples[i][0]
            action.append(replay_samples[i][1])
            reward.append(replay_samples[i][2])
            next_states[i, :, :, :] = replay_samples[i][3]
            done.append(replay_samples[i][4])

        z = self.model.predict(next_states)  # Return a list [32x51, 32x51, 32x51]
        z_ = self.target_model.predict(next_states)  # Return a list [32x51, 32x51, 32x51]

        # print("=z")
        # print(np.asarray(z).shape)

        # Get Optimal Actions for the next states (from distribution z)
        optimal_action_idxs = []
        z_concat = np.vstack(z)
        q = np.sum(np.multiply(z_concat, np.array(self.z)), axis=1)  # length (num_atoms x num_actions)

        # print("=q1")
        # print(q.shape)

        q = q.reshape((num_samples, self.action_size), order='F')
        optimal_action_idxs = np.argmax(q, axis=1)

        # print("=z_concat")
        # print(z_concat.shape)
        # print("=q2")
        # print(q.shape)
        # print("=optimal_action_idxs")
        # print (optimal_action_idxs)

        # Project Next State Value Distribution (of optimal action) to Current State
        for i in range(num_samples):
            if done[i]:  # Terminal State
                # Distribution collapses to a single point
                Tz = min(self.v_max, max(self.v_min, reward[i]))
                bj = (Tz - self.v_min) / self.delta_z
                m_l, m_u = math.floor(bj), math.ceil(bj)
                m_prob[action[i]][i][int(m_l)] += (m_u - bj)
                m_prob[action[i]][i][int(m_u)] += (bj - m_l)
            else:
                for j in range(self.num_atoms):
                    Tz = min(self.v_max, max(self.v_min, reward[i] + self.gamma * self.z[j]))
                    bj = (Tz - self.v_min) / self.delta_z
                    m_l, m_u = math.floor(bj), math.ceil(bj)
                    m_prob[action[i]][i][int(m_l)] += z_[optimal_action_idxs[i]][i][j] * (m_u - bj)
                    m_prob[action[i]][i][int(m_u)] += z_[optimal_action_idxs[i]][i][j] * (bj - m_l)

        loss = self.model.fit(state_inputs, m_prob, batch_size=self.batch_size, nb_epoch=1, verbose=0)

        return loss.history['loss']

    # load the saved model
    def load_model(self, name):
        print ("loading model... ")
        self.model.load_weights(name)

    # save the model which is under training
    def save_model(self, name):
        self.model.save_weights(name)


from tkinter import *

root = Tk()
root.title("Radio buttons")
root.iconbitmap("./python.ico")

def updatePizza(pizza):
    global myLabel
    myLabel.pack_forget()
    myLabel = Label(root, text=pizza.get())
    myLabel.pack()

toppings = [
    ("Pepperoni", "Pepperoni"),
    ("Cheese", "Cheese"),
    ("Mushroom", "Mushroom"),
    ("Onion", "Onion"),
]

pizza = StringVar()
pizza.set("Pepperoni")

for (topping, value) in toppings:
    Radiobutton(root, text=topping, variable=pizza, value=topping, command= lambda: updatePizza(pizza)).pack(anchor=W)

# r = IntVar()
# r.set(2)
# Radiobutton(root, text="Option 1", variable=r, value=1, command=updateR).pack()
# Radiobutton(root, text="Option 2", variable=r, value=2, command=updateR).pack()

myLabel = Label(root, text=pizza.get())
myLabel.pack()

root.mainloop()