# coding: utf-8

# # Biologic Programs

# ## Biologic Program
# `Abstract Class`
# Represents a program to be run on a device channel.
#
# ### Methods
# **BiologicProgram( device, channel, params, autoconnect = True, barrier = None ):** Creates a new program.
#
# **channel_state( channels = None):** Returns the state of channels.
#
# **on_data( callback, index = None ):** Registers a callback function to run when data is collected.
#
# **run():** Runs the program.
#
# **stop():** Sets the stop event flag.
#
# **save_data( file, append = False ):** Saves data to the given file.
#
# **sync():** Waits for barrier, if set.
#
# **_connect():** Connects to the device.
#
# ### Properties
# **device:** BiologicDevice.

# **channel:** Device channel.

# **params:** Passed in parameters.

# **autoconnect:** Whether connection to the device should be automatic or not.

# **barrier:** A threading.Barrier to use for channel syncronization. [See ProgramRummer]

# **field_titles:** Column names for saving data.

# **data:** Data collected during the program.

# **status:** Status of the program.

# **fields:** Data fields the program returns.

# **technqiues:** List of techniques the program uses.
#
# ## Program Runner
# Represents a program to be run on a device channel.
#
# ### Methods
# **ProgramRunner( programs, sync = False ):** Creates a new program runner.
#
# **start():** Runs the programs.
#
# **wait():** Wait for all threads to finish.
#
# **stop():** Sets the stop event.
#
# ### Properties
# **threads:** List of threads for each program.
# **sync:** Whether to sync the threads or not.
#

import os
import logging
import signal
import asyncio
import threading
from abc import ABC
from collections import namedtuple

from .lib import ec_lib as ecl
from .lib import data_parser as dp

DataSegment = namedtuple( 'DataSegment', [
    # parsed data, raw info, raw data
    'data', 'info', 'values'
] )

CallBack = namedtuple( 'CallBack', [ 'function', 'args', 'kwargs' ] )

class BiologicProgram( ABC ):
    """
    Abstract Class
    Represents a Biologic Program

    Stores data.
    """

    def __init__(
        self,
        device,
        params,
        channels     = None,
        autoconnect  = True,
        field_values = None,
        barrier      = None,
        stop_event   = None,
        threaded     = False
    ):
        """
        Initialize instance parameters.

        :param device: A BiologicDevice to run the program on.
        :param params: Dictionary of parameters to be passed to the program as values,
            or Dictionary of params keyed by channel.
            If dictionary of parameters, channels param must also be passed,
            same parameters used for all channels.
            If keyed by channels, channels are assumed.
        :param channels: List of channels to run the program on, or
             None to interpret channels from params.
             [Default: None]
        :param autoconnect: Automatically connect and disconnect to device during run.
            [Default: True]
        :param barrier: threading.Barrier used for synchronization across channels.
            [Default: None]
        :param stop_event: threading.Event indicating to stop the program.
            [Default: None]
        :param threaded: Indicated if the program is running as a thread.
            [Default: False]
        :raises ValueError: If data_window is a dictionary and missing channel keys.

        Properties
        :write_attempts: Number of attempts to try to write data, afterwhich an exception is raised.
            If None, never raise an exception.
            [Default: 0]
        :field_titles: List of names for each field to be used when writing data.
            Should have same length as number of fields.
        :_fields: A named tuple representing the program data.
        :_field_values: Function returning a tuple of fields or None.
            If a function, data is appended to self.data by calling the function
            with input ( datum, segment ). Returned data should be of type self._fields.
            If None, no data retrieval or processing occurs.
            [Default: None]
        :_data_fields: Technique field types.
            [See lib.data_parser.SP300_Fields and lib.data_parser.VMP3_Fields]
        :_parameter_types: Enum of parameter types for each techinuqe field.
            [See lib.technique_fields]
        """
        self.device       = device
        self.autoconnect  = autoconnect
        self.barrier      = barrier
        self.field_titles = []  # column names for saving data
        self.write_attempts = 0
        self._writes_failed = 0

        if channels is None:
            # assume channels from params
            self._channels = list( params.keys() )
            self.params = params

        else:
            # use same params for all channels
            self.params = { ch: params.copy() for ch in channels }
            self._channels = channels

        self._techniques   = []  # program techniques
        self._fields       = None  # program fields object
        self._field_values = None  # fucntion to compute program fields
        self._data_fields  = None  # technique input fields
        self._parameter_types = None  # parameter types for the technique

        self._data = { ch: [] for ch in self.channels }  # data store
        self._unsaved_data = { ch: [] for ch in self.channels }  # data store for unwritten data
        self.data_window = None  # initialize to keep all data

        self._threaded = threaded
        self._stop_event = stop_event
        self._cb_data = []

        # TODO: signal handling
        # register interupt signal
        if self._threaded:
            signal.signal(
                signal.SIGINT,
                self.stop
            )

    #--- properties ---

    @property
    def channels( self ):
        """
        :returns: List of channels.
        """
        return self._channels

    @property
    def data( self ):
        """
        :returns: Data collected from program.
        """
        return self._data

    @property
    def data_window( self ):
        """
        :returns: Data window used to trim channel data.
        """
        return self._data_window

    @data_window.setter
    def data_window( self, value ):
        """
        Sets the data window.
        :param value: Dictionary of { channel: amount } for amount of data to keep,
            or an amount to apply to all channels, after each save.
            If amount is a non-negative integer, keeps at that many data points,
                clearing data after each write to file.
            If amount is None, keeps all data.
            [Default: 0]
        """
        if isinstance( value, dict ):
            missing = [ ch for ch in self.channels if ch not in value.keys() ]
            if len( missing ):
                # missing channels
                raise ValueError( f'Channel(s) {missing} missing.' )

            self._data_window = value

        else:
            # static value
            self._data_window = { ch: value for ch in self.channels }

    @property
    def fields( self ):
        """
        :returns: Fields object.
        """
        return self._fields

    @property
    def field_values( self ):
        """
        :returns: Field values function.
        """
        return self._field_values

    @property
    def techniques( self ):
        """
        :returns: Technqiue(s) of the program
        """
        return self._techniques

    @property
    def writes_failed(self):
        """
        :returns: Number of failed writes since last success.
        """
        return self._writes_failed

    #--- public methods ---

    def channel_state( self, channels = None ):
        """
        Returns the state of the channels.

        :param channels: Channel or list of channels,
            or None to retrieve all statuses.
            [Default: None]
        :returns: ChannelState, or dictionary of ChannelStates keyed by channel.
        """
        single_ch = False

        if channels is None:
            channels = self.channels

        elif not isinstance( channels, list ):
            # single channel provided, put in list
            single_ch = True
            channels = [ channels ]

        states = {}
        for ch in channels:
            info = self.device.channel_info( channel )
            states[ ch ] = ecl.ChannelState( info.State )

        if single_ch:
            # single channel provided
            return states[ channel ]

        return states

    def on_data( self, cb, index = None ):
        """
        Register a callback when data is collected.

        :param cb: A callback function to be called.
            The function should have signature (segment, program ).
            segment is a DataSegment, a namedtuple with properties
            [ data, info, values ], as returned by BiologicDevice.get_data().
            program is the program being run.
        :param index: Index at which to run the callback or None to append.
            If index exceeds current length of callback list, then function is appended.
            [Default: None]
        """
        if index is None:
            index = len( self._cb_data )

        self._cb_data.insert( index, cb )

    def run( self, auto_retrieve = True  ):
        """
        Runs the program.

        :param auto_retrieve: Automatically retrieve data. [Default: True]
        """
        pass

    def save_data(
        self,
        file,
        append = False,
        by_channel = False
    ):
        """
        Saves data to a CSV file.

        :param file: File or folder path.
        :param append: True to append to file, false to overwrite.
            [Default: False]
        :param by_channel: Save each channel to own file (True)
            or in same file (False).
            If True, file should be a folder path.
            If False, file should be a file path.
            [Default: False]
        """
        if append is False:
            if not hasattr( self, '_write_header' ):
                self._write_header = True

            else:
                # header already written
                # append to file
                append = True

        else:
            # don't write header if appending
            self._write_header = False

        try:
            if by_channel:
                self._save_data_individual(
                    file,
                    append = append,
                    write_header = self._write_header
                )

            else:
                self._save_data_together(
                    file,
                    append = append,
                    write_header = self._write_header
                )

        except Exception as err:
            self._writes_failed += 1

            if (
                ( self.write_attempts is not None ) and
                ( self.writes_failed > self.write_attempts )
            ):
                raise err

        else:
            # only write header at most once
            self._write_header = False

            # successful write
            # reset failed attempts
            self._writes_failed = 0

        # drop data outside data window
        self.trim_data()

    def trim_data( self, window = None ):
        """
        Trims data to a specific length.

        :param window: Dictionary of { channel: amount } for amount of data to save
            of None to use self.data_window.
            If amount is a non-negative integer, saves at most that many data points,
                clearing data after each write to file.
            If amount is None does not trim any data.
            [Default: None]
        :raises ValueError: If an invalid channel is provided as a window key.
        """
        if window is None:
            window = self.data_window

        # validate channels
        invalid_channels = [ ch for ch in window.keys() if ch not in self.channels ]
        if len( invalid_channels ) > 0:
            raise ValueError( f'Invalid channel(s): {invalid_channels}' )

        # trim data
        for ch, ch_window in window.items():
            if ch_window is None:
                # don't trim data
                continue

            elif ch_window == 0:
                # clear data
                self._data[ ch ] = []

            else:
                self._data[ ch ] = self._data[ ch ][ -ch_window: ]

    def sync( self ):
        """
        Waits for barrier, if set.
        """
        if self.barrier is not None:
            self.barrier.wait()

    def stop( self, signal, frame ):
        """
        Sets stop event.
        """
        if self._stop_event is None:
            logging.warning( f'No stop event is present on channels {self.channels}.' )
            return

        self._stop_event.set()

    #--- protected methods ---

    def _connect( self ):
        """
        Connects device if needed
        """
        if not self.device.is_connected():
            self.device.connect()

    def _disconnect( self ):
        """
        Disconnects device
        """
        if self.device.is_connected():
            self.device.disconnect()

    def _run( self, technique, params, read_interval = 1, retrieve_data = True ):
        """
        Runs the program.

        :param technqiue: Name of technique.
        :param params: Technique parameters.
        :param read_interval: Time between data fetches. [Default: 1]
        :param retrieve_data: Whether data should be retrieved or not.
            self.field_values must be valid.
            [Default: True]
        """
        # run technique
        if self.autoconnect:
            self._connect()

        for ch, ch_params in params.items():
            self.device.load_technique(
                ch,
                technique,
                ch_params,
                types = self._parameter_types
            )

        self.device.start_channels( self.channels )

        if retrieve_data:
            asyncio.run( self._retrieve_data( read_interval ) )

            if self.autoconnect is True:
                self._disconnect()

    async def _retrieve_data_segment( self, channel ):
        """
        @async
        Gets the current data segment, and parses the data.

        :param channel: Channel to retrieve data from.
        :returns: DataSegment.
        """
        raw = await self.device.get_data( channel )

        try:
            parsed = dp.parse(
                raw.data,
                raw.info,
                self._data_fields,
                self.device
            )

        except RuntimeError as err:
            msg = f'ch {channel}: {err}'
            logging.debug( msg )

            return DataSegment( [], raw.info, raw.values )

        segment = DataSegment( parsed, raw.info, raw.values )

        if self._fields and self._field_values:
            parsed = [
                self._fields( *self._field_values( datum, segment ) )
                for datum in segment.data
            ]
            self._data[ channel ] += parsed
            self._unsaved_data[ channel ] += parsed

        # run callbacks
        for cb in self._cb_data:
            cb( segment, self )

        return segment

    async def _retrieve_data_segments( self, channels = None ):
        """
        @async
        Gets the current data segment for active channels, and parses the data.

        :param channels: List of channels ro get data from, or None for all.
            [Default: None]
        :returns: Dictionary of DataSegments keyed by channel.
        """
        if channels is None:
            channels = self.channels

        segments = {}
        for ch in channels:
            segments[ ch ] = await self._retrieve_data_segment( ch )

        return segments

    async def _retrieve_data( self, interval = 1 ):
        """
        @async
        Retrieves data from the device until it is stopped.
        Data is parsed.

        :param interval: How often to collect data in seconds.
            [Default: 1]
        :returns: Dictionary of lists of DataSegments with properties
            [ data, info, values ], keyed by channel.
        """
        complete = { ch: False for ch in self.channels }
        while not all( complete.values() ):
            if (  # stop signal received
                self._stop_event is not None
                and self._stop_event.is_set()
            ):
                logging.warning(
                    f'Halting program on channel {self.channel}.'
                )

                break

            await asyncio.sleep( interval )  # wait

            # retrieve data
            active_channels = [ ch for ch, done in complete.items() if ( not done ) ]
            segments = await self._retrieve_data_segments( active_channels )

            for ch, ch_segment in segments.items():
                done = ( ecl.ChannelState( ch_segment.values.State  ) is ecl.ChannelState.STOP )
                complete[ ch ] = done
                if done:
                    logging.debug( f'Channel {ch} complete.' )

    def _save_data_together(
        self,
        file,
        append = False,
        write_header = False
    ):
        """
        Saves data to a CSV file.

        :param file: File path.
        :param append: True to append to file, false to overwrite.
            [Default: False]
        :param write_header: Whether to write header or not.
            [Default: False]
        """
        mode = 'a' if append else 'w'
        try:
            with open( file, mode ) as f:
                num_titles = len( self.field_titles )

                if write_header:
                    # write header only if not appending
                    # write channel header if multichanneled
                    ch_header = ''
                    for ch in self.channels:
                        ch_header += f'{ch},'* num_titles

                    # replace last comma with line end
                    ch_header = ch_header[ :-1 ] + '\n'

                    f.write( ch_header )

                    # field titles
                    title_header = ''
                    for _ in self.channels:
                        # create titles for each channnel
                        title_header += ','.join( self.field_titles ) + ','

                    # replace last comma with new line
                    title_header = title_header[ :-1 ] + '\n'

                    try:
                        f.write( title_header )

                    except Exception as err:
                        logging.warning( f'Error writing header: {err}' )

                # write data
                # get maximum rows
                num_rows = { ch: len( data ) for ch, data in self._unsaved_data.items() }
                written = { ch: [] for ch in self._unsaved_data.keys() }
                for index in range( max( num_rows.values() ) ):
                    written_row = { ch: None for ch in self._unsaved_data.keys() }
                    row_data = ''
                    for ch, ch_data in self._unsaved_data.items():
                        if index < num_rows[ ch ]:
                            # valid row for channel
                            ch_datum = ch_data[ index ]
                            row_data += ','.join( map( self._datum_to_str, ch_datum ) ) + ','
                            written_row[ ch ] = ch_datum

                        else:
                            # channel data exhausted, write placeholders
                            row_data += ','* num_titles

                    # new row
                    row_data = row_data[ :-1 ] + '\n'

                    try:
                        f.write( row_data )

                    except Exception as err:
                        logging.warning( f'Error writing data: {err}' )

                    else:
                        # successful write
                        for ch, ch_datum in written_row.items():
                            written[ ch ].append( ch_datum )

                # data written, remove data from unsaved
                for ch, ch_data in written.items():
                    self._unsaved_data[ ch ] = [
                        datum for datum in ch_data
                        if datum not in written[ ch ]
                    ]

        except Exception as err:
            if self._threaded:
                logging.warning( f'[#save_data] CH{ch}: {err}' )

            else:
                raise err

    def _save_data_individual(
        self,
        folder,
        append = False,
        write_header = False
    ):
        """
        Saves data to a CSV file.

        :param folder: Folder path.
        :param append: True to append to file, false to overwrite.
            [Default: False]
        :param write_header: Whether to write header or not.
            [Default: False]
        """
        mode = 'a' if append else 'w'

        if not os.path.exists( folder ):
            os.makedirs( folder )

        for ch, ch_data in self._unsaved_data.items():
            file = os.path.join( folder, f'ch-{ch}.csv' )

            csv_data = ''
            for datum in ch_data:
                csv_data += ','.join( map( self._datum_to_str, datum ) )
                csv_data += '\n'

            try:
                with open( file, mode ) as f:
                    if write_header:
                        # write header only if not appending
                        f.write( ','.join( self.field_titles ) + '\n' )

                    # write data
                    f.write( csv_data )

            except Exception as err:
                logging.warning( f'[#_save_data_individual] CH{ch}: {err}' )

            else:
                # data written successfully
                self._unsaved_data[ ch ] = []

    def _datum_to_str( self, datum ):
        """
        Casts data to string.
        If datum is None, casts to empty string intead of 'None'.

        :param datum: Datum to cast.
        :returns: Datum as a string.
        """
        if datum is None:
            return ''

        # datum is not None
        return str( datum )

class ProgramRunner():
    """
    Runs programs on multiple channels simultaneously,
    each in its own thread.
    """

    def __init__( self, programs, sync = False, timeout = 5 ):
        """
        Create a Program Runner.

        :param programs: A list BiologicPrograms to run.
            If run parameters are required use a dictionary
            with keys [program, params] where the program value
            is the program instance to run, and params is a dictionary
            of parameters to pass to #run of the program.
        :param sync: Whether threads should be synced or not.
            Relies on programs using threading.Barrier.
            [Default: False]
        :param timeout: Threading timeout in seconds. Used for signal interuptions.
            [Default: 5]
        """
        self.programs = programs
        self.sync = sync
        self.timeout = timeout

        self.__threads = []
        self._stop_event = threading.Event()

        if self.sync:
            self.barrier = threading.Barrier( len( programs ) )
            for program in programs:
                prg = (
                    program[ 'program' ]
                    if isinstance( program, dict )
                    else program
                )

                prg.barrier = self.barrier

        # register interupt signal
        signal.signal(
            signal.SIGINT,
            self.stop
        )

    @property
    def threads( self ):
        """
        :returns: Current threads.
        """
        return self.__threads

    def start( self ):
        """
        Start the programs
        """
        self.__threads = []
        for prg in self.programs:
            if isinstance( prg, dict ):
                # run params passed in
                program = prg[ 'program' ]
                params  = prg[ 'params' ]

            else:
                # only program passed, no run params
                program = prg
                params = {}

            program._stop_event = self._stop_event

            t = threading.Thread(
                target = program.run,
                kwargs = params
            )

            self.__threads.append( t )
            t.start()

    def wait( self ):
        """
        Wait for all threads to finish.
        """
        for thread in self.threads:
            thread.join()

        # TODO: Poll is alive with join timeout to allow signals
#         is_alive = [ False ]* len( self.threads )
#         while not all( is_alive ):
#             for index, thread in enumerate( self.threads ):
#                 is_alive[ index ] = thread.is_alive()
#                 thread.join( self.timeout )

    def stop( self, signal, frame ):
        """
        Sets stop event.
        """
        logging.warning( "Halting programs..." )
        self._stop_event.set()


from PyQt5.QtWidgets import QDialog, QLabel, QDialogButtonBox, QLineEdit, \
    QVBoxLayout, QListWidget, QMessageBox
from PyQt5.QtGui import QKeyEvent
from PyQt5.QtCore import Qt
from settings import NO_GRAPHS_WITH_NAME, NOT_SELECTED, ARE_YOU_SURE

class ChooseGraphForm(QDialog):
    """Класс формы для выбора графа"""
    def __init__(self, names: list, to_delete: bool, parent=None) -> None:
        super().__init__(parent)
        self.names = names
        self.name_to_return = None
        self.to_delete = to_delete
        self.setWindowTitle("Choose graph")
        self.buttonBox = QDialogButtonBox(
            QDialogButtonBox.Ok | QDialogButtonBox.Cancel
        )
        self.buttonBox.accepted.connect(self.accept)
        self.buttonBox.rejected.connect(self.reject)
        self.inputData = QLineEdit()
        self.inputData.textChanged.connect(self.findNames)
        self.list = QListWidget()
        self.list.addItems(names)
        self.list.itemClicked.connect(self.saveName)
        self.list.itemDoubleClicked.connect(self.accept)
        self.layout = QVBoxLayout()
        self.layout.addWidget(QLabel("Choose the graph:"))
        self.layout.addWidget(self.inputData)
        self.layout.addWidget(self.list)
        self.layout.addWidget(self.buttonBox)
        self.setLayout(self.layout)

    def saveName(self) -> None:
        """Метод установки имени выбранного графа в строку поиска"""
        if self.list.currentItem() is not None:
            self.name_to_return = self.list.currentItem().text()

    def findNames(self) -> None:
        """Метод поиска графов по имени и вывода их в списке"""
        self.list.setEnabled(True)
        self.list.clear()
        text = self.inputData.text()
        found_names = [name for name in self.names if text in name]
        if found_names:
            self.list.addItems(found_names)
        else:
            self.list.setEnabled(False)
            self.list.addItems([NO_GRAPHS_WITH_NAME])

    def keyPressEvent(self, event: QKeyEvent) -> None:
        """Обработчик нажатия на клавиатуру
        (переключение между элементами списка"""
        if self.inputData.hasFocus():
            return
        if event.key() in (Qt.Key_Up, Qt.Key_W):
            self.list.setCurrentRow(
                (self.list.currentRow() - 1) % self.list.count()
            )
        elif event.key() in (Qt.Key_Down, Qt.Key_S):
            self.list.setCurrentRow(
                (self.list.currentRow() + 1) % self.list.count()
            )
        self.saveName()

    def accept(self) -> None:
        """Метод обработки события нажатия на кнопку ОК"""
        flag = QMessageBox.Yes
        if self.name_to_return not in self.names and self.name_to_return:
            QMessageBox.critical(self, "Error", NO_GRAPHS_WITH_NAME)
            return
        elif self.name_to_return is None or not self.name_to_return:
            QMessageBox.warning(self, "Warning", NOT_SELECTED)
            return
        if self.to_delete:
            flag = QMessageBox.question(
                self, "Delete", f"{ARE_YOU_SURE} graph {self.name_to_return}"
            )
        if flag == QMessageBox.Yes:
            self.done(1)


def powerset(l):
    if l == []:
        return [[]]
    x = powerset(l[1:])
    return x + [[l[0]] + y for y in x]

def budgeting2(budget, products, wishlist):
    lista = []
    buy = []
    save = budget
    bought = {}
    for product in wishlist:
        lista += [product,] * wishlist[product]
    power = list(powerset(lista))
    for i in power:
        if i not in buy:
            buy.append(i)
    for i in buy:
        price = 0
        budget_control = budget
        for item in i:
            price += products[item]
        budget_control -= price
        if (budget_control < save) and (budget_control) >= 0:
            save = budget_control
            tobuy_list = i
    for i in tobuy_list:
        if i not in bought:
            bought[i] = 1
        elif i in bought:
            bought[i] += 1
    return bought


import os

import torch
from bistiming import Stopwatch
import numpy as np
from sklearn.preprocessing import OneHotEncoder, StandardScaler
import joblib

from .utils import set_random_seed
from lolip.utils import estimate_local_lip_v2
from lolip.variables import get_file_name
from .experiment02 import load_model

def load_result(auto_var, result_dir="./results/experiment03"):
    model_path = get_file_name(auto_var) + ".pkl"
    model_path = os.path.join(result_dir, model_path)
    res = joblib.load(model_path)
    ret = {
        'trn_acc': res['trn_acc'],
        'tst_acc': res['tst_acc'],
        'avg_trn_lip_1': res['avg_trn_lip_1'],
        'avg_tst_lip_1': res['avg_tst_lip_1'],
    }
    del res
    return ret

def run_mtattack(auto_var):
    _ = set_random_seed(auto_var)
    trnX, trny, tstX, tsty = auto_var.get_var("dataset")
    lbl_enc = OneHotEncoder(categories=[np.sort(np.unique(trny))], sparse=False).fit(trny.reshape(-1, 1))
    auto_var.set_intermidiate_variable("lbl_enc", lbl_enc)
    n_classes = len(np.unique(trny))
    n_channels = trnX.shape[-1]

    result = load_result(auto_var)
    print(f"loaded results: {result}")
    result['model_path'], model = load_model(auto_var, trnX, trny, tstX, tsty, n_channels)

    print(f"train acc: {result['trn_acc']}")
    print(f"test acc: {result['tst_acc']}")

    attack_model = auto_var.get_var("attack", model=model,
            n_classes=n_classes, clip_min=0, clip_max=1)
    with Stopwatch("Attacking"):
        adv_trnX = attack_model.perturb(trnX, trny)
        adv_tstX = attack_model.perturb(tstX, tsty)
    result['adv_trn_acc'] = (model.predict(adv_trnX) == trny).mean()
    result['adv_tst_acc'] = (model.predict(adv_tstX) == tsty).mean()
    print(f"adv trn acc: {result['adv_trn_acc']}")
    print(f"adv tst acc: {result['adv_tst_acc']}")

    #with Stopwatch("Estimating trn Lip"):
    #    trn_lip = estimate_local_lip_v2(model.model, trnX, top_norm=2, btm_norm=norm,
    #                                 epsilon=auto_var.get_var("eps"))
    #result['avg_trn_lip'] = calc_lip(model, trnX, trn_lip, top_norm=2, btm_norm=norm).mean()
    #with Stopwatch("Estimating tst Lip"):
    #    tst_lip = estimate_local_lip_v2(model.model, tstX, top_norm=2, btm_norm=norm,
    #                                 epsilon=auto_var.get_var("eps"))
    #result['avg_tst_lip'] = calc_lip(model, tstX, tst_lip, top_norm=2, btm_norm=norm).mean()
    #print(f"avg trn lip: {result['avg_trn_lip']}")
    #print(f"avg tst lip: {result['avg_tst_lip']}")

    print(result)
    return result


# 直接套用二分法就可以
class Solution:
    def isPerfectSquare(self, num: int) -> bool:
        start = 1
        end = num
        while start + 1 < end:
            mid = (start + end) // 2
            if mid ** 2 == num:
                return True
            elif mid ** 2 < num:
                start = mid
            else:
                end = mid

        if start ** 2 == num or end ** 2 == num:
            return True
        return False


import json
import os
import re
import signal
import unicodedata
import urllib
from contextlib import contextmanager
import subprocess
from datetime import datetime, timedelta
from time import mktime, strptime
import time
import html2text as html2text
from bs4 import BeautifulSoup, UnicodeDammit
from dateutil import parser
import PyPDF2 as PyPDF2
import magic
from googlesearch import search
import logging, logging.handlers
import src.params as params
from src.evaluate_similarity import compute_best_doc

class Log(object):
    def __init__(self, dossier, nomFichier, niveau=logging.DEBUG):
        super(Log, self).__init__()

        self.__logger__ = logging.getLogger(nomFichier)
        self.__logger__.setLevel(niveau)

        format = logging.Formatter('%(asctime)s :: %(levelname)s :: %(message)s', datefmt='%d-%m-%Y %H:%M:%S')
        fichierLog = logging.handlers.RotatingFileHandler("{0}{1}.log".format(dossier, nomFichier), 'a', 1000000, 1)

        fichierLog.setLevel(niveau)
        fichierLog.setFormatter(format)
        self.__logger__.addHandler(fichierLog)

        console = logging.StreamHandler()
        console.setLevel(niveau)
        self.__logger__.addHandler(console)

    def info(self, message):
        self.__logger__.info(message)

    def debug(self, message):
        self.__logger__.debug(message)

    def warning(self, message):
        self.__logger__.warning(message)

    def error(self, message):
        self.__logger__.error(message)

    def critical(self, message):
        self.__logger__.critical(message)

    def exception(self, message):
        self.__logger__.exception(message)

    def close(self):
        for handler in  self.__logger__.handlers[:] :
            handler.close()
            self.__logger__.removeHandler(handler)

class TextCleaner():
    """docstring for TextCleaner."""
    def __init__(self):
        """ Expressions régulières """
        self.match_alpha = re.compile(r'\w{3,}') # Existence d'un mot d'au moins 3 lettres
        self.match_formfeed = re.compile(r'\f') # Saut de page
        self.match_useless_line = re.compile(r'^[^\w•]+$|^[\d\. ]+$', re.MULTILINE) # Ligne ne contenant aucun caractère alphanumérique

        self.match_bad_nl = re.compile(r'([^\.?!\n:])\n+(?![IVX\d]+[\.\)]|ANNEXE)([\w\(«\"=<>])') # Mauvais saut de ligne
        self.match_bad_nl2 = re.compile(r'(\.{4,} ?\d+) (\w)') # Mauvais saut de ligne
        self.make_paragraph = re.compile(r'\.\n(?=\w)') # On sépare mieux les paragraphes
        self.match_toomuch_nl = re.compile(r'\n{3,}') # Nouvelles lignes surnuméraires
        self.match_begin_end_nl = re.compile(r'^\n+|\n+$') # Nouvelles lignes au début et à la fin

        self.match_begin_end_space = re.compile(r'^[ \t]+|[ \t]+$', re.MULTILINE) # Espace ou tabulation en début et fin de ligne
        self.match_toomuch_spaces = re.compile(r' {2,}|\t+') # Espaces surnuméraires et tabulations

        self.match_link = re.compile(r'!?\[.*?\]\(.*?\)|https?://[^ ]+', re.DOTALL) # Lien issu de la conversion depuis HTML
        self.match_cesure = re.compile(r'(\w)-(\w)') # Césure
        self.match_stuckwords = re.compile(r'(/\d{2,4}|[a-z])([A-ZÉÈÀÔ])') # Dates et mots collés

        self.match_odd = re.compile(r'[�●§\\\|]+|\(cid:\d+\)|(?<=- )W ') # caractère bizarre
        self.match_accent1 = re.compile(r'Ã©') # é
        self.match_accent2 = re.compile(r'Ã¨') # è
        self.match_accent3 = re.compile(r'Ã') # à
        self.match_puce1 = re.compile(r'')
        self.match_puce2 = re.compile(r'[]')
        self.match_diam = re.compile(r'diam\.')
        self.match_apostrophe = re.compile(r'’')

    def clean(self, text):
        # Remplacement des espaces insécables
        text = text.replace(u'\xa0', ' ')

        text = self.match_link.sub('', text)
        text = self.match_formfeed.sub('\n\n', text)
        text = self.match_useless_line.sub('\n', text)

        text = self.match_diam.sub('diamètre', text)
        text = self.match_accent1.sub('é', text)
        text = self.match_accent2.sub('è', text)
        text = self.match_accent3.sub('à', text)
        text = self.match_puce1.sub('*', text)
        text = self.match_puce2.sub('-', text)
        text = self.match_odd.sub('', text)
        text = self.match_apostrophe.sub('\'', text)

        text = self.match_begin_end_space.sub('', text)
        text = self.match_bad_nl.sub(r'\g<1> \g<2>', text)
        # On double la réparation des lignes, meilleurs résultats
        text = self.match_bad_nl.sub(r'\g<1> \g<2>', text)
        text = self.match_bad_nl2.sub(r'\g<1>\n\g<2>', text)
        text = self.make_paragraph.sub('.\n\n', text)
        text = self.match_stuckwords.sub(r'\g<1> \g<2>', text)
        text = self.match_toomuch_spaces.sub(' ', text)
        text = self.match_toomuch_nl.sub('\n\n', text)
        text = self.match_begin_end_nl.sub('', text)

        return text

    def exists_alpha(self, text):
        """ Contrôle l'existence de caractères alphanumériques """
        return self.match_alpha.search(text) is not None

def query_generator(city, motscles, logger, site):
    logger.info("Génération des requêtes")

    requetes_effectuees = []

    with open("{0}{1}/.sauvegarde.txt".format(params.CHEMIN_RESULTATS, city)) as file:
        to_be_queried = file.readlines()

    for i in range(0, len(motscles), 1):
        # motscles_couple = motscles[i].split("+")
        # Si aucun site n'est spécifié alors on n'exclut
        # seulement ceux définis dans les constants
        if site == "":
            site_or_not_sites = params.NOT_SITES
        # Sinon on cherche uniquement sur le site spécifié
        else:
            site_or_not_sites = "site:" + site
        query = "\"{0}\" AND {1}".format(city, motscles[i])
        if not any(query in s for s in to_be_queried):
            yield query

def format_word(mot):
    """
        Supprime les retour charriot et remplace les espace par un ?.

        @type  mot: String.
        @param mot: Mot à formaté.

        @rtype: String.
        @return: Mot formaté.
    """
    liste_caracteres = dict()

    liste_caracteres["\n"] = ""
    liste_caracteres[" "] = "?"

    for i in liste_caracteres :
        mot = mot.replace(i, liste_caracteres[i])

    return mot

class TimeoutException(Exception): pass

@contextmanager
def sleep_time(seconds):
    def signal_handler(signum, frame):
        raise TimeoutException
    signal.signal(signal.SIGALRM, signal_handler)
    signal.alarm(seconds)
    try:
        yield
    finally:
        signal.alarm(0)

def convert_pdf_to_txt(src_file_path):
    """
        Appel externe à pdftotext.

        -q : pas de message d'erreur dans la sortie.
         - : envoie la sortie dans la console au lieu d'un fichier texte.

        Capture de la sortie texte.

        @type  src_file_path: String.
        @param src_file_path: Chemin du fichier source.

        @rtype: String.
        @return: Texte brut.
    """
    completed_process = subprocess.run(["pdftotext", "-q", src_file_path, "-"], stdout=subprocess.PIPE)
    return completed_process.stdout.decode('utf-8')

def norm_string(string):
    """ On supprime les accents et on remplace les caractères spéciaux par un tiret bas """
    new_string = unicodedata.normalize('NFKD', string).encode('ascii', 'ignore').decode('utf-8')
    return re.sub(r'[^\w\[\]\./]+', '_', new_string)

def format_to_iso_date(pdfdate):
    datestring = ''
    if pdfdate !='':
        if len(pdfdate)==23:
            datestring = pdfdate[2:-7]
        elif len(pdfdate)==17:
            datestring = pdfdate[2:-1]
        elif len(pdfdate)==21:
            datestring = pdfdate[:-7]
        ts = strptime(datestring, "%Y%m%d%H%M%S")# "%Y-%m-%dT%H:%M:%S.%fZ" )#
        dt = datetime.fromtimestamp(mktime(ts))
        new_format = dt.isoformat()
    else:
        new_format = "no_date"
    return new_format

def get_pdf_info(f, logger):
    fd = PyPDF2.PdfReader(f, 'rb')
    doc_info = fd.metadata
    #print("doc_info :", doc_info)
    if '/ModDate' in doc_info:
        val_ = doc_info['/ModDate']
        try:
            val_ = format_to_iso_date(val_)
        except:
            logger.exception("cannot format the extracted date.\n")
    elif '/CreationDate' in doc_info:
        val_ = doc_info['/CreationDate']
        try:
            val_ = format_to_iso_date(val_)
        except:
            logger.exception("cannot format the extracted date.\n")
    else:
        val_ = None # 'no_date'

    if '/Title' in doc_info:
        title = doc_info['/Title']
    else:
        title = None # 'no_title'
    #print("title :", str(title))
    #print("date :", str(parser.parse(val_).date()))
    return str(title), str(parser.parse(val_).date())

def extract_date(text):
    # Motif : JJ/MM/AAAA
    match_date = re.search(r'(\d{2})/(\d{2})/(\d{4})', text)
    # Motif : JJ mois AAAA
    match_date2 = re.search(r'(\d{1,2}) (\w+) (\d{4})', text)

    if match_date:
        date = datetime(
            int(match_date[3]), # Année
            int(match_date[2]), # Mois
            int(match_date[1])  # Jour
        )

        return date

    elif match_date2:
        mapping_month = {
            'janvier': 1,
            'février': 2,
            'mars': 3,
            'avril': 4,
            'mai': 5,
            'juin': 6,
            'juillet': 7,
            'août': 8,
            'septembre': 9,
            'octobre': 10,
            'novembre': 11,
            'décembre': 12,
        }

        month = match_date2[2].lower()

        if month in mapping_month:
            date = datetime(
                int(match_date2[3]), # Année
                mapping_month[month], # Mois
                int(match_date2[1])  # Jour
            )

            return date

def less_html(html_doc):
    """ Prend du code HTML en entrée et retourne un code épuré de certaines balises """
    soup = BeautifulSoup(html_doc, 'html.parser')
    # Regex pour matcher les attributs contenant ces termes
    bad_classes = re.compile(r'menu|head|publici|share|social|button|alert|prev|next|foot|tags|label|sidebar|author|topics|contact|modal|nav|snippet|register|aside|logo|bandeau|immobilier', re.IGNORECASE)
    # Suppression des espaces ou des sauts de ligne au début et à la fin du titre
    title = re.sub(r'^\s|\s$', '', soup.find('title').text)

    # Dictionnaire des métadonnées
    metadata = {}
    metadata['title'] = title
    # Recherche d'une éventuelle balise dont la classe contient "date"
    # En principe la première date est la date de publication
    bloc_date = soup.find(class_=re.compile(r'date', re.IGNORECASE))

    if bloc_date:
        # Recherche du premier motif JJ/MM/AAAA,
        # en principe la date de publication
        metadata['post_date'] = ''
        date = extract_date(bloc_date.text)
        print(">>>> html_date :", str(date))
        if date: metadata['post_date'] = str(parser.parse(str(date)).date())# str(date)# html_date_to_isoformat(repr(date))
#         if date: metadata['post_date'] = html_date_to_isoformat(repr(date))

    for balise in soup.find_all():
        conditions = (
            balise.name == 'head',
            balise.name == 'nav',
            balise.name == 'footer',
            balise.name == 'aside',
            balise.name == 'script',
            balise.name == 'style',
            balise.name == 'a',
            balise.name == 'figure',
            balise.name == 'img',
            balise.name == 'svg',
            balise.name == 'noscript',
            balise.name == 'form',
            balise.name == 'button'
        )

        if any(conditions):
            balise.extract()
        # On ajoute un espace devant chaque span, pour éviter
        # parfois d'avoir des mots collés
        elif balise.name == 'span' and balise.string:
            balise.string = ' ' + balise.string

    for balise in soup.find_all(attrs={'class': bad_classes}):
        balise.decompose()

    for balise in soup.find_all(attrs={'id': bad_classes}):
        balise.decompose()

    for balise in soup.find_all():
        if balise.text == '': balise.extract()

    return metadata, str(soup)

def convert_html_to_txt(src_file_path):
    """
        Conversion à l'aide de html2text.
        Détection automatique de l'encodage (UnicodeDammit).
        On capture la sortie texte.
        @type  src_file_path: String.
        @param src_file_path: Chemin du fichier source.
        @rtype: String.
        @return: Texte brut.
    """
    html_file = open(src_file_path, 'rb').read()
    dammit = UnicodeDammit(html_file) # src_file_path
#     metadata, html_mini = less_html(html_file.decode(dammit.original_encoding))
    metadata, html_mini = less_html(html_file.decode(dammit.original_encoding))
    handler = html2text.HTML2Text()
    handler.ignore_links = True
    handler.ignore_emphasis = True
    text = handler.handle(html_mini)

    return metadata, text

# extract spatial named entities
def SNE_Extract(title_mtd):
    SNE = {}
    dc = params.nlp_model(title_mtd)
    i = 0
    for ent in dc.ents:
        if ent.label_ in ['LOC']:
            SNE['ent'+repr(i)] = repr(ent)
            i+=1
    return SNE

# ajout de meta données suplementaires pour l'intdexation spatiale et temporelle
def enrich_mtd(mtd):
    if 'title' in mtd:
        title_mtd = mtd['title']
        SNE = SNE_Extract(title_mtd)
        mtd['SNE'] = SNE #liste d'ENS

    mtd['TNE'] = {}
    #if 'post_date' in mtd and '$date' in mtd['post_date']:
    #    mtd['TNE']['date'] = mtd['post_date']['$date']
    if "post_date" in mtd:
        mtd['TNE']['date'] = []
        mtd['TNE']['date'] = mtd['post_date']
    #mtd = TNE_extract(mtd)
    return mtd

def sauvegarde_fichier_advanced(ville, url, logger, tc):
    """
		Enregistre un fichier quelqu'il soit.

		@type  ville: String.
		@param ville: Nom de la ville.

		@type  url: String.
		@param url: URL vers le fichier pdf.

		@type  log: Logger.
		@param log: Fichier de log.

		@type  tc: TextCleaner.
		@param tc: Nettoyeur de texte.
	"""
    # On prend l'URL de base pour désigner l'origine du fichier à laquelle on enlève le "www."
    origine = re.sub(r'^www\.', '', urllib.request.urlparse(url).netloc)

    url_split = url.split('/')

    # Extraction du nom du document
    if url_split[-1] == '':
        my_doc = norm_string(url_split[-2])
    else:
        my_doc = norm_string(url_split[-1])

    src_file_path = "{0}{1}/Documents_SRC/[{2}]{3}".format(params.CHEMIN_RESULTATS, ville, origine, my_doc)

#     print(">>>>>>>", url)
#     print("#####", src_file_path)

    if not os.path.isfile(src_file_path):
        req = urllib.request.Request(url, headers={'User-Agent': "Magic Browser"})
        response = urllib.request.urlopen(req)
        read_buffer = response.read()
        mime_type = magic.from_buffer(read_buffer, mime=True)

        source = {
            'file_link': src_file_path,
            'complete_url': url,
            'base_url': origine,
            'open_access': False,
        }

        document = {
            'name': my_doc,
            'mime_type': mime_type,
            'source': source,
            'manually_validated': False,
        }

        # Écriture du fichier téléchargé
        with open(src_file_path, "wb") as fichier:
            fichier.write(read_buffer)
            logger.info("Document sauvegardé.")

        # On détecte le type du document source pour utiliser la conversion adaptée
        # Si le document est un PDF
        if mime_type == 'application/pdf':
            try:
                # Conversion en texte brut
                raw_text = convert_pdf_to_txt(src_file_path)
                logger.info("Document PDF converti en texte brut.")
                title, p_d = get_pdf_info(src_file_path,logger)
                document['text'] = tc.clean(raw_text)
                document['title'] = title
                document['post_date'] = p_d
                logger.info("Texte nettoyé.\n")
                #print("title, p_d :", title, p_d)
            except:
                logger.exception("Erreur lors de la conversion en texte du pdf.\n")

        # Si le document est un HTML (ou autre fichier web)
        elif mime_type == 'text/html':
            try:
                # Conversion en texte brut
                metadata, raw_text = convert_html_to_txt(src_file_path)
                logger.info("Document web converti en texte brut.")
                document['text'] = tc.clean(raw_text)
                document.update(metadata)
                logger.info("Texte nettoyé.\n")

            except:
                logger.exception("Erreur lors de la conversion en texte de la page web.\n")

        else:
            logger.info("Autre type de document téléchargé.\n")

        document = enrich_mtd(document)

        return document

    else:
        logger.info("Le document existe déjà.\n")
        return None

### save database
def save_to_jsonl(mtd,spatial_extent):
#     with open('3m_db.jsonl', 'a+', encoding='utf8') as outfile:
    with open("{0}{1}/urbanisation_herelle.jsonl".format(params.CHEMIN_RESULTATS, spatial_extent), 'a+', encoding='utf8') as outfile:
        #for entry in JSON_file:
        json.dump(mtd, outfile, ensure_ascii=False)
        outfile.write('\n')

def pause(logger, minutes=0.5):
    """
		Effectue une pause.
		@type  log: Logger.
		@param log: Fichier de log.
		@type  minutes: Entier.
		@param minutes: Temps en minute à attendre.
	"""
    date = datetime.now().strftime('%d-%m-%Y:%Hh%M')
    current_time = datetime.now() + timedelta(minutes=minutes)

    logger.info("{0} : Nombre limite de requete atteint. Reprise du programme : {1}".format(date, current_time.strftime(
        '%d-%m-%Y:%Hh%M')))

    while datetime.now() < current_time:
        time.sleep(0.5)

### save database
def db_to_jsonl(mtd, thematic):
    head, tail = os.path.split(thematic)
    #print("tail :", tail)
    #     head,tail.split('.')[0]
    thematic = tail.split('.')[0]# keyword_file
    with open("./documents/" + str(thematic)+"_db.jsonl", 'a+', encoding='utf8') as outfile:
    #with open("{0}{1}/"+thematic+".jsonl".format(params.CHEMIN_RESULTATS, spatial_extent), 'a', encoding='utf8') as outfile:
        #for entry in JSON_file:
        json.dump(mtd, outfile, ensure_ascii=False)
        outfile.write('\n')

def get_doc(spatial_extent,voc_concept,motscles,thematic, logger, site):
    print("starting doc collector...")
    """
		Effecue la recherche.

		@type  ville: String.
		@param ville: Nom de la ville.

		@type  motscles: Liste de string.
		@param motscles: Liste des mots clés à rechercher.

		@type  logger: Logger.
		@param logger: Fichier de log.
	"""

    # Initialisation du nettoyeur de texte
    tc = TextCleaner()

    # Génération de la liste des requêtes
    queries_list = query_generator(format_word(spatial_extent), motscles, logger, site)
    #print('>>#', liste_requetes)
    for requete in queries_list:
        #print('###', requete)
        response = search(requete, lang='fr', stop=10) # stop=10
        # response = bing_search.search(requete)  # stop=10
        try:
            #liste_url = list(response)
            liste_url = set(response)
        except:
            liste_url = []
#         resume = 'resume'
        with (open("{0}{1}/resume.txt".format(params.CHEMIN_RESULTATS, spatial_extent), "a")) as res:
            logger.info("Nouvelle requête : {0}\n".format(requete))
            res.write("Requête : {0}\n".format(requete))
            res.write("Nombre de résultats affichés : {0}\n".format(len(liste_url)))
            res.write("\nListe des résultats\n")
            res.write("\n")
            for url in liste_url:
                if not url.endswith('xml.gz'):
                    logger.info("URL : {0}\n".format(url))
                    res.write("{0}\n".format(url))
                    try:
                        # On temporise chaque requête Google pour ne pas être bloqué
                        with sleep_time(30):
                            document = sauvegarde_fichier_advanced(spatial_extent, url, logger, tc)

                            # On s'assure que le document existe et qu'un texte y est associé
                            if document and 'text' in document:
                                #print("ok")
                                document = compute_best_doc(voc_concept, document)
                                document['source']['type'] = 'web_request'
                                document['source']['raw_request'] = requete
                                document['spatial_extent'] = spatial_extent
                                document['thematic'] = thematic
                                #print("thematic :", thematic)
                                # insert the doc into the Jsonl file
                                db_to_jsonl(document, thematic)
                                logger.info("Document inséré dans l'inventaire.\n")
                    except Exception as e:
                        print(e)
                        logger.info("Erreur pour l'URL : {0}\n".format(url))
                        res.write("Erreur pour l'URL : {0}\n".format(url))
            logger.info("***********************************************************************\n")
            res.write("***********************************************************************\n\n")
        with open("{0}{1}/.sauvegarde.txt".format(params.CHEMIN_RESULTATS, spatial_extent), "a") as f:
            f.write("{0}\n".format(requete))

        pause(logger)

def creation_dossier_resultat(chemin_resultats, ville, log, keyword_file) :
    """
        Création des dossiers de stockage et renvoi des mots clés.

        @type  ville: String.
        @param ville: Nom de la ville.

        @rtype: Liste de string.
        @return: Liste des mots clés à rechercher.
    """
    head, tail = os.path.split(keyword_file)
#     head,tail.split('.')[0]
    thematic = tail.split('.')[0]# keyword_file
    if not os.path.exists(chemin_resultats + ville):
        os.makedirs(chemin_resultats + ville)
        log.info("Création des dossier pour {0}".format(ville))
    if not os.path.exists("{0}{1}/Documents_SRC".format(chemin_resultats, ville)):
        os.makedirs("{0}{1}/Documents_SRC".format(chemin_resultats, ville))
        log.info("Création du dossier Documents_SRC\n")

    with open(keyword_file) as keywords_file:
        words = keywords_file.readlines()
        log.info("Lecture de keywords.txt ... \n")
    open("{0}{1}/.sauvegarde.txt".format(chemin_resultats, ville),"a").close()
    for i in range(0, len(words)):
        words[i] = format_word(words[i])
    return words, thematic

def scrapper(spatial_extent,voc_concept, site=''):
    logger = Log(params.CHEMIN_LOG, 'collecteDeDonnees_{0}'.format(datetime.today().strftime("%d-%m-%y")))
    spatial_extent = spatial_extent.title()
    motscles, thematic = creation_dossier_resultat(params.CHEMIN_RESULTATS, spatial_extent, logger, voc_concept)
    logger.info("Début de la recherche de document concernant la ville de {0}".format(spatial_extent))
    get_doc(spatial_extent,voc_concept, motscles,thematic, logger, site)


"""Test the integrity of the datasources references."""

import csv
import unittest
from pathlib import Path

import bioregistry
from bioregistry.external.miriam import get_miriam

HERE = Path(__file__).parent.resolve()
ROOT = HERE.parent.resolve()
HEADERS_PATH = ROOT.joinpath("datasources_headers.tsv")
DATASOURCES_PATH = ROOT.joinpath("datasources.tsv")

class TestIntegrity(unittest.TestCase):
    """Test data integrity."""

    @classmethod
    def setUpClass(cls) -> None:
        """Load the data sources rows."""
        with HEADERS_PATH.open() as file:
            _, *cls.columns = [row[1] for row in csv.reader(file, delimiter="\t")]
        with DATASOURCES_PATH.open() as file:
            cls.rows = list(csv.reader(file, delimiter="\t"))

    def test_lengths(self):
        """Test the row lengths."""
        for i, line in enumerate(self.rows, start=1):
            with self.subTest(line=i, resource=line[0]):
                self.assertEqual(
                    len(self.columns),
                    len(line),
                    msg=f"Row {i} has the wrong number of columns",
                )

    def test_valid_miriam(self):
        """Test that MIRIAM prefixes are valid."""
        valid_miriam_prefixes = set(get_miriam())
        for i, line in enumerate(self.rows, start=1):
            resource, uri = line[0], line[8]
            if not uri.startswith("urn:miriam"):
                continue
            miriam_prefix = uri.removeprefix("urn:miriam:")
            with self.subTest(resource=resource, miriam=miriam_prefix):
                self.assertIn(
                    miriam_prefix,
                    valid_miriam_prefixes,
                    msg=f"\n\t[line {i}] Invalid MIRIAM prefix for {resource}: {uri}",
                )

    def test_patterns(self):
        """Check the example identifiers pass the given regular expressions."""
        for i, line in enumerate(self.rows, start=1):
            resource, example, pattern = line[0], line[4], line[9]
            if not example or not pattern:
                continue
            with self.subTest(resource=resource, example=example, pattern=pattern):
                self.assertRegex(example, pattern)

    def test_valid_bioregistry(self):
        """Test that Bioregistry prefixes are valid."""
        for i, line in enumerate(self.rows, start=1):
            resource, bioregistry_prefix = line[0], line[12]
            if not bioregistry_prefix:
                continue
            with self.subTest(resource=resource, prefix=bioregistry_prefix):
                norm_prefix = bioregistry.normalize_prefix(bioregistry_prefix)
                self.assertIsNotNone(
                    norm_prefix,
                    msg=f"unrecognized Bioregistry prefix: {bioregistry_prefix} in {resource}",
                )
                self.assertEqual(
                    bioregistry_prefix,
                    norm_prefix,
                    msg="unstandardized Bioregistry prefix",
                )


#!/usr/bin/python3

# Imports (might cleanup in future, like just get the needed imports from argparse/nmap/requests)
import argparse
import nmap
import requests
from zlib import compress
from base64 import b64encode

def main():
    """
    Main function to run everything
    :return: None
    """
    # Gather args into a variable
    arguments = args()

    print(f"[Running nmap scan against {arguments.TargetIPAddress}]")
    # Run NMAP, returns list (0=scan,1=scan_time)
    nmap_output = runnmap(arguments)

    print(f"[Converting scan into UML]")
    # Run planuml conversion
    toplanuml(nmap_output[0],nmap_output[1],arguments)

    pass

def args():
    """
    A function to gather arguments for parsing
    :return: args: Populated namespace of argument values
    """
    # Setup basic parser
    arg_parser = argparse.ArgumentParser(
        prog="ntpu",
        description="NmapToPlanUml (NTPU) is a command line program that runs nmap to create visual diagrams of a network via PlanUML"
    )

    # Add arguments
    arg_parser.add_argument(
     "-I","--OutputUML",
     help="Filename to store UML code"
    )

    arg_parser.add_argument(
     "-S","--Server",
     help="HTTP/s address of self hosted PlanUML server, Example: http://localhost:8080 (MUST CONTAIN NO '/' AT THE END)"
    )

    arg_parser.add_argument(
        "-O","--OutputAscii",
        action="store_true",
        help="Enable sending UML code to PlanUML servers and return ASCII art of scanned computers"
    )

    arg_parser.add_argument(
        "TargetIPAddress",
        metavar="IP",
        nargs="+",
        help="The IP address/es of the host/s you wish to scan. Input separated with a space, and can make use of input ranges from NMAP ('-' and CIDR notation). Example: 192.168.0.1 192.168.0.2 192.168.0.5-10 192.168.0.1/23"
        )

    # Parse the arguments then return objects
    args = arg_parser.parse_args()

    return args

def runnmap(args):
    """
    A function that runs nmap
    :param: Populated namespace of argument values
    :return: Json formatted results of nmap
    """
    # Take the list of target ips from arguments and join them
    ipaddrs = " ".join(args.TargetIPAddress)
    nm = nmap.PortScanner()
    # Scan the results, clean them, then return
    scan_results = cleanscan(nm.scan(hosts=ipaddrs))

    return scan_results

def cleanscan(scan):
    """
    A function to cleanup nmap scan results with default nmap console settings
    :param: scan: Dict of raw nmap scan results
    :return: results: List with clean nmap scan results/scan timers
    """
    # Parse the scan and gather values from certain keys for planuml formatting
    clean_results = scan['scan']
    scan_time = scan['nmap']['scanstats']

    results = [clean_results,scan_time]

    return results

def toplanuml(nmap_json,scan_time,args):
    """
    A function to convert Dict formmatted results of nmap into planuml
    :param: nmap_json: Dict formmatted nmap scan results
    :param: scan_time: Dict of formmated nmap scan time results
    :param: args: Populated namespace of argument values
    :return: None
    """

    # Create a string of beginning uml code
    uml_code=f"""@startuml
!define osaPuml https://raw.githubusercontent.com/Crashedmind/PlantUML-opensecurityarchitecture2-icons/master
!include osaPuml/Common.puml
!include osaPuml/User/all.puml
!include osaPuml/Hardware/all.puml
!include osaPuml/Misc/all.puml
!include osaPuml/Server/all.puml
!include osaPuml/Site/all.puml
allowmixing

title Scan results:
legend
Scanned IPs: {scan_time['totalhosts']} in {scan_time['elapsed']} seconds
| Uphosts: {scan_time['uphosts']}, Downhosts: {scan_time['downhosts']}, Totalhosts: {scan_time['totalhosts']}|
end legend

left footer {scan_time['timestr']}

object self {{
    <$osa_desktop>
}}
frame "Uphosts" {{\n"""

    # For each host in the json
    for num, host in enumerate(nmap_json.keys()):
        # Determine hostname (whether or not nmap collected one)
        if nmap_json[host]['hostnames'][0]['name'] != '':
            hostname = nmap_json[host]['hostnames'][0]['name']
        else:
            hostname = host
        ###

        # Handling ports
        total_ports = []
        open_ports = []

        # Append each port with its port and name in a certain format to a list, if it is also open, append that to another list
        try:
            # Need this in try and except as if the system has no tcp ports open, will yield error
            for port in nmap_json[host]['tcp'].keys():
                total_ports.append(f"{port}/{nmap_json[host]['tcp'][port]['name']}")

                if nmap_json[host]['tcp'][port]['state'] == 'open':
                    open_ports.append(f"{port}")
        except Exception:
            pass
        ###

        # From data above, generate a object with information collected from nmap (ports are the joining of the entire list with commas)
        uml_code += f"""object "{hostname}" as {num} {{
                <$osa_server>
                status = {nmap_json[host]['status']['state']}\n
                IPs = {nmap_json[host]['addresses']}
                total_ports/services = {','.join(total_ports)}
                open_ports = {','.join(open_ports)}
                }}
                self <--> {num}\n"""

    # Standard end of formatting
    uml_code += f"@enduml"

    # Compress the uml code
    compressed_uml = planuml_encode(uml_code)

    # Argument selection stuff
    # If outputuml has value, then record uml code to file
    if args.OutputUML:
        try:
            with open(args.OutputUML, "w") as output_file:
                output_file.write(uml_code)
            print(f"Output saved in: {args.OutputUML}")
        except Exception as e:
            print(e)

    # If self hosted option is selected, set the uml_link to whatever it is, if not, default to plantuml.com
    if args.Server:
        uml_link = f"{args.Server}"
    else:
        uml_link = f"https://www.plantuml.com/plantuml"

    # If outputascii flag is set, output generated ascii art
    if args.OutputAscii:
        # r = requests.get(f"https://www.plantuml.com/plantuml/txt/~h{compressed_uml}")
        # r = requests.get(f"https://www.plantuml.com/plantuml/txt/{compressed_uml}")
        r = requests.get(f"{uml_link}/txt/{compressed_uml}")
        print(r.text)

    # print(f"Link to the generated scan on PlanUML: https://www.plantuml.com/plantuml/uml/~h{compressed_uml}")
    # print(f"Link to the generated scan on PlanUML: https://www.plantuml.com/plantuml/uml/{compressed_uml}")
    print(f"Link to the generated scan on PlanUML: {uml_link}/uml/{compressed_uml}")

def planuml_encode(planuml_data):
    """
    A function to encode planuml data, heavily inspired by https://github.com/dougn/python-plantuml/blob/master/plantuml.py
    :param: planuml_data: String of planuml input
    :return: String of compressed values
    """
    # Take UML, encode with UTF-8, convert bytes to hex
    # encoded_uml = planuml_data.encode('utf-8').hex()

    # Establish base64 and planuml character mapping encoded in utf-8
    b64_mapping = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/".encode('utf-8')
    planuml_mapping = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-_".encode('utf-8')

    # Compress the data, encode in utf-8, strip headers and checksum
    compressed_uml = compress(planuml_data.encode('utf-8'))[2:-4]

    # Create a mapping table of replacing b64 with planuml
    b64_planuml_mapping = compressed_uml.maketrans(b64_mapping, planuml_mapping)

    # Encode the compression with base64, translate it with the mapping, decode the utf-8
    encoded_uml = b64encode(compressed_uml).translate(b64_planuml_mapping).decode('utf-8')

    # Return compressed UML
    return encoded_uml

# Check whether importing (__name__ will equal import name) or run itself (will have "__main__" value)
if __name__ == "__main__":
    main()


# Super () can be used in the methods
# class P:
#     def property(self):
#         print('cash+land+gold')
#     def marriage(self):
#         print("Priya")
# class C(P):
#     def marriage(self):
#         # super().marriage()
#         print("Shilpa")
# c=C()
# c.property()
# c.marriage()

# class Animal(object):
#     def __init__(self, AnimalName):
#         print(AnimalName, 'is a warm-blooded animal.')
#
# class Dog(Animal):
#     def __init__(self):
#         print('Dog has four legs.')
#         super().__init__('lion')
# d1 = Dog()

class college:
    def __init__(self,name,age):
        self.name=name
        self.age=age
class Student(college):
    def __init__(self,name,age,rollno,marks):
        # super().__init__(name,age)
        self.rollno=rollno
        self.marks=marks
s=Student('xyz',20,10,70)
print(s.name)
print(s.age)
print(s.rollno)
print(s.marks)


'''
	Author: Mohit Sharma
	Email: sharm267@purdue.edu
'''

from NewtonStep import NewtonStep
from Step import Step
from StepSize import StepSize
import numpy as np
import math
import util

'''
	For the current point, completes the predictor step and the corrector step and returns the
	step for primal and dual variables
'''
class PredictorCorrector(Step):

	def __init__(self,LP,curr_pt):
		self.LP = LP
		self._curr_pt = curr_pt

		self._gamma = None
		self._eta = None

		self.beta_1 = 0.1

		step = self.predictor_corrector()
		super(PredictorCorrector,self).__init__(dx=step.dx,dy=step.dy,ds=step.ds,dtau=step.dtau,dkappa=step.dkappa)

	@property
	def gamma(self):
		return self._gamma

	@property
	def eta(self):
		return self._eta

	@gamma.setter
	def gamma(self,val):
		self._gamma = val

	@eta.setter
	def eta(self,val):
		self._eta = val

	'''
		Predictor steps get the Newton Step for gamma = 0 and eta = 1
	'''
	def predictor_corrector(self):
		self.gamma = 0
		self.eta = 1

		predictor_step = NewtonStep(self.LP,self._curr_pt,self.eta,self.gamma)

		# Get the step lengths for primal and dual and calculate the alpha and eta parameters for the corrector step

		step_size = StepSize(self._curr_pt,predictor_step)

		self.set_corrector_params(step_size,predictor_step)

		corrector_step = NewtonStep(self.LP,self._curr_pt,self.eta,self.gamma,residual_step=predictor_step)

		return corrector_step

	'''
		Set the values of Gamma and Eta for the corrector step
	'''
	def set_corrector_params(self,step_size,predictor_step):

		# alpha_t = np.min((step_size.alpha_p,step_size.alpha_d)) if step_size.same_step else step_size.alpha_t
		# alpha_k = np.min((step_size.alpha_p,step_size.alpha_d)) if step_size.same_step else step_size.alpha_k

		s1 = util.dot((self._curr_pt.x + step_size.alpha_p*predictor_step.dx),self._curr_pt.s + step_size.alpha_d*predictor_step.ds)
		s2 = (self._curr_pt.tau + step_size.alpha_t*predictor_step.dtau)*(self._curr_pt.kappa + step_size.alpha_k*predictor_step.dkappa)

		mu_a = s1 + s2
		mu_k = util.dot(self._curr_pt.x,self._curr_pt.s) + self._curr_pt.tau*self._curr_pt.kappa

		# Calculate the Gamma and Eta parameters

		if mu_a/mu_k <= 0.01:
			self.gamma = math.pow(mu_a/mu_k,2)
		else:
			self.gamma = np.min([0.1,np.max([math.pow(mu_a/mu_k,3),0.0001])])

		self.eta = 1 - self.gamma

	'''
	Get the maximum step sizes for each variable without violating the non-negativity constraints
	'''
	def compute_non_negative_step_size(self):
		keep_x = np.where(self._step.dx <=0)[0]
		alpha_x = np.min(np.true_divide(self._curr_pt.x[keep_x],self._step.dx[keep_x]))

		keep_s = np.where(self._step.ds <=0)[0]
		alpha_s = np.min(np.true_divide(self._curr_pt.s[keep_s],self._step.ds[keep_s]))

		alpha_tau = -1*self._curr_pt.tau / self._step.dtau
		alpha_kappa = -1*self._curr_pt.kappa / self._step.dkappa

		assert(alpha_x>0)
		assert(alpha_s>0)
		assert(alpha_tau>0)
		assert(alpha_kappa >0)

		self.alpha_p = np.min([alpha_x,alpha_tau,alpha_kappa])
		self.alpha_d = np.min([alpha_s,alpha_tau,alpha_kappa])

	def get_alpha_max(self,step):
		z = np.vstack((self.curr_pt.x,self._curr_pt.tau,self._curr_pt.s,self._curr_pt.kappa))
		d = np.vstack((step.dx,step.dtau,step.ds,step.dkappa))

		assert(z.shape == d.shape)

		keep = np.where(d <= 0)[0]

		alpha_max = np.min(np.true_divide(z[keep],d[keep]))

		assert(alpha_max >= 0)

		return alpha_max


from . import Atomic
from . import util
import argparse
import json as Json
import tty
import sys
import termios
import select
from os import isatty
from operator import itemgetter
import requests
from Atomic.backends._docker_errors import NoDockerDaemon

def check_negative(value):
    ivalue = int(value)
    if ivalue < 1:
        raise argparse.ArgumentTypeError("%s must be a positive integer value greater than 0." % value)
    return ivalue

def cli(subparser):
    # atomic top
    topp = subparser.add_parser("top",
                                help=_("Show top-like stats about processes running in containers"))
    topp.set_defaults(_class=Top, func='atomic_top')
    topp.add_argument("-d", type=int, default=1, help=_("Interval (secs) to refresh process information"))
    topp.add_argument("-o", "--optional", help=_("Additional fields to display"), nargs='?', action='append',
                      choices=['time', 'stime', 'ppid', 'uid', 'gid', 'user', 'group'])
    topp.add_argument("-n", help=_("Number of iterations"), type=check_negative)
    topp.add_argument("containers", nargs="*", help=_("list of containers to monitor, leave blank for all"))

class Top(Atomic):
    """
    Class to support atomic top; based on the Atomic class
    """
    # Set to true to output debug information

    def __init__(self):
        super(Top, self).__init__()
        self.input_var = None
        self._sort = 'CID'
        self.name_id = {}
        self.optional = None
        self.titles = None
        self.debug = False
        # To add a new column to the output, create a new dict inside self.headers
        # The fields are as follows:
        # shortname: <str> a unique key used to describe the dict
        # column: <str> column title, use parans to define a sort character
        # character: <str> a single character the user presses to sort on that column
        # sort: <bool> to determine IF the column can be sorted upon
        # index: <int> order the columns display left to right. Do not change 0-5
        # _field: <str> If the width of the column needs to be dynamically sized, put
        #               the formatting in _field, set field to None, and be sure to
        #               add a key for _min_width
        # field: <str> the preferred format for the column
        # active: if the column should be active by default
        # ps_opt: <str> the descriptor for ps to get the column's data (ps -o)
        # sort_order: <bool> True reverses the sort order from ascending to descending
        # _min_width: <int> Minimum field width if you are using dynamic fields
        self.headers = [
            {'shortname': '%CPU', 'column': '(C)PU', 'character': 'c', 'sort': True, 'index': 3,
             'field': '{:6}', 'active': True, 'ps_opt': 'pcpu', 'sort_order': True},
            {'shortname': 'CID', 'column': 'CONTA(I)NER', 'character': 'i', 'sort': True, 'index': 0,
             'field': '{:12.12}', 'active': True, 'ps_opt': None, 'sort_order': False},
            {'shortname': 'NAME', 'column': '(N)AME', 'character': 'n', 'sort': True, 'index': 1,
             '_field': '{:WIDTH}', 'field': None, 'active': True, 'ps_opt': None, 'sort_order': False,
             '_min_width': 10},
            {'shortname': 'PID', 'column': '(P)ID', 'character': 'p', 'sort': True, 'index': 2,
             'field': '{:<10}', 'active': True, 'ps_opt': 'pid', 'sort_order': False},
            {'shortname': '%MEM', 'column': '(M)EM', 'character': 'm', 'sort': True, 'index': 4,
             'field': '{:6}', 'active': True, 'ps_opt': 'pmem', 'sort_order': True},
            {'shortname': 'CMD', 'column': 'CMD', 'character': None, 'sort': False, 'index': 99,
             '_field': '{:WIDTH}', 'field': None, 'active': True, 'ps_opt': 'cmd', '_min_width': 10},
            {'shortname': 'TIME', 'column': '(T)IME', 'character': 't', 'sort': True, 'index': 6,
             'field': '{:10}', 'active': False, 'ps_opt': 'time', 'sort_order': True},
            {'shortname': 'STIME', 'column': '(S)TIME', 'character': 's', 'sort': True, 'index': 7,
             'field': '{:10}', 'active': False, 'ps_opt': 'stime', 'sort_order': True},
            {'shortname': 'PPID', 'column': 'PPI(D)', 'character': 'd', 'sort': True, 'index': 8,
             'field': '{:10}', 'active': False, 'ps_opt': 'ppid', 'sort_order': False},
            {'shortname': 'UID', 'column': '(U)ID', 'character': 'u', 'sort': True, 'index': 9,
             'field': '{:6}', 'active': True, 'ps_opt': 'uid', 'sort_order': True},
            {'shortname': 'GID', 'column': '(G)ID', 'character': 'g', 'sort': True, 'index': 10,
             'field': '{:6}', 'active': True, 'ps_opt': 'gid', 'sort_order': True},
            {'shortname': 'USER', 'column': 'USER', 'character': None, 'sort': False, 'index': 11,
             'field': '{:10}', 'active': False, 'ps_opt': 'user'},
            {'shortname': 'GROUP', 'column': 'GROUP', 'character': None, 'sort': False, 'index': 12,
             'field': '{:10}', 'active': False, 'ps_opt': 'group'}
        ]

    def _activate_optionals(self):
        """
        Sets the active bool to True for any optional  ps
        arguments.
        :return: None
        """
        if self.args.optional:
            for option in self.args.optional:
                for header in self.headers:
                    if header['ps_opt'] == option:
                        header['active'] = True

    def _set_dynamic_column_widths(self, ps_info):
        for header in self.headers:
            if '_field' in header:
                max_widths = [len(x[header['shortname']]) for x in ps_info]
                max_width = header['_min_width'] if not max_widths else max(max_widths)
                header['field'] = header['_field'].replace('WIDTH', str(max_width))

    def json(self):
        """
        Main sub-function for top
        :return: None
        """
        # Make sure the docker daemon is running
        self.ping()
        # Activate optional columns
        self._activate_optionals()
        proc_info = []
        if len(self.args.containers) < 1:
            try:
                con_ids = [x['Id'] for x in self.get_active_containers(refresh=True)]
            except requests.exceptions.ConnectionError:
                raise NoDockerDaemon()
        else:
            con_ids = []
            self.get_active_containers(refresh=True)
            # verify the inputs are valid
            for user_input in self.args.containers:
                con_ids.append(self.get_input_id(user_input))
        for cid in con_ids:
            proc_info += self.get_pids_by_container(cid)

        return Json.dumps(self.reformat_ps_info(proc_info))

    def atomic_top(self):
        """
        Main sub-function for top
        :return: None
        """
        # Make sure the docker daemon is running
        self.ping()
        # Set debug bool
        self.set_debug()
        # Activate optional columns
        self._activate_optionals()
        # Do we have a tty?
        # Can run ./atomic top <&- to replicate no tty
        has_tty = isatty(0)
        # Set up terminal, input handling
        if has_tty:
            fd = sys.stdin.fileno()
            old_settings = termios.tcgetattr(fd)
        sort_vals = [x['character'].upper() for x in self.headers if x['active'] and x['sort']]
        counter = 0
        while True:
            proc_info = []
            if len(self.args.containers) < 1:
                try:
                    con_ids = [x['Id'] for x in self.get_active_containers(refresh=True)]
                except requests.exceptions.ConnectionError:
                    raise NoDockerDaemon()
            else:
                con_ids = []
                self.get_active_containers(refresh=True)
                # verify the inputs are valid
                for user_input in self.args.containers:
                    con_ids.append(self.get_input_id(user_input))
            for cid in con_ids:
                proc_info += self.get_pids_by_container(cid)
            # Reset screen
            if not self.debug and has_tty:
                util.write_out("\033c")
            sorted_info = self.reformat_ps_info(proc_info)
            self._set_dynamic_column_widths(sorted_info)
            self.output_top(sorted_info)
            if has_tty:
                tty.setraw(sys.stdin.fileno())
            i, _, _ = select.select([sys.stdin], [], [], self.args.d)
            if i and has_tty:
                ch = sys.stdin.read(1)
                # Detect 'q' or Cntrl-c
                if ch.upper() == 'q'.upper() or ord(ch) == 3:
                    # reset terminal
                    termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
                    raise SystemExit
                # Detect of the character pushed is one in self.headers
                elif ch.upper() in sort_vals:
                    self._sort = next((header['shortname'] for header in self.headers if header['character'] is not
                                       None and header['character'].upper() == ch.upper()), False)

            # reset terminal
            if has_tty:
                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
            counter += 1
            if counter == self.args.n:
                raise SystemExit

    def get_pids_by_container(self, con_id):
        """
        Call the custom API for docker python
        :param con_id: id of the container
        :return: dict
        """

        if len(self.active_containers) == 0:
            raise ValueError("No containers running")
        f_procs = []
        # Get the name of the container by ID
        con_name = str(next((l for l in self.active_containers if l['Id'].startswith(con_id)), None)['Names'][0])
        if con_name.startswith("/"):
            con_name = con_name.replace("/", "")
        # Assemble the ps args
        ps_args = [header['ps_opt']for header in sorted(self.headers, key=itemgetter('index')) if header['ps_opt']
                   is not None and header['active']]
        con_procs = self.d.top(con_id, ps_args="-eo {}".format(",".join(ps_args)))
        # Set the column header titles one-time
        if self.titles is None:
            self.titles = con_procs['Titles']

        if con_procs['Processes'] != None:
            # Massage the information into a dict
            for proc in con_procs['Processes']:
                t_dict = {'CID': con_id,
                          'NAME': con_name}
                for place in range(0, len(proc)):
                    t_dict[self.titles[place]] = proc[place]
                f_procs.append(t_dict)
        return f_procs

    def output_top(self, sorted_info):
        """
        Primary function for output to stdout
        :param sorted_info: list of dicts containing process information
        :return: None
        """
        cols = [col for col in sorted(self.headers, key=itemgetter('index')) if col['active']]
        active_column_names = [x['shortname'] for x in cols]
        out_format = " ".join([x['field'] for x in cols])
        col_headers = []

        # Add a '*' to the column header name currently being
        # sorted on
        for col in cols:
            title = col['column']
            if col['shortname'] == self._sort:
                title += "*"
            col_headers.append(title)
        formatted_col_headers = "\033[7m" + out_format.format(*col_headers) + "\033[0m"
        # output ATOMIC TOP title
        almost_center = len(formatted_col_headers)
        center_col = '{:^%WIDTH%}'.replace("%WIDTH%", str(almost_center))
        util.write_out(center_col.format("ATOMIC TOP\n"))
        # Output the headers
        util.write_out(formatted_col_headers)
        for ps in sorted_info:
            line_out = []
            for val in active_column_names:
                line_out.append(ps[val])
            # Output the ps information
            util.write_out(out_format.format(*line_out))

    def reformat_ps_info(self, proc_info):
        """
        Takes a structure of process information and re-organizes it into
        a list of dictionaries, where each dict is a process.
        :param proc_info:
        :return: a list of dicts
        """
        # Determine if the sort field needs to reverse the order of the sort
        _reverse = next((header['sort_order'] for header in self.headers if header['shortname'] == self._sort), False)
        if self.debug:
            util.write_out("sorting on {0} and reverse is {1}".format(self._sort, _reverse))
        return sorted(proc_info, key=itemgetter(self._sort), reverse=_reverse)


import csv
import requests

def check_url(url):

    custom_header = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'
    }

    try:
        response = requests.get(url, timeout = 5)
        if response.status_code == 404:
            return 'Failure'
        else:
            return 'Success'
    except requests.exceptions.RequestException as e:
        return 'Error: ' + str(e)

def check_urls_from_csv(file_name, url_column):
    with open(file_name, 'r') as file:
        reader = csv.DictReader(file)
        row_number = 0
        websites_down = 0
        for row in reader:
            row_number += 1
            url = row[url_column]
            status = check_url(url)
            if(status == 'Failure'):
                websites_down += 1
                print(f'Row: {row_number}, URL: {url}, Status: {status}')
            elif(row_number%10 == 0):
                print(f'Current row: {row_number}')

        print(f'Done! Found {websites_down} website/s down\r\n')

# Replace with the name of your CSV file (same directory than py file)
# And the name of the column that contains the URLs
check_urls_from_csv('urls.csv', 'URL')


import os.path
import module.rclone as rclone

class Settings:
    import module.utility
    import json

    REMOTE_NAME = 'unibackup'
    REMOTE_DIR = REMOTE_NAME + ':unibackup/current/'
    REMOTE_TYPE = 'onedrive'

    MAINFILE_NAME = '.unibackup'
    MAINFILE_DEFAULT = ''
    MAINFILE_PATH = MAINFILE_NAME

    LOCALFILE_NAME = '.unibackup_local'

    first_sync = 'first_sync'
    LOCALFILE_DEFAULT = json.dumps(
        {
            first_sync: True
        },
        indent=4)

    LOCALFILE_PATH = LOCALFILE_NAME

    EXCLUDEFILE_NAME = '.unibackup_ignore'
    EXCLUDEFILE_DEFAULT = '''
    # EXCLUDE

    # do not touch the next 2 lines
    - /.unibackup_local
    - /.unibackup_ignore.md5

    # put your exclude here
    # - .git/ (example)
    '''
    EXCLUDEFILE_PATH = EXCLUDEFILE_NAME

    class Localfile(module.utility.OptionFile):
        def __init__(self):
            super().__init__(Settings.LOCALFILE_PATH)

        def __del__(self):
            super().__del__()

        def is_first_sync(self) -> bool:
            return super().get(Settings.first_sync)

        def remember_sync(self):
            super().set(Settings.first_sync, False)

class Backup:
    def __init__(self, path):
        self.path = os.path.abspath(path)
        self.local = self.path
        self.remote = self.local_to_remote(self.path)

        self.exludefile_local = self.local_to_local(Settings.EXCLUDEFILE_PATH)

    def local_to_local(self, path: str) -> str:
        return os.path.join(self.path, path)

    def local_to_remote(self, path: str) -> str:
        return os.path.join(Settings.REMOTE_DIR, os.path.basename(path))

    def remote_to_remote(self, path: str) -> str:
        return os.path.join(Settings.REMOTE_DIR, path)

    def create_mainfile(self):
        f = open(self.local_to_local(Settings.MAINFILE_PATH), 'w')
        f.write(Settings.MAINFILE_DEFAULT)
        f.close()

    def create_excludefile(self):
        f = open(self.local_to_local(Settings.EXCLUDEFILE_PATH), 'w')
        f.write(Settings.EXCLUDEFILE_DEFAULT)
        f.close()

    def create_localfile(self):
        f = open(self.local_to_local(Settings.LOCALFILE_PATH), 'w')
        f.write(Settings.LOCALFILE_DEFAULT)
        f.close()

    def is_backup_dir(self) -> bool:
        """ check if a directory is configured for unibackup

        Returns:
            bool: True if is a directory configured for unibackup, False otherwise
        """
        return os.path.exists(self.local_to_local(Settings.MAINFILE_PATH))

    def is_configured(self) -> bool:
        """check if is properly configured

        Returns:
            bool: True if is properly configured, False otherwise
        """
        return rclone.remote_exists(Settings.REMOTE_NAME)

    def config(self):
        """configure csunibackup"""
        if not self.is_configured():
            rclone.remote_add(Settings.REMOTE_NAME, Settings.REMOTE_TYPE)
        else:
            # if is configured, readd the remote, useful when is expired
            rclone.remote_delete(Settings.REMOTE_NAME)
            rclone.remote_add(Settings.REMOTE_NAME, Settings.REMOTE_TYPE)

        # make csunibackup/ path in the remote
        rclone.mkdir(Settings.REMOTE_DIR)

    def init(self):
        """ initialize unibakcup in the given path"""
        if not self.is_backup_dir():
            # create unibackup files
            self.create_mainfile()
            self.create_localfile()
            self.create_excludefile()

    def listbackups(self):
        if self.is_configured():
            data = rclone.lsjson(Settings.REMOTE_DIR, filters=['Name', 'IsDir'])

            print("list of folders saved in remote:")
            for d in data:
                if d['IsDir']:
                    print(d['Name'])

    def clone(self, remote_dir: str):
        """clone a remote unibackup directory in a respective local folder
        """
        remote = self.remote_to_remote(remote_dir)
        local = self.local_to_local(remote_dir)
        rclone.mkdir(local)

        rclone.copy(remote, local, options=[
            rclone.progress,
            rclone.verbose
            ])

        self.__init__(local)
        self.create_localfile()

    def deletebackup(self, remote_dir: str):
        """delete a remote unibackup directory"""
        remote = self.remote_to_remote(remote_dir)

        rclone.purge(remote, options=[
            rclone.verbose
            ])

    def safepush(self):
        """perform a rclone.copy of a local path to the remote"""
        if self.is_backup_dir():
            rclone.copy(self.local, self.remote, options=[
                rclone.filter_from, Settings.EXCLUDEFILE_PATH,
                rclone.progress,
                rclone.verbose
                ])
        else:
            raise Exception("unibackup is not initialized in this directory")

    def safepull(self):
        """perform a rclone.copy from the remote to local"""
        if self.is_backup_dir():
            rclone.copy(self.remote, self.local, options=[
                rclone.filter_from, Settings.EXCLUDEFILE_PATH,
                rclone.progress,
                rclone.verbose
                ])
        else:
            raise Exception("unibackup is not initialized in this directory")

    def push(self):
        """perform a rclone.sync of a local path to the remote
        similar to git push
        """
        if self.is_backup_dir():
            rclone.sync(self.local, self.remote, options=[
                rclone.filter_from, Settings.EXCLUDEFILE_PATH,
                rclone.progress,
                rclone.verbose
                ])
        else:
            raise Exception("unibackup is not initialized in this directory")

    def pull(self):
        """perform a rclone.sync of remote into a local path
        similar to git pull
        """
        if self.is_backup_dir():
            rclone.sync(self.remote, self.local, options=[
                rclone.filter_from, Settings.EXCLUDEFILE_PATH,
                rclone.progress,
                rclone.verbose
                ])
        else:
            raise Exception("unibackup is not initialized in this directory")

    def sync(self):
        """perform a rclone.bisync of a local path to the remote
        similar to git pull & push
        """
        settings = Settings.Localfile()

        if self.is_backup_dir():
            if settings.is_first_sync():
                rclone.mkdir(self.remote)
                rclone.bisync(self.local, self.remote, options=[
                    rclone.resync,
                    rclone.progress,
                    rclone.verbose
                    ])
                settings.remember_sync()
            else:
                try:
                    rclone.bisync(self.local, self.remote, options=[
                        rclone.filters_file, Settings.EXCLUDEFILE_PATH,
                        rclone.progress,
                        rclone.verbose
                        ])
                except:
                    # when updating .unibakcup_ignore
                    print('\nretry with --resync')
                    rclone.bisync(self.local, self.remote, options=[
                        rclone.resync,
                        rclone.filters_file, Settings.EXCLUDEFILE_PATH,
                        rclone.progress,
                        rclone.verbose
                        ])
        else:
            raise Exception("unibackup is not initialized in this directory")

    def status(self):
        if self.is_backup_dir():
            print('''
            = path means path was found in source and destination and was identical
            - path means path was missing on the source, so only in the destination
            + path means path was missing on the destination, so only in the source
            * path means path was present in source and destination but different.
            ! path means there was an error reading or hashing the source or dest.
            ''')
            rclone.check(self.local, self.remote, options=[
                '--combined', '-',
                rclone.filter_from, Settings.EXCLUDEFILE_PATH
                ])
        else:
            raise Exception("unibackup is not initialized in this directory")