#!/usr/bin/env python

import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)
import numpy as np
import pandas as pd
#warnings.resetwarnings()
import re
import os
import time
import argparse
import prob_dist as prob
import fano_calc as fc
import resfuncRead as rfr
import time
from argparse import ArgumentParser, ArgumentTypeError
import h5py
import EdwRes as er
from sklearn.neighbors import KernelDensity
import scipy.interpolate as inter
# Author: A. Villano
#

#the stuff below is so this functionality can be used as a script
########################################################################
if __name__ == "__main__":

        #make a parser for the input
        parser = argparse.ArgumentParser(description='options')
        parser.add_argument('-n','--nsamp', type=int, dest='Nsamp', default=4, help='number of samples to draw')
        parser.add_argument('-i','--Eidx', type=int, dest='Eidx', default=-1, help='NR point index for energy')
        parser.add_argument('-E','--energy', type=float, dest='Etest', default=25.0, help='NR test energy')
        #parser.add_argument('-x','--regex', type=str, dest='regstr', default=r'(.*?)', help='regex for picking up files')
        #parser.add_argument('-o','--outfile', type=str, dest='outfile', default='data.h5', help='output file for data')
        #parser.add_argument('-c','--cuts', type=str, dest='cuts', default='NR', help='kind of cuts to apply')
        #parser.set_defaults(filedir='./');

        args = parser.parse_args()

        try:

         #find what energies relevant for NR/ER data
         # import data from Edelweiss
         resNR_data = pd.read_csv("data/edelweiss_NRwidth_GGA3_data.txt", skiprows=1, \
                       names=['E_recoil', 'sig_NR', 'E_recoil_err', 'sig_NR_err'], \
                       delim_whitespace=True)

         resER_data = pd.read_csv("data/edelweiss_ERwidth_GGA3_data.txt", skiprows=1, \
                         names=['E_recoil', 'sig_ER', 'sig_ER_err'], \
                         delim_whitespace=True)

         # set the data up for the fits
         # Edelweiss discards ER points near peaks
         # and first two NR points since they're affected by the threshold
         mask = [True, True, False, False, True, True, True, True, True]
         ER_data = {'Erecoil': resER_data["E_recoil"][mask], 'sigma': resER_data["sig_ER"][mask], 'sigma_err': resER_data["sig_ER_err"][mask]}
         NR_data = {'Erecoil': resNR_data["E_recoil"][2::], 'sigma': resNR_data["sig_NR"][2::], 'sigma_err': resNR_data["sig_NR_err"][2::]}

         E = np.sort(NR_data['Erecoil'])
         print(E)

         if args.Eidx==-1:
           Etest = args.Etest
         else:
           Etest = E[args.Eidx]

         print('Energy is: {:03.1f}'.format(Etest))

         filename = 'data/edelweiss_C_systematicErrors_sampler_nll_allpars_gausPrior.h5'
         #remove vars
         f = h5py.File(filename,'r')

         #get path within file
         path='{}/{}/'.format('mcmc','sampler')

         samples = np.asarray(f[path+'samples'])
         A_bf = np.asarray(f[path+'A'])
         B_bf = np.asarray(f[path+'B'])
         aH_bf = np.asarray(f[path+'aH'])
         mu_bf = np.asarray(f[path+'scale'])

         ndim = 6
         print(np.shape(samples))
         samples_corner = samples[:, 300:, :].reshape((-1, ndim))
         print(np.shape(samples_corner))
         f.close()

         #print(samples_corner[0:5,:])

         Nsamp = args.Nsamp
         permute = np.zeros((Nsamp+2,4))
         #start with nominal params, then the best-fit params
         permute[0,0] = 0.16
         permute[0,1] = 0.18
         permute[0,2] = 0.0381
         permute[0,3] = 1.0

         permute[1,0] = A_bf
         permute[1,1] = B_bf
         permute[1,2] = aH_bf/er.FWHM_to_SIG
         permute[1,3] = mu_bf

         #for i, aH, C, m, scale, A, B in enumerate(samples_corner[np.random.randint(len(samples_corner), size=Nsamp)]):
         for i, parvec in enumerate(samples_corner[np.random.randint(len(samples_corner), size=Nsamp)]):
           #print(i)
           #print(parvec[0]/er.FWHM_to_SIG)
           aH = parvec[0]/er.FWHM_to_SIG
           A = parvec[4]
           B = parvec[5]
           mu = parvec[3]
           permute[i+2,0] = A
           permute[i+2,1] = B
           permute[i+2,2] = aH
           permute[i+2,3] = mu

         print(permute)

         #getting some sample parameter permutations
         #Nsamp = 3
         #A0 = 0.16
         #dA = 0.01
         #B0 = 0.18
         #dB = 0.01
         #aH0 = 0.0381
         #daH = 0.001
         #mu0 = 1.0
         #dmu = 0.05

         #Avec = np.concatenate(([A0],np.random.uniform(A0-dA,A0+dA,Nsamp)))
         #Bvec = np.concatenate(([B0],np.random.uniform(B0-dB,B0+dB,Nsamp)))
         #aHvec = np.concatenate(([aH0],np.random.uniform(aH0-daH,aH0+daH,Nsamp)))
         #muvec = np.concatenate(([mu0],np.random.uniform(mu0-dmu,mu0+dmu,Nsamp)))
         #print(Avec)
         #print(Bvec)
         #print(aHvec)
         #print(muvec)

         #permute=np.array(np.meshgrid(Avec, Bvec, aHvec,muvec)).T.reshape(-1,4)
         #print(np.shape(permute))

         output,frac = prob.diffmap(permute,Etest=Etest,outfile='data/sigdiff_test1.h5')
         print(output)
         print(frac)

         start = time.time()
         corr_func = inter.NearestNDInterpolator(permute,output)
         print(corr_func([0.13,0.17,0.0381,1.0]))
         end = time.time()
         print('Interpolate Time: {:1.5f} sec.'.format(end-start))
        except KeyboardInterrupt:
          print('Shutdown requested .... exiting')
        except Exception:
          traceback.print_exc(file=sys.stderr)


import json
import os
import datetime
import tarfile
import torch
import warnings
import copy
import yaml

from . import constants
from ... import utils
from . import datasets
from . import training
from . import compilation
from .params import init_params
from . import descriptions

class ModelRunner():
    @classmethod
    def init_params(self, *args, **kwargs):
        params = init_params(*args, **kwargs)
        # set the checkpoint download folder
        # (for the models that are downloaded using torch.hub eg. mmdetection uses that)
        torch.hub.set_dir(os.path.join(params.common.download_path, 'pretrained', 'torch', 'hub'))
        return params

    def __init__(self, *args, verbose=True, **kwargs):
        self.params = self.init_params(*args, **kwargs)

        # print the runner params
        if verbose:
            [print(key, ':', value) for key, value in vars(self.params).items()]
        #
        # normalize the paths
        self.params.common.projects_path = utils.absolute_path(self.params.common.projects_path)
        self.params.dataset.input_data_path = utils.absolute_path(self.params.dataset.input_data_path)
        self.params.dataset.input_annotation_path = utils.absolute_path(self.params.dataset.input_annotation_path)
        self.params.common.project_path = os.path.join(self.params.common.projects_path, self.params.dataset.dataset_name)

        self.params.common.run_name = self.resolve_run_name(self.params.common.run_name, self.params.training.model_name)
        self.params.common.project_run_path = os.path.join(self.params.common.project_path, 'run', self.params.common.run_name)

        self.params.dataset.dataset_path = os.path.join(self.params.common.project_path, 'dataset')
        self.params.dataset.extract_path = self.params.dataset.dataset_path

        self.params.training.training_path = os.path.join(self.params.common.project_run_path, 'training')
        self.params.training.model_packaged_path = os.path.join(self.params.training.training_path,
                                    '_'.join(os.path.split(self.params.common.run_name))+'.tar.gz')

        assert self.params.common.target_device in constants.TARGET_DEVICES_ALL, f'common.target_device must be set to one of: {constants.TARGET_DEVICES_ALL}'
        target_device_compilation_folder = self.params.common.target_device
        self.params.compilation.compilation_path = os.path.join(self.params.common.project_run_path, 'compilation', target_device_compilation_folder)

        if self.params.common.target_device in self.params.training.target_devices:
            performance_infer_time_ms_list = {k:v['performance_infer_time_ms'] for k,v in self.params.training.target_devices.items()}
            print('---------------------------------------------------------------------')
            print(f'Run Name: {self.params.common.run_name}')
            print(f'- Model: {self.params.training.model_name}')
            print(f'- TargetDevices & Estimated Inference Times (ms): {performance_infer_time_ms_list}')
            print(f'- This model can be compiled for the above device(s).')
            print('---------------------------------------------------------------------')
        #

    def resolve_run_name(self, run_name, model_name):
        if not run_name:
            return ''
        #
        # modify or set any parameters here as required.
        if '{date-time}' in run_name:
            run_name = run_name.replace('{date-time}', datetime.datetime.now().strftime("%Y%m%d-%H%M%S"))
        #
        if '{model_name}' in run_name:
            run_name = run_name.replace('{model_name}', model_name)
        #
        return run_name

    def clear(self):
        pass

    def prepare(self):
        ret_val = True

        # create folders
        os.makedirs(self.params.common.project_path, exist_ok=True)
        os.makedirs(self.params.common.project_run_path, exist_ok=True)

        #####################################################################
        # handle all downloads here
        utils.download_all(self.params)

        #####################################################################
        # prepare for dataset handling (loading, splitting, limiting files etc).
        self.dataset_handling = datasets.DatasetHandling(self.params)
        self.params.update(self.dataset_handling.get_params())
        # actual dataset handling
        if self.params.dataset.enable:
            self.dataset_handling.clear()
            ret_val = self.dataset_handling.run()
        #

        #####################################################################
        # prepare model training
        self.training_target_module = training.get_target_module(self.params.training.training_backend,
                                                              self.params.common.task_type)
        self.model_training = self.training_target_module.ModelTraining(self.params)
        self.params.update(self.model_training.get_params())

        #####################################################################
        # prepare for model compilation
        self.model_compilation = compilation.edgeai_benchmark.ModelCompilation(self.params)
        self.params.update(self.model_compilation.get_params())

        # write out the description of the current run
        run_params_file = self.write_status_file()
        print(f'Run params is at: {run_params_file}')

        if not ret_val:
            # dataset is incorrect - training could not start. create a training log file and write error.
            self.params.training.log_file_path = os.path.join(self.params.training.training_path, 'run.log')
            os.makedirs(os.path.dirname(self.params.training.log_file_path), exist_ok=True)
            print('\nERROR: ModelMaker - Training could not start (dataset error)')
            with open(self.params.training.log_file_path, 'a') as lfp:
                lfp.write('\nERROR: ModelMaker - Training could not start (dataset error)')
            #
        #

        return ret_val

    def run(self):
        #####################################################################
        # actual model training
        if self.params.training.enable:
            self.model_training.clear()
            self.model_training.run()
            # remove special characters
            utils.cleanup_special_chars(self.params.training.log_file_path)
            # training frameworks don't create a compact package after training. do it here.
            model_training_package_files = [
                self.params.dataset.annotation_path_splits,
                self.params.training.model_checkpoint_path, self.params.training.model_export_path,
                self.params.training.model_proto_path, self.params.training.log_file_path]
            self.package_trained_model(model_training_package_files, self.params.training.model_packaged_path)
            # we are done with training
            print(f'Trained model is at: {self.params.training.training_path}', flush=True)
            print('\nSUCCESS: ModelMaker - Training completed.')
            with open(self.params.training.log_file_path, 'a') as lfp:
                lfp.write('\nSUCCESS: ModelMaker - Training completed.')
            #
        #

        #####################################################################
        # actual model compilation
        if self.params.compilation.enable:
            self.model_compilation.clear()
            self.model_compilation.run()
            print(f'Compiled model is at: {self.params.compilation.model_packaged_path}', flush=True)
            with open(self.params.compilation.log_file_path, 'a') as lfp:
                lfp.write('\nSUCCESS: ModelMaker - Compilation completed.')
            #
        #
        return self.params

    def get_params(self):
        return self.params

    def write_status_file(self):
        run_params_file = os.path.join(self.params.common.project_run_path, 'run.yaml')
        utils.write_dict(self.params, run_params_file)
        # create or update the status file
        status_params_file = os.path.join(self.params.common.project_run_path, 'status.yaml')
        status_params = dict()
        if os.path.exists(status_params_file):
            with open(status_params_file) as fp:
                status_params = yaml.safe_load(fp)
            #
        #
        status_params = utils.ConfigDict(status_params)
        # format the run_params to create status_params
        run_params_formatted = copy.deepcopy(self.params)
        run_params_formatted.compilation = {self.params.common.target_device:run_params_formatted.compilation}
        run_params_formatted = utils.ConfigDict(run_params_formatted)
        status_params.update(run_params_formatted)
        utils.write_dict(status_params, status_params_file)
        return run_params_file

    def package_trained_model(self, input_files, tarfile_name):
        tfp = tarfile.open(tarfile_name, 'w:gz', dereference=True)
        for inpf in input_files:
            inpf_list = inpf if isinstance(inpf, (list,tuple)) else [inpf]
            for inpf_entry in inpf_list:
                if inpf_entry is not None and os.path.exists(inpf_entry):
                    outpf = os.path.basename(inpf_entry)
                    tfp.add(inpf_entry, arcname=outpf)
                #
            #
        #
        tfp.close()
        tarfile_size = os.path.getsize(tarfile_name)
        return tarfile_size

    @staticmethod
    def get_training_module_descriptions(*args, **kwargs):
        return descriptions.get_training_module_descriptions(*args, **kwargs)

    @staticmethod
    def get_model_descriptions(*args, **kwargs):
        return descriptions.get_model_descriptions(*args, **kwargs)

    @staticmethod
    def get_model_description(*args, **kwargs):
        return descriptions.get_model_description(*args, **kwargs)

    @staticmethod
    def set_model_description(*args, **kwargs):
        return descriptions.set_model_description(*args, **kwargs)

    @staticmethod
    def get_preset_descriptions(*args, **kwargs):
        return descriptions.get_preset_descriptions(*args, **kwargs)

    @staticmethod
    def get_target_device_descriptions(*args, **kwargs):
        return descriptions.get_target_device_descriptions(*args, **kwargs)

    @staticmethod
    def get_task_descriptions(*args, **kwargs):
        return descriptions.get_task_descriptions(*args, **kwargs)

    @staticmethod
    def get_sample_dataset_descriptions(*args, **kwargs):
        return descriptions.get_sample_dataset_descriptions(*args, **kwargs)

    @staticmethod
    def get_version_descriptions(*args, **kwargs):
        return descriptions.get_version_descriptions(*args, **kwargs)

    @staticmethod
    def get_tooltip_descriptions(*args, **kwargs):
        return descriptions.get_tooltip_descriptions(*args, **kwargs)

    @staticmethod
    def get_help_descriptions(*args, **kwargs):
        return descriptions.get_help_descriptions(*args, **kwargs)


class ListNode:
    def __init__(self, x):
        self.val = x
        self.next = None

class Solution:
    def deleteNode(self, node):
        if not node:
            return
        elif not node.next:
            node = None
            return

        node.val = node.next.val
        delNode = node.next
        node.next = delNode.next
        delNode = None

        return

Python/237. Delete Node in a Linked List.py


import datetime as dt
import pytz
import json
import os
from typing import Any, List, Tuple
from operator import attrgetter, itemgetter

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import pandas_datareader as web
import seaborn as sns
from functools import wraps
from dotenv import load_dotenv
from loguru import logger
from telegram import ChatAction
from telegram.ext import CommandHandler, Updater, MessageHandler, Filters, Dispatcher
from telegram.ext.callbackcontext import CallbackContext

from sklearn.preprocessing import MinMaxScaler
from telegram.ext.jobqueue import JobQueue
from tensorflow.keras.layers import LSTM, Dense, Dropout
from tensorflow.keras.models import Sequential

load_dotenv()

DB_FILE_PATH = "db.json"
TEST_PREDICTION_IMAGE_PATH = "test_prediction.png"
CHAT_ID_CRYPTO_INFO = "-698951802"
DB_INITIAL = {
    "config": {
        "crypto_currency": "BTC",
        "against_currency": "USD",
        "prediction_days": 60,
    }
}
TZ = pytz.timezone("Asia/Singapore")

def tg_error_handler(update, context: CallbackContext) -> None:
    """
    Error handler for telegram bot.
    See https://github.com/python-telegram-bot/python-telegram-bot/blob/master/examples/errorhandlerbot.py
    """

    logger.error("Exception occurred while handling an update:", exc_info=context.error)

def tg_all_handler(update, context: CallbackContext):
    logger.info("Chat ID: {}", update.effective_chat.id)
    logger.info("Received message: {}", update.effective_message.text)

# def start(update, context):
#     context.bot.send_message(
#         chat_id=update.effective_chat.id, text="I'm a bot, please talk to me!"
#     )

def send_typing_action(func):
    """Sends typing action while processing func command."""

    @wraps(func)
    def command_func(update, context, *args, **kwargs):
        context.bot.send_chat_action(
            chat_id=update.effective_message.chat_id, action=ChatAction.TYPING
        )
        return func(update, context, *args, **kwargs)

    return command_func

@logger.catch
def load_db() -> Any:
    with open(DB_FILE_PATH, "r") as db_file:
        return json.load(db_file)

@logger.catch
def save_db(db: Any) -> None:
    with open(DB_FILE_PATH, "w") as db_file:
        json.dump(db, db_file)

@logger.catch
@send_typing_action
def set_config(update, context: CallbackContext):
    logger.info("User requested setting config")

    db = load_db()
    if context.args is None or context.args == []:
        context.bot.send_message(
            chat_id=update.effective_chat.id,
            text="ERROR: Setting name not specified.",
        )
        return
    args = context.args

    if args[0] in db["config"]:
        db["config"][args[0]] = args[1]
        save_db(db)

        # success
        context.bot.send_message(
            chat_id=update.effective_chat.id,
            text=f"Successfully set {args[0]} to {args[1]}",
        )
        logger.info("Setting config successful")
    else:
        context.bot.send_message(
            chat_id=update.effective_chat.id,
            text="Please provide a valid setting name!",
        )

@logger.catch
def get_data(
    crypto_currency, against_currency, prediction_days
) -> Tuple[Any, Any, Any, MinMaxScaler]:
    """
    Fetch data from sources using DataReader and fit them into a feature range of 0 to 1.
    """

    start = dt.datetime(2016, 1, 1)
    end = dt.datetime.now()
    data = web.DataReader(f"{crypto_currency}-{against_currency}", "yahoo", start, end)
    logger.info("Yahoo finance data:\n{}", data.head())  # show first few rows of data

    # Prepare data
    scaler = MinMaxScaler(feature_range=(0, 1))
    scaled_data = scaler.fit_transform(data["Close"].values.reshape(-1, 1))

    x_train, y_train = [], []

    for x in range(prediction_days, len(scaled_data)):
        x_train.append(scaled_data[x - prediction_days : x, 0])
        y_train.append(scaled_data[x, 0])

    x_train, y_train = np.array(x_train), np.array(y_train)
    x_train = np.reshape(x_train, (x_train.shape[0], x_train.shape[1], 1))

    logger.debug("x_train: {}", x_train)
    logger.debug("y_train: {}", y_train)

    return x_train, y_train, data, scaler

@logger.catch
def create_neural_net(x_train, y_train) -> Sequential:
    """
    Create the model that is used to predict from the data
    """

    model = Sequential()
    model.add(LSTM(units=50, return_sequences=True, input_shape=(x_train.shape[1], 1)))
    model.add(Dropout(0.2))
    model.add(LSTM(units=50, return_sequences=True))
    model.add(Dropout(0.2))
    model.add(LSTM(units=50))
    model.add(Dropout(0.2))
    model.add(Dense(units=1))

    model.compile(optimizer="adam", loss="mean_squared_error")
    model.fit(x_train, y_train, epochs=25, batch_size=32)

    return model

@logger.catch
def test_model(
    crypto_currency: str,
    against_currency: str,
    data,
    prediction_days,
    scaler: MinMaxScaler,
    model: Sequential,
):
    test_start = dt.datetime(2020, 1, 1)
    test_end = dt.datetime.now()

    test_data = web.DataReader(
        f"{crypto_currency}-{against_currency}", "yahoo", test_start, test_end
    )
    actual_prices = test_data["Close"].values

    total_dataset = pd.concat((data["Close"], test_data["Close"]), axis=0)

    global model_inputs
    model_inputs = total_dataset[
        len(total_dataset) - len(test_data) - prediction_days :
    ].values
    model_inputs = model_inputs.reshape(-1, 1)  # type: ignore
    model_inputs = scaler.fit_transform(model_inputs)

    x_test = []

    for x in range(prediction_days, len(model_inputs)):
        x_test.append(model_inputs[x - prediction_days : x, 0])

    x_test = np.array(x_test)
    x_test = np.reshape(x_test, (x_test.shape[0], x_test.shape[1], 1))

    prediction_prices = model.predict(x_test)
    prediction_prices = scaler.inverse_transform(prediction_prices)

    sns.set_style("whitegrid")  # darkgrid, whitegrid, dark, white and ticks
    plt.rc("axes", titlesize=18)  # fontsize of the axes title
    plt.rc("axes", labelsize=14)  # fontsize of the x and y labels
    plt.rc("xtick", labelsize=13)  # fontsize of the tick labels
    plt.rc("ytick", labelsize=13)  # fontsize of the tick labels
    plt.rc("legend", fontsize=13)  # legend fontsize
    plt.rc("font", size=13)  # controls default text sizes

    plt.plot(actual_prices, color="black", label="Actual Prices")
    plt.plot(prediction_prices, color="green", label="Predicted Prices")
    plt.title(f"{crypto_currency} price prediction")
    plt.xlabel("Time")
    plt.ylabel("Price")
    plt.legend(loc="upper left")
    plt.savefig(TEST_PREDICTION_IMAGE_PATH)
    plt.clf()

@logger.catch
def predict_next_day(
    prediction_days: int, model: Sequential, scaler: MinMaxScaler
) -> float:
    real_data = [
        model_inputs[len(model_inputs) - prediction_days : len(model_inputs) + 1, 0]
    ]
    real_data = np.array(real_data)
    real_data = np.reshape(real_data, (real_data.shape[0], real_data.shape[1], 1))

    prediction = model.predict(real_data)
    prediction = scaler.inverse_transform(prediction)

    return prediction[0][0]

@logger.catch
@send_typing_action
def crypto_prediction(update, context: CallbackContext):
    db = load_db()
    logger.debug("loaded DB!")
    logger.debug("DB: {}", db)

    crypto_currency, against_currency, prediction_days = itemgetter(
        "crypto_currency", "against_currency", "prediction_days"
    )(db["config"])

    context.bot.send_message(
        chat_id=update.effective_chat.id,
        text=f"Predicting crypto prices for {crypto_currency} based on the past {prediction_days} days.",
    )

    # run the predictions
    x_train, y_train, data, scaler = get_data(
        crypto_currency, against_currency, prediction_days
    )
    model = create_neural_net(x_train, y_train)
    test_model(crypto_currency, against_currency, data, prediction_days, scaler, model)
    prediction = predict_next_day(prediction_days, model, scaler)

    logger.info("Prediction: {}", prediction)

    context.bot.send_photo(
        chat_id=update.effective_chat.id,
        photo=open(TEST_PREDICTION_IMAGE_PATH, "rb"),
    )
    context.bot.send_message(
        chat_id=update.effective_chat.id,
        text=f"Tomorrow's {crypto_currency} price is predicted to be at ${prediction}.",
    )
    os.remove(TEST_PREDICTION_IMAGE_PATH)

@send_typing_action
@logger.catch
def send_crypto_update(context: CallbackContext):
    logger.info("Sending crypto update...")

    context.bot.send_message(
        chat_id=CHAT_ID_CRYPTO_INFO,
        text="Good morning! Here's your crypto update for today:",
    )
    tg_update = {"effective_chat": {"id": CHAT_ID_CRYPTO_INFO}}
    crypto_prediction(tg_update, context)

@logger.catch
def main():
    updater = Updater(token=os.getenv("TELEGRAM_BOT_TOKEN"), use_context=True)
    dispatcher: Dispatcher = updater.dispatcher
    job_queue: JobQueue = updater.job_queue

    if not os.path.exists(DB_FILE_PATH):
        save_db(DB_INITIAL)

    # dispatcher.add_handler(CommandHandler("start", start))
    dispatcher.add_handler(CommandHandler("predictcrypto", crypto_prediction))
    dispatcher.add_handler(CommandHandler("set", set_config))

    dispatcher.add_handler(MessageHandler(Filters.all, tg_all_handler), group=1)
    dispatcher.add_error_handler(tg_error_handler)

    job_queue.run_daily(
        send_crypto_update,
        time=dt.time(hour=7, minute=00, second=00, tzinfo=TZ),
    )
    logger.info(
        "TZ: {}",
        dt.time(hour=00, minute=00, second=00, tzinfo=TZ).tzinfo,
    )

    updater.start_polling()
    logger.info("Bot started polling!")
    updater.idle()

if __name__ == "__main__":
    main()


import asyncio
import copy
import json
import re
import uuid
from datetime import datetime

from async_generator import aclosing
from jupyterhub.utils import maybe_future
from jupyterhub.utils import url_path_join
from traitlets import Callable
from traitlets import Dict
from traitlets import Union

from .backendspawner import BackendSpawner

class EventBackendSpawner(BackendSpawner):
    _yielded_events = []
    _cancel_event_yielded = False
    latest_events = []
    events = {}
    yield_wait_seconds = 1

    def get_state(self):
        """get the current state"""
        state = super().get_state()
        if self.events:
            if type(self.events) != dict:
                self.events = {}
            self.events["latest"] = self.latest_events
            # Clear logs older than 24h or empty logs
            events_keys = copy.deepcopy(list(self.events.keys()))
            for key in events_keys:
                value = self.events.get(key, None)
                if value and len(value) > 0 and value[0]:
                    stime = self._get_event_time(value[0])
                    dtime = datetime.strptime(stime, "%Y_%m_%d %H:%M:%S")
                    now = datetime.now()
                    delta = now - dtime
                    if delta.days:
                        del self.events[key]
                else:  # empty logs
                    del self.events[key]
            state["events"] = self.events
        return state

    def load_state(self, state):
        """load state from the database"""
        super().load_state(state)
        if "events" in state:
            self.events = state["events"]
            if "latest" in self.events:
                self.latest_events = self.events["latest"]

    def clear_state(self):
        """clear any state (called after shutdown)"""
        super().clear_state()
        self._cancel_event_yielded = False

    async def _generate_progress(self):
        """Private wrapper of progress generator

        This method is always an async generator and will always yield at least one event.
        """
        if not self._spawn_pending:
            self.log.warning(
                "Spawn not pending, can't generate progress for %s", self._log_name
            )
            return

        async with aclosing(self.progress()) as progress:
            async for event in progress:
                yield event

    async def progress(self):
        spawn_future = self._spawn_future
        next_event = 0

        break_while_loop = False
        while True:
            # Ensure we always capture events following the start_future
            # signal has fired.
            if spawn_future.done():
                break_while_loop = True

            len_events = len(self.latest_events)
            if next_event < len_events:
                for i in range(next_event, len_events):
                    yield self.latest_events[i]
                    if self.latest_events[i].get("failed", False) == True:
                        self._cancel_event_yielded = True
                        break_while_loop = True
                next_event = len_events

            if break_while_loop:
                break
            await asyncio.sleep(self.yield_wait_seconds)

    def status_update_url(self, server_name=""):
        """API path for status update endpoint for a server with a given name"""
        url_parts = ["users", "progress", "status", self.user.escaped_name]
        if server_name:
            url_parts.append(server_name)
        return url_path_join(*url_parts)

    @property
    def _status_update_url(self):
        return self.status_update_url(self.name)

    def get_env(self):
        env = super().get_env()
        env["JUPYTERHUB_STATUS_URL"] = self._status_update_url
        return env

    def _get_event_time(self, event):
        # Regex for date time
        pattern = re.compile(
            r"([0-9]+(_[0-9]+)+).*[0-9]{2}:[0-9]{2}:[0-9]{2}(\\.[0-9]{1,3})?"
        )
        message = event["html_message"]
        match = re.search(pattern, message)
        return match.group()

    cancelling_event = Union(
        [Dict(), Callable()],
        default_value={
            "failed": False,
            "ready": False,
            "progress": 99,
            "message": "",
            "html_message": "JupyterLab is cancelling the start.",
        },
        help="""
        Event shown when single-user server was cancelled.
        """,
    ).tag(config=True)

    async def get_cancelling_event(self):
        if callable(self.cancelling_event):
            cancelling_event = await maybe_future(self.cancelling_event(self))
        else:
            cancelling_event = self.cancelling_event
        return cancelling_event

    stop_event = Union(
        [Dict(), Callable()],
        default_value={
            "failed": True,
            "ready": False,
            "progress": 100,
            "message": "",
            "html_message": "JupyterLab was stopped.",
        },
        help="""
        Event shown when single-user server was stopped.
        """,
    ).tag(config=True)

    async def get_stop_event(self):
        if callable(self.stop_event):
            stop_event = await maybe_future(self.stop_event(self))
        else:
            stop_event = self.stop_event
        return stop_event

    async def run_failed_spawn_request_hook(self, exception):
        now = datetime.now().strftime("%Y_%m_%d %H:%M:%S.%f")[:-3]
        event = {
            "progress": 99,
            "failed": False,
            "html_message": f"{now}: JupyterLab start failed. Deleting related resources...This may take a few seconds.",
        }
        self.latest_events.append(event)
        # Ensure that we're waiting 2*yield_wait_seconds, so that
        # events will be shown to the spawn-pending page.
        await asyncio.sleep(2*self.yield_wait_seconds)

        summary = "Unknown Error"
        details = ""
        if hasattr(exception, "args") and len(exception.args) > 2:
            try:
                error = json.loads(exception.args[2].body.decode())
                if "error" in error.keys() and "detailed_error" in error.keys():
                    summary = error.get("error")
                    details = error.get("detailed_error")
                else:
                    summary = f"{exception.args[0]} - {exception.args[1]}"
                    details = str(error)
            except:
                self.log.exception("Could not load detailed error message")
                summary = f"{exception.args[0]} - {exception.args[1]}"
        elif hasattr(exception, "args") and len(exception.args) > 1:
            summary = f"{exception.args[0]} - {exception.args[1]}"

        async def _get_stop_event(spawner):
            now = datetime.now().strftime("%Y_%m_%d %H:%M:%S.%f")[:-3]
            event = {
                "progress": 100,
                "failed": True,
                "html_message": f"{now}: {summary}{details}",
            }
            return event

        self.stop_event = _get_stop_event
        await super().run_failed_spawn_request_hook(exception)

    def run_pre_spawn_hook(self):
        """Some commands are required."""
        if self.already_stopped:
            raise Exception("Server is in the process of stopping, please wait.")

        self.start_id = uuid.uuid4().hex[:8]
        if self.internal_ssl:
            service_address = self.get_service_address()
            self.ssl_alt_names += [f"DNS:{service_address}"]

        # Save latest events with start event time
        if self.latest_events != []:
            try:
                start_event = self.latest_events[0]
                start_event_time = self._get_event_time(start_event)
                self.events[start_event_time] = self.latest_events
            except:
                self.log.info(
                    f"Could not retrieve latest_events. Reset events list for {self._log_name}"
                )
                self.latest_events = []
                self.events = {}
        self.latest_events = []
        if type(self.events) != dict:
            self.events = {}
        self.events["latest"] = self.latest_events

        now = datetime.now().strftime("%Y_%m_%d %H:%M:%S.%f")[:-3]
        start_pre_msg = "Sending request to backend service to start your service."
        start_event = {
            "failed": False,
            "progress": 10,
            "html_message": f"{now}: {start_pre_msg}\
                  Start ID: {self.start_id}
  Options:

{json.dumps(self.user_options, indent=2)}
",
        }
        self.latest_events = [start_event]

        """Run the pre_spawn_hook if defined"""
        if self.pre_spawn_hook:
            return self.pre_spawn_hook(self)

    def run_post_spawn_request_hook(self, resp_json):
        now = datetime.now().strftime("%Y_%m_%d %H:%M:%S.%f")[:-3]
        submitted_event = {
            "failed": False,
            "ready": False,
            "progress": 30,
            "html_message": f"{now}: Backend communication successful.You will receive further information about the service status from the service itself.",
        }
        self.latest_events.append(submitted_event)
        return self.post_spawn_request_hook(self, resp_json)

    async def stop(self, now=False, cancel=False, event=None):
        if self.already_stopped:
            return

        if cancel:
            cancelling_event = await self.get_cancelling_event()
            self.latest_events.append(cancelling_event)

        try:
            # always use cancel=False, and call the function later if neccessary.
            # Otherwise the stop_event would be attached after run_post_stop_hook was called.
            await super().stop(now, cancel=False)
        finally:
            if not event:
                event = await self.get_stop_event()
            elif callable(event):
                event = await maybe_future(event(self))
            self.latest_events.append(event)

        if cancel:
            await self.cancel()


import sys
sys.stdin = open('input.txt', 'r')

q = int(input())

for test_case in range(1, q+1):
    numbers = input()
    cnt = 0
    guest = 0
    print('#{} '.format(test_case), end='')
    guest = int(numbers[0])
    for i in range(1,len(numbers)):
        #print(i,int(numbers[i]), guest+cnt)
        if i > guest + cnt:
            cnt += 1
        if i <= guest + cnt:
            guest += int(numbers[i])

    print(cnt)

SW-Expert-Academy/D3/4789. 성공적인 공연 기획.py

4789. 성공적인 공연 기획.py


from pathlib import Path
import pytest
import time
import json
import platform

from usb_audio_test_utils import (
    check_analyzer_output,
    get_xtag_dut,
    XrunDut,
    XsigInput,
)
from conftest import list_configs, get_config_features

def OS_uncollect(features, board, config):
    if (
        platform.system() == "Darwin"
        and board == "xk_316_mc"
        and "2AMi2o2xxxxxx" in config
    ):
        # macOS defaults to the 16-bit audio profile on this config and xsig and portaudio are not forcing a change in bit depth
        return True
    return False

def loopback_dac_uncollect(pytestconfig, board, config):
    features = get_config_features(board, config)
    xtag_id = get_xtag_dut(pytestconfig, board)
    if not xtag_id:
        # XTAGs not present
        return True
    if OS_uncollect(features, board, config):
        return True
    if not features["i2s_loopback"]:
        return True
    if not features["analogue_o"]:
        # No output channels
        return True
    return False

def analogue_duration(level, partial):
    if level == "weekend":
        duration = 90 if partial else 1200
    elif level == "nightly":
        duration = 15 if partial else 180
    else:
        duration = 10
    return duration

@pytest.mark.uncollect_if(func=loopback_dac_uncollect)
@pytest.mark.parametrize(["board", "config"], list_configs())
def test_loopback_dac(pytestconfig, board, config):
    features = get_config_features(board, config)

    xsig_config = f'mc_i2s_loopback_{features["analogue_o"]}ch'
    if board == "xk_316_mc" and features["tdm8"]:
        xsig_config = "mc_i2s_loopback_2ch"
    xsig_config_path = Path(__file__).parent / "xsig_configs" / f"{xsig_config}.json"

    adapter_dut = get_xtag_dut(pytestconfig, board)
    duration = analogue_duration(pytestconfig.getoption("level"), features["partial"])
    fail_str = ""

    with XrunDut(adapter_dut, board, config) as dut:
        for fs in features["samp_freqs"]:
            # Issue 120
            if (
                platform.system() == "Windows"
                and board == "xk_316_mc"
                and "winbuiltin" in config
                and fs in [44100, 48000]
            ):
                continue
            with XsigInput(fs, duration, xsig_config_path, dut.dev_name) as xsig_proc:
                time.sleep(duration + 6)
                xsig_lines = xsig_proc.get_output()
            with open(xsig_config_path) as file:
                xsig_json = json.load(file)
            failures = check_analyzer_output(xsig_lines, xsig_json["in"])
            if len(failures) > 0:
                fail_str += f"Failure at sample rate {fs}\n"
                fail_str += "\n".join(failures) + "\n\n"
                fail_str += f"xsig stdout at sample rate {fs}\n"
                fail_str += "\n".join(xsig_lines) + "\n\n"

    if len(fail_str) > 0:
        pytest.fail(fail_str)


import cv2
import numpy as np
import math

cap = cv2.VideoCapture(0)

def nothing():
    pass

cv2.namedWindow('original')
cv2.createTrackbar('h_min', 'original', 0, 180, nothing)
cv2.createTrackbar('h_max', 'original', 0, 180, nothing)
cv2.createTrackbar('s_min', 'original', 0, 255, nothing)
cv2.createTrackbar('s_max', 'original', 0, 255, nothing)
cv2.createTrackbar('v_min', 'original', 0, 255, nothing)
cv2.createTrackbar('v_max', 'original', 0, 255, nothing)
cv2.createTrackbar('area', 'original', 0, 1500, nothing)

cv2.setTrackbarPos('h_min', 'original', 0)
cv2.setTrackbarPos('h_max', 'original', 180)
cv2.setTrackbarPos('s_min', 'original', 54)
cv2.setTrackbarPos('s_max', 'original', 255)
cv2.setTrackbarPos('v_min', 'original', 5)
cv2.setTrackbarPos('v_max', 'original', 253)

while True:
    # flag, image_origin = cap.read()
    image_origin = cv2.imread('Снимок.PNG')
    image_hsv = cv2.cvtColor(image_origin, cv2.COLOR_BGR2HSV)
    h_min = cv2.getTrackbarPos('h_min', 'original')
    h_max = cv2.getTrackbarPos('h_max', 'original')
    s_min = cv2.getTrackbarPos('s_min', 'original')
    s_max = cv2.getTrackbarPos('s_max', 'original')
    v_min = cv2.getTrackbarPos('v_min', 'original')
    v_max = cv2.getTrackbarPos('v_max', 'original')
    image_blue = cv2.inRange(image_hsv, (h_min, s_min, v_min), (h_max, s_max, v_max))
    # edges = cv2.Canny(image_blue, 10, 250)
    contours, hierarchy = cv2.findContours(image_blue, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)

    # cnt = max(contours, key=cv2.contourArea)
    if len(contours) != 0:
        for cnt in contours:
            rect = cv2.minAreaRect(cnt)
            box = cv2.boxPoints(rect)
            box = np.int0(box)
            area = int(rect[1][0] * rect[1][1])
            center = int(rect[0][0]), int(rect[0][1])
            area_bar = cv2.getTrackbarPos('area', 'original')

            edge_one = np.int0((box[1][0] - box[0][0], box[1][1] - box[0][1]))
            edge_two = np.int0((box[2][0] - box[1][0], box[2][1] - box[1][1]))

            used_edge = edge_one
            if cv2.norm(edge_two) > cv2.norm(edge_one):
                used_edge = edge_two
            ref = (1, 0)
            angle = math.degrees(
                math.acos((ref[0] * used_edge[0] + ref[1] * used_edge[1]) / (cv2.norm(ref) * cv2.norm(used_edge))))

            if area > area_bar:
                cv2.drawContours(image_origin, [box], 0, (255, 255, 0), 2)
                cv2.circle(image_origin, center, 5, (0, 255, 255), 2)
                cv2.putText(image_origin, str(angle), (center[0] + 20, center[1] - 20), cv2.FONT_HERSHEY_SCRIPT_SIMPLEX,
                            1, (0, 0, 255), 2)
            center_image = image_origin.shape[1] // 2, image_origin.shape[0] // 2
            print(center_image[0], center[0])

            if center_image[0] > center[0]:
                print('left')
            elif center_image[0] < center[0]:
                print('right')

    cv2.imshow('original', image_origin)
    # cv2.imshow('image blue', image_blue)
    # cv2.imshow('edges', edges)
    if cv2.waitKey(5) == 27:
        break

cap.release()
cv2.destroyAllWindows()


from math import cos, radians
import matplotlib.pyplot as plt
import numpy as np

def finger_path(phase):
    """
    If you plot this function it is a graph of the motion of each finger
    phase: 0-259, phase the finger is in
    returns:
        angle: offset angle from finger's step start position
        z: height offset from globe plane
    """
    stepHeight = 40 # mm

    if 0 <= phase <= 215:
        angle = phase/215
        z = 0
    elif 215 < phase < 360:
        angle = -phase/145 + 72/29
        z = stepHeight/2*cos(radians(phase-215)/0.4) - stepHeight/2
    else:
        raise ValueError('Input must be 0-259')

    return angle, z

x = list(range(360))
y1 = []
y2 = []
for val in x:
    angleOffset, zOffset = finger_path(val)
    y1.append(angleOffset)
    y2.append(zOffset)

plt.xlim(0, 360)
plt.ylim(-40, 0)
plt.plot(x, y2, 'bo')
plt.show()


import librosa

import joblib

import numpy as np
import pandas as pd

from typing import List

# Load the StandardScaler used during training
scaler = joblib.load("./resources/standard_scaler_pytorch_model_last.pkl")

def audio_to_csv(audio) -> List[pd.DataFrame]:
    dfs = []
    segments = get_3sec_sample(audio)

    for audio in segments:
        # Perform audio feature extraction
        features = audio_pipeline(audio)

        # Scale the features using the loaded scaler
        scaled_features = scaler.transform([features])

        # Create a DataFrame
        column_names = [
            "chroma_stft_mean",
            "chroma_stft_var",
            "rms_mean",
            "rms_var",
            "spectral_centroid_mean",
            "spectral_centroid_var",
            "spectral_bandwidth_mean",
            "spectral_bandwidth_var",
            "rolloff_mean",
            "rolloff_var",
            "zero_crossing_rate_mean",
            "zero_crossing_rate_var",
            "harmony_mean",
            "harmony_var",
            "tempo",
            "mfcc1_mean",
            "mfcc1_var",
            "mfcc2_mean",
            "mfcc2_var",
            "mfcc3_mean",
            "mfcc3_var",
            "mfcc4_mean",
            "mfcc4_var",
            "mfcc5_mean",
            "mfcc5_var",
            "mfcc6_mean",
            "mfcc6_var",
            "mfcc7_mean",
            "mfcc7_var",
            "mfcc8_mean",
            "mfcc8_var",
            "mfcc9_mean",
            "mfcc9_var",
            "mfcc10_mean",
            "mfcc10_var",
            "mfcc11_mean",
            "mfcc11_var",
            "mfcc12_mean",
            "mfcc12_var",
            "mfcc13_mean",
            "mfcc13_var",
            "mfcc14_mean",
            "mfcc14_var",
            "mfcc15_mean",
            "mfcc15_var",
            "mfcc16_mean",
            "mfcc16_var",
            "mfcc17_mean",
            "mfcc17_var",
            "mfcc18_mean",
            "mfcc18_var",
            "mfcc19_mean",
            "mfcc19_var",
            "mfcc20_mean",
            "mfcc20_var",
        ]

        df = pd.DataFrame(scaled_features, columns=column_names)
        dfs.append(df)

    return dfs

def audio_pipeline(audio):
    features = []

    # Chromagram
    chroma_stft = librosa.feature.chroma_stft(y=audio)
    features.append(np.mean(chroma_stft))
    features.append(np.var(chroma_stft))  # var => variance

    # RMS (Root Mean Square value for each frame)
    rms = librosa.feature.rms(y=audio)
    features.append(np.mean(rms))
    features.append(np.var(rms))

    # Calcul du Spectral centroid
    spectral_centroids = librosa.feature.spectral_centroid(y=audio)
    features.append(np.mean(spectral_centroids))
    features.append(np.var(spectral_centroids))

    # Spectral bandwith
    spectral_bandwidth = librosa.feature.spectral_bandwidth(y=audio)
    features.append(np.mean(spectral_bandwidth))
    features.append(np.var(spectral_bandwidth))

    # Calcul du spectral rolloff point
    rolloff = librosa.feature.spectral_rolloff(y=audio)
    features.append(np.mean(rolloff))
    features.append(np.var(rolloff))

    # Calcul du ZCR (Zero Crossing Rate)
    zcr = librosa.zero_crossings(audio)
    # features.append(np.sum(zcr))  # Custom
    features.append(np.mean(zcr))
    features.append(np.var(zcr))

    # Harmonic
    harmony = librosa.effects.harmonic(y=audio)
    features.append(np.mean(harmony))
    features.append(np.var(harmony))

    # Tempo
    tempo = librosa.feature.tempo(y=audio)
    features.append(tempo[0])

    # Calcul des moyennes des MFCC
    mfcc = librosa.feature.mfcc(y=audio)
    for x in mfcc:
        features.append(np.mean(x))
        features.append(np.var(x))

    return features

def get_3sec_sample(audio) -> list:
    # TODO Vérif
    # TODO Rewrite
    audio, sample_rate = librosa.load(audio, sr=None)

    segment_duration = 3  # Durée de chaque segment en secondes
    segment_length = int(sample_rate * segment_duration)
    segments = []

    # Effectuez la prédiction toutes les 3 secondes
    for i in range(0, len(audio), segment_length):
        segment = audio[i : i + segment_length]
        segments.append(segment)

    return segments


import time
import requests
import pandas as pd
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.firefox.options import Options
import json
import os

print("Opening web browser ")

url = "https://www.fifa.com/fifa-world-ranking/men"
option = Options()
option.headless = True
driver = webdriver.Firefox(options=option)

print("Connecting... ")

driver.get(url)
time.sleep(10)

element = driver.find_element_by_xpath(
    "//div[@class='fc-ranking-overview_cardsContainer__1w_fo']//table")
html_content = element.get_attribute('outerHTML')

print("Done! ")

# Parse HTML content with BeautifulSoup
soup = BeautifulSoup(html_content, 'html.parser')
table = soup.find(name='table')

# Structuring content in a Dataframe with Pandas
df_full = pd.read_html(str(table))[0].head(50)
df_part = pd.read_html(str(table))[0].head(10)
df50 = df_full[['RK', 'Team', 'Total PointsPTS', 'Previous Points', '+/-']]
df10 = df_part[['RK', 'Team', 'Total PointsPTS', 'Previous Points', '+/-']]

# Create a dictionary with the collected data
top10FIFA = {}
top50FIFA = {}
top10FIFA['ranking'] = df10.to_dict('records')
top50FIFA['ranking'] = df50.to_dict('records')

# Convert and save file in JSON format
top = json.dumps(top50FIFA)
fp = open('top50FIFA.json', 'w')
fp.write(top)

top = json.dumps(top10FIFA)
fp = open('top10FIFA.json', 'w')
fp.write(top)

fp.close()

print("top50FIFA.json and top10FIFA.json were created \n")

print("Closing web browser ")
driver.quit()

os.system('clear')  # Clean console in Linux
os.system('CLS')  # Clean console in Windows

print("\n Top 10 -- Ranking FIFA")
print(df10)
print("\n Top 50 -- Ranking FIFA")
print(df50)


import json
from typing import TYPE_CHECKING, Optional, Iterable

from boxsdk.object.base_object import BaseObject
from ..pagination.marker_based_object_collection import MarkerBasedObjectCollection
from ..util.api_call_decorator import api_call

if TYPE_CHECKING:
    from boxsdk.object.user import User
    from boxsdk.object.task_assignment import TaskAssignment
    from boxsdk.pagination.box_object_collection import BoxObjectCollection

class Task(BaseObject):
    """Represents a Box task."""
    _item_type = 'task'

    @api_call
    def assign(self, assignee: 'User') -> 'TaskAssignment':
        """
        Assign a task to a single user on a single file.

        :param assignee:
            The :class:`User` to assign the task to.
        :returns:
            A task assignment object.
        """
        url = self._session.get_url('task_assignments')
        body = {
            'task': {
                'type': 'task',
                'id': self.object_id,
            },
            'assign_to': {
                'id': assignee.object_id,
            },
        }
        response = self._session.post(url, data=json.dumps(body)).json()
        return self.translator.translate(
            session=self._session,
            response_object=response,
        )

    @api_call
    def assign_with_login(self, assignee_login: Optional[str]) -> 'TaskAssignment':
        """
        Used to assign a task to a single user with the login email address of the assignee.

        :param assignee_login:
            The login of the user to assign the task to.
        :returns:
            A task assignment object.
        """
        url = self._session.get_url('task_assignments')
        body = {
            'task': {
                'type': 'task',
                'id': self.object_id,
            },
            'assign_to': {
                'login': assignee_login,
            },
        }
        response = self._session.post(url, data=json.dumps(body)).json()
        return self.translator.translate(
            session=self._session,
            response_object=response,
        )

    @api_call
    def get_assignments(self, fields: Iterable[str] = None) -> 'BoxObjectCollection':
        """
        Get the entries in the file task assignment.

        :param fields:
            List of fields to request.
        :returns:
            An iterator of the entries in the file task assignment.
        """
        return MarkerBasedObjectCollection(
            session=self._session,
            url=self.get_url('assignments'),
            limit=None,
            marker=None,
            fields=fields,
            return_full_pages=False,
        )


import cv2
import numpy as np
from IMP_SD import computeHOGs,get_svm_detector

if __name__ == '__main__':
    # 第一步计算HOG特征
    gradien_list = []
    labels = []
    hard_neg_list = []

    # 正样本以及label导入
    #  pos_num, gradien_list_pos = computeHOGs('C:\\Users\\SLJ\\Desktop\\OCR_Project\\sample_R')
    pos_num, gradien_list_pos = computeHOGs('C:\\Users\\sunlj4\\Desktop\\OCR_Project_HOG+SVM\\sample_RR')
    [labels.append(+1) for _ in range(pos_num)]
    # 负样本以及label导入
    neg_num , gradien_list_neg = computeHOGs('C:\\Users\\sunlj4\\Desktop\\OCR_Project_HOG+SVM\\sample_FF')
    [labels.append(-1) for _ in range(neg_num)]
    gradien_list = gradien_list_pos + gradien_list_neg

    # 第二部：创建并且训练SVM
    svm = cv2.ml.SVM_create()
    svm.setCoef0(0)
    svm.setCoef0(0.0)
    svm.setDegree(3)
    criteria = (cv2.TERM_CRITERIA_MAX_ITER + cv2.TERM_CRITERIA_EPS, 1000, 1e-3)
    svm.setTermCriteria(criteria)
    svm.setGamma(0)
    svm.setKernel(cv2.ml.SVM_LINEAR)
    svm.setNu(0.5)
    svm.setP(0.1)  # for EPSILON_SVR, epsilon in loss function?
    svm.setC(0.01)  # From paper, soft classifier
    svm.setType(cv2.ml.SVM_EPS_SVR)  # C_SVC # EPSILON_SVR # may be also NU_SVR # do regression task
    svm.train(np.array(gradien_list), cv2.ml.ROW_SAMPLE, np.array(labels))

    # 第三部，加入识别错误样本，进行第二轮识别

    # 第四部：保存训练结果
    winSize = (24, 32)
    blockSize = (8, 8)
    blockStride = (4, 4)
    cellSize = (4, 4)
    nbins = 9
    hog = cv2.HOGDescriptor(winSize, blockSize, blockStride, cellSize, nbins)
    hog.setSVMDetector(get_svm_detector(svm))

    # 直接测试
    img = cv2.imread('C:\\Users\\sunlj4\\Desktop\\OCR_Project_HOG+SVM\\958.bmp', cv2.IMREAD_GRAYSCALE)
    # 选择ROI
    x = 560
    y = 432
    height = 216
    width = 800
    ROI = img[x:x + height, y:y + width]
    # 用自适应阈值方法
    constValue = 8
    blockSize = 13
    maxVal = 255
    adaptiveMethod = 0
    thresholdType = 1
    auto_thre_Image = cv2.adaptiveThreshold(ROI, maxVal, adaptiveMethod, thresholdType, blockSize, constValue)
    thre_img_inv = cv2.bitwise_not(auto_thre_Image)
    cv2.imshow('auto_thre_Image', auto_thre_Image)
    cv2.waitKey(0)
    # 寻找区域
    rects, wei = hog.detectMultiScale(auto_thre_Image, winStride = (4, 4), padding = (0, 0), scale = 1.05)
    for (x, y, w, h) in rects:
        cv2.rectangle(ROI, (x, y), (x + w, y + h), (0, 0, 255), 2)
    cv2.imshow('a', ROI)
    cv2.waitKey(0)

    debug = 9


class Node:
    def __init__(self, value):
        self.value = value
        self.next = None
        self.previous = None

class LinkedList:
    def __init__(self):
        self.head = None
        self.tail = None

    def __str__(self):
        if self.isEmpty():
            return "Empty"
        cur, s = self.head, str(self.head.value) + " "
        while cur.next != None:
            s += str(cur.next.value) + " "
            cur = cur.next
        return s

    def reverse(self):
        if self.isEmpty():
            return "Empty"
        cur,s = self.tail, str(self.tail.value) + " "
        while cur.previous != None:
            #print('test '+str(cur.value))
            s += str(cur.previous.value) + " "
            cur = cur.previous
        return s

    def isEmpty(self):
        return self.head == None

    def append(self, item):
        newnode = Node(value=item)
        last = self.head
        newnode.next = None

        if self.head is None:
          #  self.head.previous = None
            self.head = newnode
            self.tail = newnode
            return

        while (last.next is not None):
            last = last.next

        last.next = newnode
        newnode.previous = last
        self.tail = newnode

    def addHead(self, item):

        newnode = Node(value=item)
        newnode.next = self.head
        newnode.previous = None

        if self.head is not None:
            self.head.previous = newnode

        if self.head is None:
            self.tail = newnode

        self.head = newnode

    def insert(self, pos, item):
        if pos==0:
            self.addHead(item)
            return
        elif pos<0:
            pos = self.size()+pos
            if pos<0:
                self.addHead(item)
                return
        elif pos==self.size() or self.isEmpty() or pos>=self.size():
            self.append(item)
            return

        count=0

        newnode = Node(value=item)

        itr = self.head
        while itr:
            if count == pos-1:
                itr.next.previous = newnode
                newnode.next = itr.next
                newnode.previous = itr
                itr.next = newnode

                break

            itr = itr.next
            count +=1

    def replace(self,pos,item):
        count=0
        newnode = Node(value=item)
        itr = self.head
        while itr:
            if count == pos-1:
                itr.value = newnode.value
                break
            itr = itr.next
            count +=1

    def search(self, item):
        itr = self.head
        while itr:
            if itr.value == item:
                return 'Found'
            itr = itr.next
        return 'Not Found'

    def index(self, item):
        itr = self.head
        count = 0
        while itr:
            if itr.value == item:
                return count
            itr = itr.next
            count+=1
        return -1

    def size(self):
        size = 0
        itr = self.head
        while itr:
            itr = itr.next
            size+=1
        #print(size)
        return size

    def pop(self, pos):
        if pos<0 or self.size()<=pos:
            return 'Out of Range'

        count=0
        itr = self.head
        while itr:
            if pos==0:
                self.head = self.head.next
                return 'Success'
            elif count == pos-1:
                itr.next = itr.next.next
                return 'Success'
            itr = itr.next
            count+=1

    def print(self):
        if self.head is None:
            print('This linked list is empty')
            return

        itr = self.head
        liststr = ''

        while itr:
            #print(itr)
            if itr.next is None:
                liststr += str(itr.value)
            else:
                liststr += str(itr.value) + ' <- '
            itr = itr.next

        print(liststr)

    def findmax(self):
        maxNum = self.head.value
        itr = self.head
        while itr:
            if itr.value > maxNum:
                maxNum = itr.value
            itr = itr.next

        print(maxNum)
        return maxNum

    def removeAt(self,index):
        if index==0:
            self.head = self.head.next

        count=0
        itr = self.head
        while itr:
            if count == index-1:
                itr.next = itr.next.next
                break
            itr = itr.next
            count+=1

    def radixsort(self):
        box = []
        for i in range(10):
            box.append(LinkedList(0))
        print("------------------------------------------------------------")
        stop = False
        counter = 0
        for i in range(self.max_length+1):
            print("Round : " + str(i+1))
            while not self.isEmpty():
                temp = self.removeAt(0)[::-1]
                try:
                    box[int(temp[i])].addHead(temp[::-1])
                    print(temp[i],end=' from ')
                    print(temp[::-1])
                except:
                    box[0].addHead(temp[::-1])
            if box[0].size() == self.max:
                stop = True
            for j in range(10):
                box[j].sort()
                print(str(j) + ' : ' + box[j].print())
                while not box[j].isEmpty():
                    temp = box[j].addHead()
                    self.append(temp)
            print("------------------------------------------------------------")
            counter += 1
            if stop:
                break
        self.max_length = counter

# def countingSort(arr, exp1):

#     n = arr.size()
#     # output = []
#     # count = []

#     # for i in range(n):
#     #     output.append(0)

#     # for j in range(10):
#     #     count.append(0)

#     output = [0] * (n)
#     count = [0] * (10)

#     itr = arr.head
#     while itr:
#         index = int(itr.value)//exp1
#         count[index%10] +=1
#         itr = itr.next

#     for k in range(1,10):
#         count[k] += count[k-1]

#     itr = arr.head
#     i = n - 1
#     while i>=0:
#         index = int(itr.value)//exp1
#         output[count[index%10]-1] = int(itr.value)
#         count[index%10]-=1
#         i-=1
#         itr = itr.next

#     LLoutput = LinkedList()
#     for element in output:
#         LLoutput.append(int(element))

#     arr = LLoutput
#     arr.print()
#     return arr

# def radixsort(arr):

#     maxNum = arr.findmax()

#     divide = 1
#     while maxNum / divide >= 1:
#         countingSort(arr, divide)
#         divide *= 10

L =LinkedList()
inputNum = [e for e in input('Enter Input : ').split()]
for i in inputNum:
    L.append(int(i))

#L.findmax()
L.radixsort()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Jun 26 08:24:46 2019
"Brown Corpus Analysis"
@author: Singh Gagan Deep
The Brown Corpus was the first million-word electronic corpus of English,
created in 1961 at Brown University.
This corpus contains text from 500 sources,
and the sources have been categorized by genre, such as news, editorial, and so on.
(for a complete list, see http://icame.uib.no/brown/bcm-los.html).

Reference http://www.nltk.org/book/ch02.html
"""

# Importing brown corpus
from nltk.corpus import brown
category=brown.categories()

# function to analysis the various keywords for a specific topic
def brown_words(new):
    new_list=[]
    for words in new:
        new_list.append(words)
    return new_list
total_words=brown_words(new=brown.words(categories='editorial'))

print(len(brown_words(new=brown.words(categories='editorial'))))

set(brown_words(new=brown.words(categories='news')))

print(len(brown_words(new=brown.words(categories='news'))))

print(len(set(total_words)))

'''
There are 61604 words are in "editorial "category from which around 9890 words are unique
And  1000554 total words in " news category" from which around 14394 words are unique
'''
'''
The Brown Corpus is a convenient resource for studying systematic differences between genres,
a kind of linguistic inquiry known as stylistics.
Let's compare genres in their usage of modal verbs.
The first step is to produce the counts for a particular genre.
'''
import nltk
news_text=brown.words(categories='news')
fdist=nltk.FreqDist(w.lower() for w in news_text)
model=['can', 'could','may','might','must','will']
for m in model:
    print(m + ':', fdist[m], end=' ')

'''
we need to obtain counts for each genre of interest.
We'll use NLTK's support for conditional frequency distribution
'''
model1=['When','where','why','is','are']

cfd=nltk.ConditionalFreqDist((genre,word)
for genre in brown.categories() for word in brown.words(categories=genre))
genre=['news','religion','hobbies','science_fiction','romance','humor']
model=['can', 'could','may','might','must','will']
#model1=['When','where','why','is','are']
cfd.tabulate(conditions=genre,samples=model)


import os

def helloworld():
    path = os.path.dirname(__file__)
    print('path: ' + path)
    f = open(path + '/data2.txt')
    # 一行ずつ読み込んで表示する
    for name in f.read().splitlines():
        message = 'Hello ' + name + ' !'
        print(message)

helloworld()


INPUT_FILE = "input"
PART = 2

def is_fully_contained(assignment: str):
        sections = [list(map(int, x.split("-"))) for x in assignment.split(",")]
        return (sections[0][0] - sections[1][0]) * (sections[0][1] - sections[1][1]) <= 0

def has_overlap(assignment: str):
        sections = [list(map(int, x.split("-"))) for x in assignment.split(",")]
        return (sections[0][0] - sections[1][1]) * (sections[0][1] - sections[1][0]) <= 0

if __name__ == "__main__":
    with open(INPUT_FILE)  as input_f:
        assignments = input_f.read().splitlines()

    total = 0
    for assignment in assignments:
        if(PART == 1):
            if is_fully_contained(assignment):
                total += 1
        elif(PART == 2):
            if has_overlap(assignment):
                total += 1

    print(f"Total is: {total}")


from typing import Any, Callable, Iterable, List

from volatility3.framework import interfaces, renderers
from volatility3.framework.configuration import requirements
from volatility3.framework.objects import utility
from volatility3.framework.renderers import format_hints
from volatility3.framework.symbols import intermed
from volatility3.framework.symbols.linux.extensions import elf
from volatility3.plugins.linux import elfs

class PsList(interfaces.plugins.PluginInterface):
    """Lists the processes present in a particular linux memory image."""

    _required_framework_version = (2, 0, 0)

    _version = (2, 1, 0)

    @classmethod
    def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]:
        return [
            requirements.ModuleRequirement(
                name="kernel",
                description="Linux kernel",
                architectures=["Intel32", "Intel64"],
            ),
            requirements.PluginRequirement(
                name="elfs", plugin=elfs.Elfs, version=(2, 0, 0)
            ),
            requirements.ListRequirement(
                name="pid",
                description="Filter on specific process IDs",
                element_type=int,
                optional=True,
            ),
            requirements.BooleanRequirement(
                name="threads",
                description="Include user threads",
                optional=True,
                default=False,
            ),
            requirements.BooleanRequirement(
                name="decorate_comm",
                description="Show `user threads` comm in curly brackets, and `kernel threads` comm in square brackets",
                optional=True,
                default=False,
            ),
            requirements.BooleanRequirement(
                name="dump",
                description="Extract listed processes",
                optional=True,
                default=False,
            ),
        ]

    @classmethod
    def create_pid_filter(cls, pid_list: List[int] = None) -> Callable[[Any], bool]:
        """Constructs a filter function for process IDs.

        Args:
            pid_list: List of process IDs that are acceptable (or None if all are acceptable)

        Returns:
            Function which, when provided a process object, returns True if the process is to be filtered out of the list
        """
        # FIXME: mypy #4973 or #2608
        pid_list = pid_list or []
        filter_list = [x for x in pid_list if x is not None]
        if filter_list:

            def filter_func(x):
                return x.pid not in filter_list

            return filter_func
        else:
            return lambda _: False

    def _generator(
        self,
        pid_filter: Callable[[Any], bool],
        include_threads: bool = False,
        decorate_comm: bool = False,
        dump: bool = False,
    ):
        """Generates the tasks list.

        Args:
            pid_filter: A function which takes a process object and returns True if the process should be ignored/filtered
            include_threads: If True, the output will also show the user threads
                             If False, only the thread group leaders will be shown
                             Defaults to False.
            decorate_comm: If True, it decorates the comm string of
                            - User threads: in curly brackets,
                            - Kernel threads: in square brackets
                           Defaults to False.
            dump: If True, the main executable of the process is written to a file
                  Defaults to False.
        Yields:
            Each rows
        """
        for task in self.list_tasks(
            self.context, self.config["kernel"], pid_filter, include_threads
        ):
            elf_table_name = intermed.IntermediateSymbolTable.create(
                self.context,
                self.config_path,
                "linux",
                "elf",
                class_types=elf.class_types,
            )
            file_output = "Disabled"
            if dump:
                proc_layer_name = task.add_process_layer()
                if not proc_layer_name:
                    continue

                # Find the vma that belongs to the main ELF of the process
                file_output = "Error outputting file"

                for v in task.mm.get_mmap_iter():
                    if v.vm_start == task.mm.start_code:
                        file_handle = elfs.Elfs.elf_dump(
                            self.context,
                            proc_layer_name,
                            elf_table_name,
                            v,
                            task,
                            self.open,
                        )
                        if file_handle:
                            file_output = str(file_handle.preferred_filename)
                            file_handle.close()
                        break

            pid = task.tgid
            tid = task.pid
            ppid = task.parent.tgid if task.parent else 0
            name = utility.array_to_string(task.comm)
            if decorate_comm:
                if task.is_kernel_thread:
                    name = f"[{name}]"
                elif task.is_user_thread:
                    name = f"{{{name}}}"

            yield 0, (
                format_hints.Hex(task.vol.offset),
                pid,
                tid,
                ppid,
                name,
                file_output,
            )

    @classmethod
    def list_tasks(
        cls,
        context: interfaces.context.ContextInterface,
        vmlinux_module_name: str,
        filter_func: Callable[[int], bool] = lambda _: False,
        include_threads: bool = False,
    ) -> Iterable[interfaces.objects.ObjectInterface]:
        """Lists all the tasks in the primary layer.

        Args:
            context: The context to retrieve required elements (layers, symbol tables) from
            vmlinux_module_name: The name of the kernel module on which to operate
            filter_func: A function which takes a process object and returns True if the process should be ignored/filtered
            include_threads: If True, it will also return user threads.
        Yields:
            Task objects
        """
        vmlinux = context.modules[vmlinux_module_name]

        init_task = vmlinux.object_from_symbol(symbol_name="init_task")

        # Note that the init_task itself is not yielded, since "ps" also never shows it.
        for task in init_task.tasks:
            if filter_func(task):
                continue

            yield task

            if include_threads:
                yield from task.get_threads()

    def run(self):
        pids = self.config.get("pid")
        include_threads = self.config.get("threads")
        decorate_comm = self.config.get("decorate_comm")
        dump = self.config.get("dump")
        filter_func = self.create_pid_filter(pids)

        columns = [
            ("OFFSET (V)", format_hints.Hex),
            ("PID", int),
            ("TID", int),
            ("PPID", int),
            ("COMM", str),
            ("File output", str),
        ]
        return renderers.TreeGrid(
            columns, self._generator(filter_func, include_threads, decorate_comm, dump)
        )


import shodan
import socket
from pprint import pprint as pp
import json
from openpyxl import Workbook
from openpyxl import styles
import argparse
import sys
from dotenv import load_dotenv
import os
from tqdm import tqdm

parser = argparse.ArgumentParser(
    prog='shodanscanner',
    description='Simple script to bulk scan IPs on Shodan',
    epilog='made by aerodiduch. https://github.com/aerodiduch'
)

parser.add_argument(
    '-f', '--file', help='File containing IP address to scan. Input must be one IP per line. ',
)

parser.add_argument(
    '-t', '--target', help='Scan a single target, e.g: -t 200.100.20.10'
)

parser.add_argument(
    '-o', '--output', help='Name of the output file without extension. Default value is "results".'
)

def load_data(filename: str) -> list:
    '''Loads data from specified target file

    Args:
        filename (str): File containing hosts data

    Returns:
        list: List contaning hosts
    '''
    hosts = []
    with open(filename, 'r') as fh:
        for line in fh:
            hosts.append(
                line.replace('\n', '')
                )
    return hosts

def save_data(data: list, workbook: Workbook, filename: str):
    '''Saves parsed data to output XLSX file.

    Args:
        data (list): API response from Shodan
        workbook (Workbook): Workbook object to write
        filename (str): Output filename
    '''

    ws = workbook.active

    if 'ports' in data:
        ports = ", ".join([str(port) for port in data['ports']])
    else:
        ports = '----'

    if 'domains' in data:
        domains = ", ".join([str(domain) for domain in data['domains']])
    else:
        domains = '----'

    if 'vulns' in data:
        vulns = ", ".join([str(vuln) for vuln in data['vulns']])
    else:
        vulns = '----'

    font = styles.Font(bold=True)
    headers = ws['A1':'I1']
    for column in headers[0]:
        column.font = font

    ws.append(
        [data['ip_str'], data['isp'], data['asn'], data['city'], ports, '', vulns, data['last_update'], domains]
    )
    workbook.save(f'{filename}.xlsx')

def request_data(api: shodan.Shodan, data: list, output_name: str):
    '''Request data to Shodan API

    Args:
        shodan (shodan.Shodan): Shodan object with API key
        data (list): List of hosts
        output_name (str): Fiilename to write to.
    '''
    print(f'-> Ready to scan {len(data)} hosts...\n')
    failed = []
    for host in tqdm(data):
        try:
            host_info = api.host(host)
            save_data(host_info, wb, output_name)
        except shodan.APIError as e:
            if 'IP' in e.args[0]:
                failed.append(host)
                #print(f'-> No results for {host}')
            continue
    print(f'\n-> Finished scan. Results dumped to "{output_name}.xlsx"')
    if failed:
        print(f'\nNo results found for: {", ".join([i for i in failed])}.')

def set_api_key():
    print('[!!!] No API KEY detected.\n')
    print(
        'This will be prompted only one time to set API KEY.\nA .env file will be created containing it.\nYou will find your Shodan API KEY on https://account.shodan.io/'
    )
    print('\nYou can change it later editing the .env file created on this directory.')
    print('If no valid API KEY is provided, shodanscanner can not make requests through Shodan API.')
    key = input('\n[!] Paste your API KEY: ')

    while len(key) < 30:
        print("\nInvalid API KEY. Shodan's API KEY must have at least 30 characters\n")
        key = input('\n[!] Paste your API KEY: ')

    with open('.env', 'w') as fh:
        fh.write(
            f"API_KEY='{key}'"
        )
    print('API KEY saved succesfully.\n\n')
    return True

if __name__ == '__main__':

    load_dotenv()
    API_KEY = os.getenv('API_KEY')
    if not API_KEY:
        set_api_key()

    api = shodan.Shodan(API_KEY)

    args = parser.parse_args()
    if len(sys.argv) < 2:
        parser.print_help()

    wb = Workbook()
    wb.active.append(
        ['IP', 'ISP', 'ASN', 'LOCATION', 'PORTS', 'PRODUCTS', 'CVEs', 'LAST UPDATED', 'DOMAINS']
    )

    if args.file:
        output_name = args.output if args.output is not None else 'results'
        data = load_data(args.file)
        request_data(api, data, output_name)


# -*- coding: utf-8 -*-
"""
Created on Mon Feb 21 14:18:27 2022

@author: mamun

Face Recognition

Face recognition problems commonly fall into one of two categories:

Face Verification "Is this the claimed person?" For example, at some airports,
you can pass through customs by letting a system scan your passport and then verifying that you
(the person carrying the passport) are the correct person. A mobile phone that unlocks using your
face is also using face verification. This is a 1:1 matching problem.

FaceNet learns a neural network that encodes a face image into a vector of 128 numbers.
By comparing two such vectors, you can then determine if two pictures are of the same person.

In this assignment, we'll do:

# Differentiate between face recognition and face verification
# Implement one-shot learning to solve a face recognition problem
# Apply the triplet loss function to learn a network's parameters in the context of face recognition
# Explain how to pose face recognition as a binary classification problem
# Map face images into 128-dimensional encodings using a pretrained model
# Perform face verification and face recognition with these encodings
# Channels-last notation

For this assignment, we'll be using a pre-trained model which represents ConvNet activations using a
"channels last" convention, as used during the lecture and in previous programming assignments.

In other words, a batch of images will be of shape  (𝑚,𝑛𝐻,𝑛𝑊,𝑛𝐶) .

"""

#%% import packages and utilites
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Conv2D, ZeroPadding2D, Activation, Input, concatenate
from tensorflow.keras.models import Model
from tensorflow.keras.layers import BatchNormalization
from tensorflow.keras.layers import MaxPooling2D, AveragePooling2D
from tensorflow.keras.layers import Concatenate
from tensorflow.keras.layers import Lambda, Flatten, Dense
from tensorflow.keras.initializers import glorot_uniform
from tensorflow.keras.layers import Layer
from tensorflow.keras import backend as K
K.set_image_data_format('channels_last')
import os
import numpy as np
from numpy import genfromtxt
import pandas as pd
import tensorflow as tf
import PIL

print("packages and utilites import completed")

#%% Naive Face Verification
'''
Encoding Face Images into a 128-Dimensional Vector

3.1 - Using a ConvNet to Compute Encodings
The FaceNet model takes a lot of data and a long time to train. So following the common practice in applied deep learning,
we'll load weights that someone else has already trained. The network architecture follows the Inception model from
Szegedy et al.. An Inception network implementation has been provided for you, and you can find it in the file
inception_blocks_v2.py to get a closer look at how it is implemented.

Hot tip: Go to "File->Open..." at the top of this notebook. This opens the file directory that contains the .py file).

The key things we have to be aware of are:

This network uses 160x160 dimensional RGB images as its input.
Specifically, a face image (or batch of  𝑚  face images) as a tensor of shape  (𝑚,𝑛𝐻,𝑛𝑊,𝑛𝐶)=(𝑚,160,160,3)
The input images are originally of shape 96x96, thus, you need to scale them to 160x160.
This is done in the img_to_encoding() function.
The output is a matrix of shape  (𝑚,128)  that encodes each input face image into a 128-dimensional vector
Run the cell below to create the model for face images!

'''
#%%
from tensorflow.keras.models import model_from_json

json_file = open('keras-facenet-h5/model.json', 'r')
loaded_model_json = json_file.read()
json_file.close()
model = model_from_json(loaded_model_json)
model.load_weights('D:/deep_learning_lab/deep_learning/convolutional_neural_network/Week4/Face Recognition/keras-facenet-h5/weights.h5')

#%%
print(model.inputs)
print(model.outputs)

#%%
# example of loading the keras facenet model
from keras.models import load_model

#%% load the model
facenet_model = load_model('D:/deep_learning_lab/deep_learning/convolutional_neural_network/Week4/Face Recognition/keras-facenet-h5/facenet_keras.h5')

#%% summarize input and output shape
print(facenet_model.inputs)
print(facenet_model.outputs)

#%%
'''
By using a 128-neuron fully connected layer as its last layer,
the model ensures that the output is an encoding vector of size 128.

So, an encoding is a good one if:

The encodings of two images of the same person are quite similar to each other.
The encodings of two images of different persons are very different.
The triplet loss function formalizes this, and tries to "push" the encodings of two images of the same person (Anchor and Positive) closer together, while "pulling" the encodings of two images of different persons (Anchor, Negative) further apart
'''

#%% The Triplet Loss

'''
The Triplet Loss

**Important Note**:
    the triplet loss is the main ingredient of the face recognition algorithm,
    and you'll need to know how to use it for training your own FaceNet model,
    as well as other types of image similarity problems.
    Therefore, you'll implement it below, for fun and edification. :)
'''
#%%
def triplet_loss(y_true, y_pred, alpha = 0.2):
    """
    Implementation of the triplet loss as defined by formula (3)

    Arguments:
    y_true -- true labels, required when you define a loss in Keras, you don't need it in this function.
    y_pred -- python list containing three objects:
            anchor -- the encodings for the anchor images, of shape (None, 128)
            positive -- the encodings for the positive images, of shape (None, 128)
            negative -- the encodings for the negative images, of shape (None, 128)

    Returns:
    loss -- real number, value of the loss
    """
    anchor, positive, negative = y_pred[0], y_pred[1], y_pred[2]

    # Step 1: Compute the (encoding) distance between the anchor and the positive
    pos_dist = tf.reduce_sum(tf.square(tf.subtract(anchor, positive)), axis = -1)
    # Step 2: Compute the (encoding) distance between the anchor and the negative
    neg_dist = tf.reduce_sum(tf.square(tf.subtract(anchor, negative)), axis = -1)
    # Step 3: subtract the two previous distances and add alpha.
    basic_loss = tf.add(tf.subtract(pos_dist, neg_dist), alpha)
    # Step 4: Take the maximum of basic_loss and 0.0. Sum over the training examples.
    loss = tf.reduce_sum(tf.maximum(basic_loss, 0))

    return loss

#%% test triplet_loss
tf.random.set_seed(1)
y_true = (None, None, None) # It is not used
y_pred = (tf.keras.backend.random_normal([3, 128], mean=6, stddev=0.1, seed = 1),
          tf.keras.backend.random_normal([3, 128], mean=1, stddev=1, seed = 1),
          tf.keras.backend.random_normal([3, 128], mean=3, stddev=4, seed = 1))
loss = triplet_loss(y_true, y_pred)

# assert type(loss) == tf.python.framework.ops.EagerTensor, "Use tensorflow functions"
# print("loss = " + str(loss))

y_pred_perfect = ([1., 1.], [1., 1.], [1., 1.,])
loss = triplet_loss(y_true, y_pred_perfect, 5)
assert loss == 5, "Wrong value. Did you add the alpha to basic_loss?"
y_pred_perfect = ([1., 1.],[1., 1.], [0., 0.,])
loss = triplet_loss(y_true, y_pred_perfect, 3)
assert loss == 1., "Wrong value. Check that pos_dist = 0 and neg_dist = 2 in this example"
y_pred_perfect = ([1., 1.],[0., 0.], [1., 1.,])
loss = triplet_loss(y_true, y_pred_perfect, 0)
assert loss == 2., "Wrong value. Check that pos_dist = 2 and neg_dist = 0 in this example"
y_pred_perfect = ([0., 0.],[0., 0.], [0., 0.,])
loss = triplet_loss(y_true, y_pred_perfect, -2)
assert loss == 0, "Wrong value. Are you taking the maximum between basic_loss and 0?"
y_pred_perfect = ([[1., 0.], [1., 0.]],[[1., 0.], [1., 0.]], [[0., 1.], [0., 1.]])
loss = triplet_loss(y_true, y_pred_perfect, 3)
assert loss == 2., "Wrong value. Are you applying tf.reduce_sum to get the loss?"

#%% Loading the Pre-trained Model
'''
FaceNet is trained by minimizing the triplet loss. But since training requires a lot of data
and a lot of computation, we won't train it from scratch here. Instead, we'll load a previously
trained model in the following cell; which might take a couple of minutes to run.

'''
FRmodel = model

#%% Applying the Model
'''
We're building a system for an office building where the building manager would like to offer facial
recognition to allow the employees to enter the building.

We'd like to build a face verification system that gives access to a list of people.
To be admitted, each person has to swipe an identification card at the entrance.
The face recognition system then verifies that they are who they claim to be
'''

#%% Face Verification
'''
Now we'll build a database containing one encoding vector for each person who is allowed to enter the office.
To generate the encoding, we'll use img_to_encoding(image_path, model),
which runs the forward propagation of the model on the specified image.

Run the following code to build the database (represented as a Python dictionary).
This database maps each person's name to a 128-dimensional encoding of their face.

'''

def img_to_encoding(image_path, model):
    img = tf.keras.preprocessing.image.load_img(image_path, target_size=(160, 160))
    img = np.around(np.array(img) / 255.0, decimals=12)
    x_train = np.expand_dims(img, axis=0)
    embedding = model.predict_on_batch(x_train)
    return embedding / np.linalg.norm(embedding, ord=2)

#%% test img_to_encoding
database = {}
database["danielle"] = img_to_encoding("D:/deep_learning_lab/deep_learning/convolutional_neural_network/Week4/Face Recognition/images/danielle.png", FRmodel)
database["younes"] = img_to_encoding("D:/deep_learning_lab/deep_learning/convolutional_neural_network/Week4/Face Recognition/images/younes.jpg", FRmodel)
database["tian"] = img_to_encoding("D:/deep_learning_lab/deep_learning/convolutional_neural_network/Week4/Face Recognition/images/tian.jpg", FRmodel)
database["andrew"] = img_to_encoding("D:/deep_learning_lab/deep_learning/convolutional_neural_network/Week4/Face Recognition/images/andrew.jpg", FRmodel)
database["kian"] = img_to_encoding("D:/deep_learning_lab/deep_learning/convolutional_neural_network/Week4/Face Recognition/images/kian.jpg", FRmodel)
database["dan"] = img_to_encoding("D:/deep_learning_lab/deep_learning/convolutional_neural_network/Week4/Face Recognition/images/dan.jpg", FRmodel)
database["sebastiano"] = img_to_encoding("D:/deep_learning_lab/deep_learning/convolutional_neural_network/Week4/Face Recognition/images/sebastiano.jpg", FRmodel)
database["bertrand"] = img_to_encoding("D:/deep_learning_lab/deep_learning/convolutional_neural_network/Week4/Face Recognition/images/bertrand.jpg", FRmodel)
database["kevin"] = img_to_encoding("D:/deep_learning_lab/deep_learning/convolutional_neural_network/Week4/Face Recognition/images/kevin.jpg", FRmodel)
database["felix"] = img_to_encoding("D:/deep_learning_lab/deep_learning/convolutional_neural_network/Week4/Face Recognition/images/felix.jpg", FRmodel)
database["benoit"] = img_to_encoding("D:/deep_learning_lab/deep_learning/convolutional_neural_network/Week4/Face Recognition/images/benoit.jpg", FRmodel)
database["arnaud"] = img_to_encoding("D:/deep_learning_lab/deep_learning/convolutional_neural_network/Week4/Face Recognition/images/arnaud.jpg", FRmodel)

#%% Load the images of Danielle and Kian:
danielle = tf.keras.preprocessing.image.load_img("D:/deep_learning_lab/deep_learning/convolutional_neural_network/Week4/Face Recognition/images/danielle.png",
                                                 target_size=(160, 160))
kian = tf.keras.preprocessing.image.load_img("D:/deep_learning_lab/deep_learning/convolutional_neural_network/Week4/Face Recognition/images/kian.jpg",
                                             target_size=(160, 160))
#%%
np.around(np.array(kian) / 255.0, decimals=12).shape

#%% print images
kian

#%%
np.around(np.array(danielle) / 255.0, decimals=12).shape
#%%
danielle

#%% verify
'''

Implement the verify() function, which checks if the front-door camera picture (image_path)
is actually the person called "identity". You will have to go through the following steps:

Compute the encoding of the image from image_path.
Compute the distance between this encoding and the encoding of the identity image stored in the database.
Open the door if the distance is less than 0.7, else do not open it.
As presented above, you should use the L2 distance np.linalg.norm.

Note: In this implementation, compare the L2 distance, not the square of the L2 distance, to the threshold 0.7.

Hints:

identity is a string that is also a key in the database dictionary.
img_to_encoding has two parameters: the image_path and model
'''

def verify(image_path, identity, database, model):
    """
    Function that verifies if the person on the "image_path" image is "identity".

    Arguments:
        image_path -- path to an image
        identity -- string, name of the person you'd like to verify the identity. Has to be an employee who works in the office.
        database -- python dictionary mapping names of allowed people's names (strings) to their encodings (vectors).
        model -- your Inception model instance in Keras

    Returns:
        dist -- distance between the image_path and the image of "identity" in the database.
        door_open -- True, if the door should open. False otherwise.
    """

    # Step 1: Compute the encoding for the image. Use img_to_encoding() see example above. (≈ 1 line)
    encoding = img_to_encoding(image_path, model)

    # Step 2: Compute distance with identity's image (≈ 1 line)
    dist = np.linalg.norm(encoding - database[identity])

    # Step 3: Open the door if dist < 0.7, else don't open (≈ 3 lines)
    if dist < 0.7:
        print("It's " + str(identity) + ", welcome in!")
        door_open = True
    else:
        print("It's not " + str(identity) + ", please go away")
        door_open = False

    return dist, door_open

#%%
assert(np.allclose(verify("images/camera_1.jpg", "bertrand", database, FRmodel), (0.54364836, True)))
assert(np.allclose(verify("images/camera_3.jpg", "bertrand", database, FRmodel), (0.38616243, True)))
assert(np.allclose(verify("images/camera_1.jpg", "younes", database, FRmodel), (1.3963861, False)))
assert(np.allclose(verify("images/camera_3.jpg", "younes", database, FRmodel), (1.3872949, False)))
# verify("images/camera_0.jpg", "younes", database, FRmodel)

#%%
verify("D:/deep_learning_lab/deep_learning/convolutional_neural_network/Week4/Face Recognition/images/camera_2.jpg", "kian", database, FRmodel)

#%% Face Recognition
'''
face verification system is mostly working.
But since Kian got his ID card stolen, when he came back to the office the next day he couldn't get in!

To solve this, we'd like to change your face verification system to a face recognition system.
This way, no one has to carry an ID card anymore.
An authorized person can just walk up to the building, and the door will unlock for them!

We'll implement a face recognition system that takes as input an image,
and figures out if it is one of the authorized persons (and if so, who).
Unlike the previous face verification system, we will no longer get a person's name as one of the inputs.

'''

#%%  who_is_it
'''

Implement who_is_it() with the following steps:

Compute the target encoding of the image from image_path
Find the encoding from the database that has smallest distance with the target encoding.
Initialize the min_dist variable to a large enough number (100).
This helps you keep track of the closest encoding to the input's encoding.
Loop over the database dictionary's names and encodings.
To loop use for (name, db_enc) in database.items().
Compute the L2 distance between the target "encoding" and the current "encoding" from the database.
If this distance is less than the min_dist, then set min_dist to dist, and identity to name.

'''
def who_is_it(image_path, database, model):
    """
    Implements face recognition for the office by finding who is the person on the image_path image.

    Arguments:
        image_path -- path to an image
        database -- database containing image encodings along with the name of the person on the image
        model -- your Inception model instance in Keras

    Returns:
        min_dist -- the minimum distance between image_path encoding and the encodings from the database
        identity -- string, the name prediction for the person on image_path
    """

    ## Step 1: Compute the target "encoding" for the image. Use img_to_encoding() see example above. ## (≈ 1 line)
    encoding = img_to_encoding(image_path, model)

    ## Step 2: Find the closest encoding ##

    # Initialize "min_dist" to a large value, say 100 (≈1 line)
    min_dist = 100

    # Loop over the database dictionary's names and encodings.
    for (name, db_enc) in database.items():

        # Compute L2 distance between the target "encoding" and the current db_enc from the database. (≈ 1 line)
        dist = np.linalg.norm(encoding - database[name])

        # If this distance is less than the min_dist, then set min_dist to dist, and identity to name. (≈ 3 lines)
        if dist < min_dist:
            min_dist = dist
            identity = name

    if min_dist > 0.7:
        print("Not in the database.")
    else:
        print ("it's " + str(identity) + ", the distance is " + str(min_dist))

    return min_dist, identity

#%% # Test 1 with Younes pictures
who_is_it("D:/deep_learning_lab/deep_learning/convolutional_neural_network/Week4/Face Recognition/images/camera_0.jpg", database, FRmodel)

# Test 2 with Younes pictures
test1 = who_is_it("D:/deep_learning_lab/deep_learning/convolutional_neural_network/Week4/Face Recognition/images/camera_0.jpg", database, FRmodel)
assert np.isclose(test1[0], 0.5992946)
assert test1[1] == 'younes'

# Test 3 with Younes pictures
test2 = who_is_it("D:/deep_learning_lab/deep_learning/convolutional_neural_network/Week4/Face Recognition/images/younes.jpg", database, FRmodel)
assert np.isclose(test2[0], 0.0)
assert test2[1] == 'younes'


import etl
import boto3
import io

ny_url = "data/ETL_data.csv"
jh_url = "data/ETL_recovery.csv"

ACCESS_KEY_ID = 'YOUR_ACCESS_KEY_ID'
SECRET_ACCESS_KEY = 'YOUR_SECRET_ACCESS_ID'

filename = 'MyPandasData.'
bucketName = 'mypyfile'
s3_client = boto3.client(
    "s3",
    aws_access_key_id=ACCESS_KEY_ID,
    aws_secret_access_key=SECRET_ACCESS_KEY,

)

df_final = etl.extract_transform(ny_url, jh_url)
with io.StringIO() as csv_buffer:
    df_final.to_csv(csv_buffer, index=False)

    response = s3_client.put_object(
        Bucket=bucketName, Key="files/covid_final.csv", Body=csv_buffer.getvalue()
    )

    status = response.get("ResponseMetadata", {}).get("HTTPStatusCode")

    if status == 200:
        print(f"Successful S3 put_object response. Status - {status}")
    else:
        print(f"Unsuccessful S3 put_object response. Status - {status}")


'''
    -1, 0, 1 을 제외한 숫자 두개씩 묶는다
    음수는 음수끼리, 양수는 양수끼리 묶는다
    절댓값이 큰 두 수를 곱해야 크기가 커진다.

    -가 하나 남고 -1 이 있을떄는 곱해서 +를 더해줌
    -가 하나 남고 0 이 있을때는 곱해서 -를 없애줌
    ** 이 조건은 -1,0 도 포함

    ====

    답안을 보고 정리
    양수와 음수, 0을 구분하여 우선순위 큐에 저장

    양수
        두개씩 묶으면 더한 것보다 커짐
        1이 포함된 경우는 더하는 것이 큼
    음수
        절대값이 큰 두개를 묶으면 양수가 되어 커짐

'''

import sys, heapq

if __name__ == "__main__":
    N = int(input())
    posHeap = []
    negHeap = []
    tmpHeap = []
    sum_val = 0

    for _ in range(N):
        num = int(sys.stdin.readline().rstrip())
        if num < -1 :
            heapq.heappush(negHeap, num)
        elif num > 1 :
            heapq.heappush(posHeap, -num)
        else:
            heapq.heappush(tmpHeap, num)

    # print(posHeap)
    # print(negHeap)
    # print(tmpHeap)

    while len(posHeap) > 0:
        if len(posHeap) == 1:
            sum_val += -(heapq.heappop(posHeap))
        else:
            sum_val += heapq.heappop(posHeap) * heapq.heappop(posHeap)

    while len(negHeap) > 0:
        # -1은 음수와 곱해 양수로 만들어 더해줌
        if len(negHeap) == 1:
            if len(tmpHeap) > 0 and tmpHeap[0] == -1:
                heapq.heappop(tmpHeap)
                sum_val += -(heapq.heappop(negHeap))
            elif len(tmpHeap) > 0 and tmpHeap[0] == 0:
                heapq.heappop(tmpHeap)
                heapq.heappop(negHeap)
            else:
                sum_val += heapq.heappop(negHeap)
        else:
            sum_val += heapq.heappop(negHeap) * heapq.heappop(negHeap)

    while len(tmpHeap) > 0:
        if tmpHeap.count(-1) >= 2:
            heapq.heappop(tmpHeap)
            heapq.heappop(tmpHeap)
            sum_val += 1
        elif tmpHeap.count(-1) == 1 and tmpHeap.count(0) > 0:
            heapq.heappop(tmpHeap)
            heapq.heappop(tmpHeap)
        else:
            sum_val += heapq.heappop(tmpHeap)

    print(sum_val)


# Definition for a  binary tree node
class TreeNode(object):
    def __init__(self, x):
        self.val = x
        self.left = None
        self.right = None

class Solution(object):
    # @param root, a tree node
    # @return a boolean
    def isSymmetric(self, root):
        if root is None:
            return True
        stack = []
        stack.append(root.left)
        stack.append(root.right)

        while stack:
            p, q = stack.pop(), stack.pop()

            if p is None and q is None:
                continue

            if p is None or q is None or p.val != q.val:
                return False

            stack.append(p.left)
            stack.append(q.right)

            stack.append(p.right)
            stack.append(q.left)

        return True

if __name__ == "__main__":
    root = TreeNode(1)
    root.left, root.right = TreeNode(2), TreeNode(2)
    root.left.left, root.right.right = TreeNode(3), TreeNode(3)
    root.left.right, root.right.left = TreeNode(4), TreeNode(4)
    print(Solution().isSymmetric(root))