import importlib
from pathlib import Path, PurePath
import glob
import os
import logging

import numpy as np
from pydub import AudioSegment
import pandas as pd
import cv2

# TODO: General solution
# TODO: Can string_filtering be called by generate_filenames?
# TODO: output only benign file?
# TODO: think about input output design of generate_filenames
# def generate_filename_list(path, file_key, dir_key='', only_filename=False):
#     input_paths, gt_paths = [], []
#     for root, dirs, files in os.walk(path):
#         for f in files:
#             if not only_filename:
#                 fullpath = os.path.join(root, f)
#             else:
#                 fullpath = f
#             if dir_key in fullpath:
#                 if file_key in fullpath:
#                     gt_paths.append(fullpath)
#                 else:
#                     input_paths.append(fullpath)
#     return input_paths, gt_paths

def get_melspec_from_cpp(wav_list_path, out_dir, sampling_rate=None):
    # Generate output path list (wav -> csv)
    out_path_list = []
    wav_list_dir, wav_list_filename = os.path.split(wav_list_path)
    with open(wav_list_path, 'r') as fw:
        in_path_list = fw.readlines()
    in_path_list = [f if f[-3:] == 'wav' else f[:-1] for f in in_path_list]

    csv_out_dir = os.path.join(out_dir, 'csv', wav_list_filename[:-4])
    os.makedirs(csv_out_dir, exist_ok=True)
    data_pair = {}
    for in_path in in_path_list:
        in_dir, in_file = os.path.split(in_path)
        out_file = in_file.replace('wav', 'csv')
        out_path = os.path.join(csv_out_dir, out_file)
        out_path_list.append(out_path)

        key = out_file[:-4]
        # label = int(os.path.split(in_dir)[1])
        label = 0
        data_pair[key] = {'label': label}

    csv_list_path = os.path.join(csv_out_dir, wav_list_filename)
    with open(csv_list_path, 'w+') as fw:
        for path in out_path_list:
            fw.write(f'{path}\n')

    # Cpp MelSpectrogram
    exe_file = Path(r'compute-mfcc.exe')
    inputlist = Path(wav_list_path)
    outputlist = Path(csv_list_path)
    command = (
        f'{exe_file.as_posix()} '
        f'--inputlist "{inputlist.as_posix()}" '
        f'--outputlist "{outputlist.as_posix()}" '
    )
    if sampling_rate is not None:
        command += f'--samplingrate {sampling_rate} '
    os.system(command)

    # Save CPP feature (csv -> npy)
    # csv_list = glob.glob(os.path.join(csv_out_dir, '*.csv'))
    csv_list = out_path_list
    img_out_dir = os.path.join(out_dir, 'img', wav_list_filename[:-4])
    os.makedirs(img_out_dir, exist_ok=True)
    img_save_paths = []
    for idx, csv_f in enumerate(csv_list):
        # if idx>10:break
        _, filename = os.path.split(csv_f)
        try:
            df = pd.read_csv(csv_f)
            # XXX: wrong data extraction for temp using
            # df = pd.read_csv(csv_f, header=None)
            data = df.to_numpy().T
        except pd.errors.EmptyDataError:
            print(f'- Empty pandas data {csv_f}')
            data = np.zeros(1)

        save_path = os.path.join(img_out_dir, filename.replace('csv', 'npy'))
        img_save_paths.append(save_path)
        data_pair[filename[:-4]]['path'] = save_path
        np.save(save_path, data)

    with open(os.path.join(out_dir, wav_list_filename), 'w+') as fw:
        fw.writelines(img_save_paths)

    inputs, paths, labels = [], [], []
    for k, v in data_pair.items():
        inputs.append(k[:-4])
        if 'path' in v:
            paths.append(v['path'])
        else:
            paths.append('')
        if 'label' in v:
            labels.append(v['label'])
        else:
            labels.append('')
        # labels.append(v['label'])
    df = pd.DataFrame({
        'input': inputs, 'img_path': paths, 'label': labels
    })
    df.to_csv(os.path.join(out_dir, wav_list_filename).replace('txt', 'csv'))

def get_dir_list(data_path, full_path=True):
    dir_list = np.array([], dtype=object)
    for f in os.listdir(data_path):
        folder_path = os.path.join(data_path, f)
        if os.path.isdir(folder_path):
            if full_path:
                dir_list = np.append(dir_list, folder_path)
            else:
                dir_list = np.append(dir_list, os.path.split(folder_path)[1])
    return list(dir_list)

def get_clips_from_audio(y, clip_time, hop_time):
    clips = []
    if y.duration_seconds >= clip_time:
        for t in range(0, int(y.duration_seconds)-clip_time+1, hop_time):
            start_t, end_t = t, t+clip_time
            clip = y[1000*start_t:1000*end_t]
            clips.append(clip)
    return clips

def continuous_split(path, clip_time, hop_time, sr, channels, add_volume, output_format='wav'):
    assert clip_time > 0 and hop_time > 0
    file_name = os.path.basename(path)
    file_format = file_name.split('.')[1]
    y = load_audio_waveform(path, file_format, sr, channels)
    save_path = os.path.join(os.path.split(path)[0], f'clips_{clip_time}_{hop_time}_{add_volume}dB')
    if not os.path.isdir(save_path):
        os.makedirs(save_path)

    y += add_volume
    clips = get_clips_from_audio(y, clip_time, hop_time)
    for idx, clip in enumerate(clips, 1):
        clip.export(os.path.join(save_path, file_name.replace(f'.{file_format}', f'_{idx:03d}.{output_format}')), output_format)
    print('Finish spitting.')
    return save_path

def save_aLL_files_name(path, name='file_names', keyword=[], filtering_mode='in', is_fullpath=True, shuffle=True, save_path=None):
    # file_names = get_file_names(path, keyword, filtering_mode, is_fullpath, shuffle)
    file_names = get_files(path, keys=keyword, return_fullpath=True, sort=True)
    if not save_path: save_path = path
    save_content_in_txt(
        file_names, os.path.join(save_path, f'{name}.txt'), filter_bank=[], access_mode='w+', dir=None)
    # with open(os.path.join(save_path, f'{name}.txt'), 'w+') as fw:
    #     for f in file_names:
    #         fw.write(f)
    #         fw.write('\n')

def get_ASUS_snoring_index(save_path, split, balancing=True):
    # TODO: Multiple class (miner)
    # Load and check dataset
    subject_list = []
    data_pair = {}
    train_pairs, valid_pairs = [], []
    df = pd.read_csv(save_path)

    for index, row in df.iterrows():
        pair = (row['path'], row['label'])
        if row['subject'] in data_pair:
            data_pair[row['subject']].append(pair)
        else:
            subject_list.append(row['subject'])
            data_pair[row['subject']] = [pair]
        # TODO: balancing

    # indexing
    if balancing:
        sample_count = {}
        for subject in subject_list:
            for sample in data_pair[subject]:
                if sample[1] in sample_count[subject]:
                    sample_count[subject][sample[1]] += 1
                else:
                    sample_count[subject][sample[1]] = 1

    else:
        sample_count = []
        for subject in subject_list:
            sample_count.append(len(data_pair[subject]))
        sorted_idx = np.argsort(np.array(sample_count))
        subject_list = np.take(subject_list, sorted_idx)
        # TODO: subject_list[::3]
        valid_subjects = subject_list[::3]
        train_subjects = list(set(subject_list)-set(valid_subjects))

        for subject in train_subjects:
            train_pairs.extend(data_pair[subject])

        for subject in valid_subjects:
            valid_pairs.extend(data_pair[subject])

    return train_pairs, valid_pairs

def load_input_data(config):
    assert isinstance(config.dataset.data_path, dict)
    total_train, total_valid = [], []
    for key in config.dataset.data_path:
        if key == 'ASUS_snoring':
            save_path = save_ASUS_snoring_index(config.dataset.data_path[key])
            train, valid = get_ASUS_snoring_index(save_path)
        elif key == 'esc-50':
            pass
        elif key == 'Kaggle_snoring':
            pass
        total_train.extend(train)
        total_valid.extend(valid)

    return total_train, total_valid

    return datasets_indexed

def get_pydub_sound(filename, audio_format, sr=None, channels=None):
    """
    Get Pydub sound object and set the basic params
    """
    # sound = AudioSegment.from_file(filename, 'mp4')
    sound = AudioSegment.from_file(filename, audio_format)
    if sr: sound = sound.set_frame_rate(sr)
    if channels: sound = sound.set_channels(channels)
    return sound

def load_audio_waveform(filename, audio_format, sr=None, channels=None):
    """
    Pydub based waveform loading function
    """
    sound = get_pydub_sound(filename, audio_format, sr, channels)
    waveform = np.array(sound.get_array_of_samples(), np.float32)
    sr = sound.frame_rate
    return waveform, sr

def get_files(path, keys=[], return_fullpath=True, sort=True, sorting_key=None):
    """Get all the file name under the given path with assigned keys
    Args:
        path: (str)
        keys: (list, str)
        return_fullpath: (bool)
        sort: (bool)
        sorting_key: (func)
    Return:
        file_list: (list)
    """
    file_list = []
    assert isinstance(keys, (list, str))
    if isinstance(keys, str): keys = [keys]
    # Rmove repeated keys
    keys = list(set(keys))

    def push_into_filelist(root, f, file_list, is_fullpath):
        if is_fullpath:
            file_list.append(os.path.join(root, f))
        else:
            file_list.append(f)

    for i, (root, dirs, files) in enumerate(os.walk(path)):
        for j, f in enumerate(files):
            if keys:
                for key in keys:
                    if key in f:
                        push_into_filelist(root, f, file_list, return_fullpath)
            else:
                push_into_filelist(root, f, file_list, return_fullpath)

    if file_list:
        if sort: file_list.sort(key=sorting_key)
    else:
        if keys:
            logging.warning(f'No file exist with key {keys}.')
        else:
            logging.warning(f'No file exist.')
    return file_list

def generate_filenames(path, keys=None, include_or_exclude=None, is_fullpath=True, loading_formats=None):
    """Get all the file name under the given path with assigned keys and including condition"""
    # TODO: index error when keys=['a','a','a'] include_or_exclude=['include', 'exclude', 'exclude']
    if len(keys) == 0: keys = None
    if len(include_or_exclude) == 0: include_or_exclude = None
    if keys and include_or_exclude:
        assert len(keys) == len(include_or_exclude)
        full_keys = [f'{condition}_{k}' for k, condition in zip(keys, include_or_exclude)]
        # TODO: logging instead of print for repeat key (think about raise error or not)
        if len(full_keys) != len(list(set(full_keys))):
            print('Warning: Repeat keys')
            full_keys = list(set(full_keys))
    if keys:
        filenames = {k: [] for k in full_keys}
    else:
        filenames = []

    for root, dirs, files in os.walk(path):
        for f in files:
            if loading_formats:
                process = False
                for format in loading_formats:
                    if format in f:
                        process = True
                        break
            else:
                process = True

            if process:
                if is_fullpath:
                    final_path = os.path.join(root, f)
                else:
                    final_path = f
                if keys:
                    for idx, k in enumerate(keys):
                        if include_or_exclude[idx] == 'include':
                            if k in final_path:
                                filenames[full_keys[idx]].append(final_path)
                        elif include_or_exclude[idx] == 'exclude':
                            if k not in final_path:
                                filenames[full_keys[idx]].append(final_path)
                        else:
                            raise ValueError('Unknown key condition')
                else:
                    filenames.append(final_path)
    return filenames

def get_class(class_name, modules):
    for module in modules:
        m = importlib.import_module(module)
        clazz = getattr(m, class_name, None)
        if clazz:
            return clazz
    raise RuntimeError(f'Unsupported dataset class: {class_name}')

def load_content_from_txt(path, access_mode='r') -> list:
    if isinstance(path, PurePath):
        path = str(path)
    with open(path, access_mode) as fw:
        content = fw.read().splitlines()
    return content

def inspect_data_split(data_split):
    # TODO: if split in 0.725 0.275
    train_split = data_split.get('train', 0)
    valid_split = data_split.get('valid', 0)
    test_split = data_split.get('test', 0)
    if train_split+valid_split+test_split != 1:
        raise ValueError('Incorrect splitting of dataset.')
    split_code = f'{10*train_split:.0f}{10*valid_split:.0f}{10*test_split:.0f}'
    return split_code

def get_data_path(data_path, index_root, data_split, keywords=[]):
    # TODO: save in csv
    # TODO: save with label
    split_code = inspect_data_split(data_split)
    dataset_name = os.path.split(data_path)[1]

    # Check if an index already exists, create one if not.
    index_path = os.path.join(index_root, dataset_name)
    if not os.path.isdir(index_path):
        os.mkdir(index_path)
        file_path_list = get_files(data_path, keys=keywords, return_fullpath=True, sort=True)
        train_split = int(data_split.get('train', 0)*len(file_path_list))
        valid_split = int(data_split.get('valid', 0)*len(file_path_list))
        test_split = int(data_split.get('test', 0)*len(file_path_list))

        train_path_list = file_path_list[:train_split]
        valid_path_list = file_path_list[train_split:train_split+valid_split]
        test_path_list = file_path_list[train_split+valid_split:train_split+valid_split+test_split]

        train_path = os.path.join(index_path, f'{dataset_name}_train_{split_code}.txt')
        valid_path = os.path.join(index_path, f'{dataset_name}_valid_{split_code}.txt')
        test_path = os.path.join(index_path, f'{dataset_name}_test_{split_code}.txt')

        save_content_in_txt(train_path_list, train_path)
        save_content_in_txt(valid_path_list, valid_path)
        save_content_in_txt(test_path_list, test_path)

        data_path_dict = {
            'train': train_path,
            'valid': valid_path,
            'test': test_path}
    else:
        file_path_list = get_files(index_root, return_fullpath=True, sort=True)
        data_path_dict
    return data_path_dict

def get_data_indices(data_name, data_path, save_path, data_split, mode, generate_index_func):
    """"Get dataset indices and create if not exist"""
    # # TODO: gt not exist condition
    # create index folder and sub-folder if not exist
    os.chdir(save_path)
    index_dir_name = f'{data_name}_data_index'
    sub_index_dir_name = f'{data_name}_{data_split[0]}_{data_split[1]}'
    input_data_path = os.path.join(save_path, index_dir_name, sub_index_dir_name)
    if not os.path.isdir(input_data_path): os.makedirs(input_data_path)

    # generate index list and save in txt file
    generate_index_func(data_path, data_split, input_data_path)

    # load index list from saved txt
    os.chdir(input_data_path)
    if os.path.isfile(f'{mode}.txt'):
        input_data_indices = load_content_from_txt(f'{mode}.txt')
        input_data_indices.sort()
    else:
        input_data_indices = None

    if os.path.isfile(f'{mode}_gt.txt'):
        ground_truth_indices = load_content_from_txt(f'{mode}_gt.txt')
        ground_truth_indices.sort()
    else:
        ground_truth_indices = None
    return input_data_indices, ground_truth_indices

def generate_kaggle_breast_ultrasound_index(data_path, save_path, data_split):
    data_keys = {'input': 'exclude_mask', 'ground_truth': 'include_mask'}
    save_input_and_label_index(data_path, save_path, data_split, data_keys, loading_format=['png', 'jpg'])

def generate_kaggle_snoring_index(data_path, save_path, data_split):
    # TODO: no ground truth case
    # TODO: keys=None, include_or_exclude=None case
    data_keys = {'input': None}
    save_input_and_label_index(data_path, save_path, data_split, data_keys, loading_format=['wav', 'm4a'])

def save_input_and_label_index(data_path, save_path, data_split, data_keys=None, loading_format=None):
    # TODO: test input output
    # assert 'input' in data_keys, 'Undefined input data key'
    # assert 'ground_truth' in data_keys, 'Undefined ground truth data key'
    class_name = os.listdir(data_path)
    os.chdir(save_path)
    include_or_exclude, keys = [], []
    if data_keys:
        for v in data_keys.values():
            if v:
                include_or_exclude.append(v.split('_')[0])
                keys.append(v.split('_')[1])
    data_dict = generate_filenames(
        data_path, keys=keys, include_or_exclude=include_or_exclude, is_fullpath=True, loading_formats=loading_format)

    def save_content_ops(data, train_name, valid_name):
        # TODO: Is this a solid solution?
        data.sort(key=len)
        split = int(len(data)*data_split[0])
        train_input_data, val_input_data = data[:split], data[split:]
        save_content_in_txt(train_input_data,  train_name, filter_bank=class_name, access_mode="w+", dir=save_path)
        save_content_in_txt(val_input_data, valid_name, filter_bank=class_name, access_mode="w+", dir=save_path)

    # if data_keys['input']:
    #     input_data = data_dict[data_keys['input']]
    # else:
    #     input_data = data_dict

    if data_keys:
        if 'input' in data_keys:
            if data_keys['input']:
                input_data = data_dict[data_keys['input']]
            else:
                input_data = data_dict
            save_content_ops(input_data, 'train.txt', 'valid.txt')
        if 'ground_truth' in data_keys:
            if data_keys['ground_truth']:
                ground_truth = data_dict[data_keys['ground_truth']]
            else:
                ground_truth = data_dict
            save_content_ops(ground_truth, 'train_gt.txt', 'valid_gt.txt')
    else:
        input_data = data_dict
        save_content_ops(input_data, 'train.txt', 'valid.txt')

# TODO: mkdir?
def save_content_in_txt(content, path, filter_bank=None, access_mode='a+', dir=None):
    # assert isinstance(content, (str, list, tuple, dict))
    # TODO: overwrite warning
    with open(path, access_mode) as fw:
        # def string_ops(s, dir, filter):
        #     pair = string_filtering(s, filter)
        #     return os.path.join(dir, list(pair.keys())[0], list(pair.values())[0])

        if isinstance(content, str):
            # if dir:
            #     content = string_ops(content, dir, filter=filter_bank)
            #     # content = os.path.join(dir, content)
            fw.write(content)
        else:
            for c in content:
                # if dir:
                #     c = string_ops(c, dir, filter=filter_bank)
                #     # c = os.path.join(dir, c)
                fw.write(f'{c}\n')

def get_path_generator_in_case_order(data_path, return_fullpath, load_format=[]):
    dir_list = get_dir_list(data_path)
    for d in dir_list:
        file_list = get_files(d, keys=load_format, return_fullpath=return_fullpath)
        for file_idx, f in enumerate(file_list):
            yield (file_idx, f)

def save_data_label_pair_in_csv(data_path, save_path=None, save_name=None, load_format='wav', return_fullpath=True):
    path_loader = get_path_generator_in_case_order(data_path, return_fullpath, load_format=load_format)
    nums, ids, labels = [], [], []
    for idx, file_and_idx in enumerate(path_loader, 1):
        file_idx, f = file_and_idx
        file_path, file_name = os.path.split(f)
        label = int(file_path[-1])
        file_name = file_name.split('.')[0]
        print(idx, file_name, label)
        nums.append(file_idx)
        ids.append(file_name)
        labels.append(label)

    pair_dict = {'case_index': nums,
                 'id': ids,
                 'label': labels}
    pair_df = pd.DataFrame(pair_dict)
    if not save_name:
       save_name = 'train.csv'
    if save_path is not None:
        pair_df.to_csv(os.path.join(save_path, save_name))
    else:
        pair_df.to_csv(save_name)

def save_fileanmes_in_txt(glob_path, save_path=None, recursive=True):
    files = glob.glob(glob_path, recursive=recursive)
    if not save_path:
        save_path = 'filenames.txt'

    with open(save_path, 'w') as fw:
        for file in files:
            fw.write(f'{file}\n')
    return files

def generate_gt_csv_for_data(id_to_label, save_path):
    pass

if __name__ == "__main__":
    from dataset.melspec import melspec
    melspec()

    # data_path = rf'C:\Users\test\Desktop\Leon\Datasets\ASUS_snoring_subset\raw_final_test\freq6_no_limit\2_21\raw_f_h_2_mono_16k'
    # save_data_label_pair_in_csv(data_path, save_name='train1.csv')
    # data_path = rf'C:\Users\test\Desktop\Leon\Datasets\ASUS_snoring_subset\raw_final_test\freq6_no_limit_shift\2_21\raw_f_h_2_mono_16k'
    # save_data_label_pair_in_csv(data_path, save_name='train2.csv')
    # data_path = rf'C:\Users\test\Desktop\Leon\Datasets\ESC-50\ESC-50_process\esc50_16k\esc50_16k_2'
    # save_data_label_pair_in_csv(data_path, save_name='train3.csv')

    # # ASUS snoring
    # train_wav_list_path = r'C:\Users\test\Desktop\Leon\Datasets\ASUS_snoring_subset\index\Freq2\2_21_2s_my2\train.txt'
    # test_wav_list_path = r'C:\Users\test\Desktop\Leon\Datasets\ASUS_snoring_subset\index\Freq2\2_21_2s_my2\test.txt'
    # out_dir = r'C:\Users\test\Desktop\Leon\Datasets\ASUS_snoring_subset\preprocess\2_21_2s_my2'
    # # out_dir = r'C:\Users\test\Desktop\Leon\Datasets\ASUS_snoring_cpp\2_21_2s_my2'
    # get_melspec_from_cpp(train_wav_list_path, out_dir)
    # get_melspec_from_cpp(test_wav_list_path, out_dir)

    # # ESC-50
    # # wav_list_path = r'C:\Users\test\Desktop\Leon\Datasets\ESC-50\ESC-50_process\esc50\esc50_2\test.txt'
    # wav_list_path = r'C:\Users\test\Desktop\Leon\Datasets\ESC-50\ESC-50_process\esc50\esc50_2\file_names.txt'
    # out_dir = r'C:\Users\test\Desktop\Leon\Datasets\ASUS_snoring_subset\preprocess\esc50\44100'
    # # out_dir = r'C:\Users\test\Desktop\Leon\Datasets\ASUS_snoring_cpp\esc50\44100'
    # get_melspec_from_cpp(wav_list_path, out_dir, sampling_rate=44100)

    # Kaggle snoring
    # wav_list_path = r'C:\Users\test\Desktop\Leon\Datasets\ASUS_snoring_subset\index\Kaggle_snoring_full\valid.txt'
    # out_dir = r'C:\Users\test\Desktop\Leon\Datasets\ASUS_snoring_cpp\kaggle'
    # get_melspec_from_cpp(wav_list_path, out_dir, sampling_rate=48000)

    # # ASUS new
    # redmi = r'C:\Users\test\Desktop\Leon\Datasets\ASUS_snoring_0727\wave_split\1658889529250_RedmiNote8\0'
    # pixel = r'C:\Users\test\Desktop\Leon\Datasets\ASUS_snoring_0727\wave_split\1658889531056_Pixel4XL\0'
    # iphone = r'C:\Users\test\Desktop\Leon\Datasets\ASUS_snoring_0727\wave_split\1658889531172_iPhone11\0'
    # sr = 16000

    # # for data_path in [redmi, pixel, iphone]:
    # for data_path in [redmi]:
    #     split = os.path.split(os.path.split(data_path)[0])[1]
    #     glob_path = os.path.join(data_path, '*.wav')
    #     save_path = os.path.join(data_path, 'filenames.txt')
    #     save_fileanmes_in_txt(glob_path, recursive=True, save_path=save_path)

    #     wav_list_path = save_path
    #     out_dir = r'C:\Users\test\Desktop\Leon\Datasets\ASUS_snoring_cpp'
    #     out_dir = os.path.join(out_dir, split, str(sr))
    #     get_melspec_from_cpp(wav_list_path, out_dir, sampling_rate=sr)


import CompanyClasses

if __name__ == "__main__":

    # Mitarbeiter erzeugen
    m1 = CompanyClasses.Mitarbeiter("Nimit", "Singh", 21, "Verkaufsleiter", 1900)
    m2 = CompanyClasses.Mitarbeiter("Phillip", "Schueler", 18, "Programmierer", 2000)

    # Gruppenleiter erzeugen
    g1 = CompanyClasses.Gruppenleiter("Sabo", "Rubner", 25)
    g2 = CompanyClasses.Gruppenleiter("Max", "Neuner", 19)

    # Abteilungen erzeugen
    a1 = CompanyClasses.Abteilungen("Einkauf")
    a1.leiter.append(g1)
    a1.Worker.append(m1)
    a2 = CompanyClasses.Abteilungen("Verkauf")
    a2.leiter.append(g2)
    a2.Worker.append(m2)

    a3 = CompanyClasses.Abteilungen("Vertrieb")

    # Company erzeugen
    c1 = CompanyClasses.Company("Company1")
    c1.Abteilung.append(a1)
    c1.Abteilung.append(a2)
    c1.Abteilung.append(a3)

    def countMitarbeiter(Worker):
        return len(Worker)

    def countGruppenleiter(leiter):
        return len(leiter)

    def countAbteilungen(Abteilung):
        return len(Abteilung)

    print("Anzahl der Mitarbeiter in der Abteilung Einkauf: ")
    print(countMitarbeiter(a1.Worker))
    print("\n")
    print("Anzahl der Abteilungen: ")
    print(countAbteilungen(c1.Abteilung))
    print("\n")
    print("Anzahl der Gruppenleiter in jeder Abteilung: ")
    print(countGruppenleiter(a1.leiter))
    print(countGruppenleiter(a2.leiter))


#
#
# UtmTrav : calculation of an open and cloded traverse
#           projected on planeor UTM projected grid.
#
#
from pygeodesy.dms import toDMS, parseDMS
from pygeodesy import Utm,parseUTM5,Ellipsoids
import pandas as pd
import geopandas as gpd
import matplotlib.pyplot as plt
import numpy as np
import yaml,sys
from pathlib import Path
###############################################################
def parse_dms( dms ):
    return parseDMS( dms, sep=':')

class UTMTraverse:
    def __init__(self, YAMLFILE):
        with open(YAMLFILE,'r') as f:
        	YAML = yaml.safe_load( f )
        self.df_Sta = pd.DataFrame.from_dict(YAML['FIXPNT'] ,
            orient='index', columns=['Easting','Northing'] )
        def UtmStr(row):
            UTM_PRE = f"{YAML['ZONE']} {YAML['HEMI']} "
            return '{} {} {}'.format( UTM_PRE, row.Easting, row.Northing )
        self.df_Sta['UTMSTR'] =  self.df_Sta.apply( UtmStr, axis='columns')
        self.df_Sta['Utm'] = self.df_Sta['UTMSTR'].apply(parseUTM5)
        self.df_Sta.drop( columns=['Easting','Northing'], axis='index', inplace=True )
        self.df_Sta['Iter'] = -1
        self.df_Sta.reset_index(inplace=True)
        self.df_Sta.rename( columns={'index':'Name'}, inplace=True)
        self.df_Sta['Name'] = self.df_Sta['Name'].astype( str )
        ##############################
        self.df_Dis = pd.DataFrame.from_dict( YAML['DIST'],orient='index',
                            columns=['dist_m'] )
        self.df_Dis['Iter'] = -1
        self.df_Dis['ScalFac'] = 0.0
        self.df_Dis.reset_index(inplace=True)
        self.df_Dis.rename( columns={'index':'Name'}, inplace=True)
        ##############################
        self.df_Ang = pd.DataFrame.from_dict( YAML['ANGL'],orient='index',
                            columns=['ang_dms'] )
        self.df_Ang['ang_deg'] = self.df_Ang['ang_dms'].apply( parse_dms )
        self.df_Ang['Iter'] = -1
        self.df_Ang[['ac_BS','ac_FS']] = [0.,0.]
        self.df_Ang['type'] = 'open'
        self.df_Ang.reset_index(inplace=True)
        self.df_Ang.rename( columns={'index':'Name'}, inplace=True)
        ##############################
        self.YAML = YAML
        #import pdb; pdb.set_trace()
        return

    def CalcTraverse(self,ITER):
        df_Sta = self.df_Sta[( self.df_Sta['Iter']==self.df_Sta['Iter'].max() )
                          |  (self.df_Sta['Iter']==-1) ]
        df_Dis = self.df_Dis[self.df_Dis['Iter']==-1]
        df_Ang = self.df_Ang[self.df_Ang['Iter']==-1]
        for i in range(len(df_Dis)):
            DIS_AT,DIS_TO = self.df_Dis.iloc[i]['Name'].split(',')
            DIST          = self.df_Dis.iloc[i]['dist_m']
            ANG_BS,ANG_AT,ANG_FS = self.df_Ang.iloc[i]['Name'].split(',')
            ANGL                 = self.df_Ang.loc[i]['ang_deg']
            TYPE                 = self.df_Ang.loc[i]['type']
            UtmBS = self.df_Sta[self.df_Sta['Name']==ANG_BS].iloc[-1]['Utm']
            UtmAT = self.df_Sta[self.df_Sta['Name']==ANG_AT].iloc[-1]['Utm']
            ######################################
            if ITER==0:
                k = 1.0 ; ac1 = 0.0 ; ac2 = 0.0
            else:
                UtmFS = self.df_Sta[self.df_Sta['Name']==ANG_FS].iloc[-1]['Utm']
                k   = self.LineSF( UtmAT, UtmFS)
                ac1 = self.LineAC( UtmAT, UtmBS)
                ac2 = self.LineAC( UtmAT, UtmFS)
                #ac1 = 27.17/3600; #ac2 = 20.67/3600
            BS_Azi = self.GridAz( UtmAT, UtmBS  )
            corrAng =  ANGL + (ac1-ac2 )
            FS_Azi = BS_Azi + corrAng
            _,FS_Azi = divmod( FS_Azi, +360 )
            ######################################
            GD = k*DIST
            ETo = UtmAT.easting  + GD*np.sin( np.radians(FS_Azi) )
            NTo = UtmAT.northing + GD*np.cos( np.radians(FS_Azi) )
            ######################################
            df_StaX = pd.DataFrame( { 'Name': ANG_FS,
                'UTMSTR':None, 'Utm': Utm( self.YAML['ZONE'], self.YAML['HEMI'],
                ETo,NTo),'Iter': ITER }, index=[0]  )
            self.df_Sta = pd.concat( [self.df_Sta, df_StaX], axis=0, ignore_index=True )
            df_DisX = pd.DataFrame( { 'Name':f'{DIS_AT},{DIS_TO}',
                'dist_m': GD, 'AdjAz':FS_Azi, 'Iter': ITER, 'ScalFac': k }, index=[0] )
            self.df_Dis = pd.concat( [self.df_Dis,df_DisX], axis=0, ignore_index=True )
            df_AngX= pd.DataFrame( { 'Name': f'{ANG_BS},{ANG_AT},{ANG_FS}',
                'ang_dms': toDMS(corrAng), 'ang_deg': corrAng, 'Iter': ITER,
                'ac_BS': ac1*3600, 'ac_FS': ac2*3600, 'type':TYPE }, index=[0] )
            self.df_Ang = pd.concat( [ self.df_Ang, df_AngX ], axis=0,ignore_index=True )
        #import pdb; pdb.set_trace()
        return self.df_Sta, self.df_Dis, self.df_Ang

    def GridAz( self, UtmFr, UtmTo ):
      dE = UtmTo.easting-UtmFr.easting
      dN = UtmTo.northing-UtmFr.northing
      az = np.arctan2( dE, dN )
      _,az = divmod( az, np.pi*2 )  # 2PI
      return np.degrees( az )

    def LineSF( self,StaFr,StaTo ):
        utm_mid = Utm( self.YAML['ZONE'], self.YAML['HEMI'],
                (StaFr.easting +StaTo.easting)/2,
                (StaFr.northing+StaTo.northing)/2  )
        sf_to  = StaTo.toLatLon().scale
        sf_mid = utm_mid.toLatLon().scale
        sf_fr  = StaFr.toLatLon().scale
        sf_line = (1/6)*(sf_fr+4*sf_mid+sf_to)  # quadratic mean
        #sf_line = (sf_fr+sf_to)/2  # average
        return sf_line

    def LineAC( self,StaFr,StaTo ):
        if 'ARC2CHORD' in self.YAML.keys() and self.YAML['ARC2CHORD'] is True:
            utm_mid = Utm( self.YAML['ZONE'], self.YAML['HEMI'],
                     (StaFr.easting+StaTo.easting)/2,
                        (StaFr.northing+StaTo.northing)/2  )
            rm = Ellipsoids.WGS84.rocGauss(utm_mid.toLatLon().lat)
            N2N1 = StaTo.northing-StaFr.northing
            E2E1 = (StaTo.easting-500_000) + 2*( StaFr.easting-500_000)
            del12 = np.degrees( N2N1*E2E1/(6*rm*rm) )
            return del12
        else:
            return 0.0

    def CloseTrav( self ):
        print('####### Adjust Interior Angles of the Close Traverse ########')
        self.JuncPnt = self.YAML['JUNCTION']
        df_JuncAng = self.df_Ang[self.df_Ang['Name'].str.contains(
                                             f',{self.JuncPnt},' )]
        assert( len(df_JuncAng) == 2),\
           f"***ERROR*** {self.JuncPnt} must have 2 records in ANGL..."
        ##### partition open & close travesre #####
        df_AngCloTrv = self.df_Ang[ self.df_Ang.Name==df_JuncAng.iloc[1].Name ]
        nameAng = df_AngCloTrv.Name.str.split(',').to_list()
        nameAng = np.array( nameAng )
        idx = df_AngCloTrv.index.values.astype(int)[0]
        df_AngCloTrv = self.df_Ang.iloc[ idx: ]
        ##### angle correction due to angel excess #####
        sumAng = df_AngCloTrv.ang_deg.sum()
        sumAng_= 180*(len(df_AngCloTrv)-2)
        nAng = len( df_AngCloTrv )
        difAng = sumAng-sumAng_
        print(f'Close traverse number of angles : {nAng}')
        print(f'Close traverse angle excess (n-2)*180 : {difAng*3600:+.0f} sec.')
        self.df_Ang.loc[ idx:idx+nAng, 'ang_deg' ] = \
            self.df_Ang.loc[ idx:idx+nAng, 'ang_deg' ]-difAng/nAng  # ***ANG_ADJ***
        self.df_Ang.loc[ idx:idx+nAng, 'type' ] = 'closed'
        self.df_Ang['ang_dms'] = self.df_Ang['ang_deg'].apply( toDMS  )
        assert(np.isclose(self.df_Ang.iloc[idx:].ang_deg.sum(),sumAng_ )),"Not closed!"
        print('Close traverse sum of angels          : {:.7f} deg'.format(
                                df_AngCloTrv.ang_deg.sum() ) )
        ##### merge junction angles #####
        jncAngSum = df_JuncAng.ang_deg.sum()
        jncAngNam = self.df_Ang.iloc[idx-1].Name+','+self.df_Ang.iloc[idx].Name
        jncAngNam = jncAngNam.split(',')
        assert( jncAngNam[1]==jncAngNam[4] or jncAngNam[2]==jncAngNam[3]),\
                      f'***ERROR*** cannot merge {jncAngNam}'
        jncAng = jncAngNam[0:2]+ [jncAngNam[-1],]
        jncAng = ",".join( jncAng )
        self.df_Ang.loc[ idx, ['Name', 'ang_deg', 'type' ] ] =\
                                       [jncAng,jncAngSum,'junction']
        self.df_Ang.drop( idx-1, axis='index', inplace=True )
        return

    def Clousure( self ):
        JPNT=trav.YAML['JUNCTION']
        jpnt  = df_Sta[df_Sta.Name==JPNT].iloc[0].Utm
        jpnt_ = df_Sta[df_Sta.Name==JPNT+'_'].iloc[0].Utm
        self.dE = jpnt.easting -jpnt_.easting
        self.dN = jpnt.northing-jpnt_.northing
        self.dJPNT = np.sqrt( self.dE*self.dE + self.dN*self.dN )

        print(f'Closure at junction {JPNT:6} : {self.dJPNT:+.3f} m.')
        self.idxJPNT =trav.df_Dis[trav.df_Dis.Name.str.startswith(f'{JPNT},')].iloc[-1].name
        self.sumTrv = trav.df_Dis.iloc[self.idxJPNT:].dist_m.sum()
        print(f'Sum of closed traverse      : {self.sumTrv:,.3f} m.')
        print(f'Linear closure              :  1: {self.sumTrv/self.dJPNT:,.0f} ')

    def AdjustCompassRule( self):
        dE= self.dE ;    dN=self.dN    ; dS=self.dJPNT
        df_DL = df_Dis.iloc[self.idxJPNT:].copy()
        df_DL.drop( columns=['ScalFac'], axis='index', inplace=True )
        #import pdb; pdb.set_trace()
        df_DL['Dep'] = df_DL['dist_m']*np.sin(np.radians(df_DL['AdjAz']))
        df_DL['Lat'] = df_DL['dist_m']*np.cos(np.radians(df_DL['AdjAz']))
        sumDep = df_DL['Dep'].sum()
        sumLat = df_DL['Lat'].sum()
        sumDis = df_DL.dist_m.sum()
        df_DL['AdjDep'] = df_DL['Dep'] - sumDep*df_DL['dist_m']/sumDis
        df_DL['AdjLat'] = df_DL['Lat'] - sumLat*df_DL['dist_m']/sumDis
        df_DL_ = df_DL.copy()
        for col in ( 'Dep','Lat','AdjDep','AdjLat' ):
            df_DL_[col] = df_DL_[col].map('{:10.3f}'.format )
        print( df_DL_ )
        #import pdb; pdb.set_trace()
        sumAdjDep = df_DL['AdjDep'].sum()
        sumAdjLat = df_DL['AdjLat'].sum()
        print( f'sumDist={sumDis:.3f}m | sumDep={sumDep:+.3f}m sumLat={sumLat:+.3f}m | '\
               f'sumAdjDep={sumAdjDep:.3f}m sumAdjLat={sumAdjLat:.3f}m' )
        #####################################
        print( '=== Adjusted Coordiate by Compass Rule ===')
        print( '=Station=====Easting========Northing=====')
        FRutm = None
        for i,row in df_DL.iterrows():
            FR,TO = row.Name.split(',')
            if FRutm is None:
                FRutm = trav.df_Sta[ trav.df_Sta.Name==FR ].iloc[-1].Utm
                E,N = FRutm.easting, FRutm.northing
                print( f'{FR:6s} {E:15,.3f}  {N:15,.3f}' )
            E = E + row.AdjDep
            N = N + row.AdjLat
            print( f'{TO:6s} {E:15,.3f}  {N:15,.3f}' )

    def Plot(self, TITLE):
        df_Sta = self.df_Sta[self.df_Sta.Iter<=0]
        df_Dis = self.df_Dis[self.df_Dis.Iter==0]
        #import pdb; pdb.set_trace()
        for i,row in df_Sta.iterrows():
	        #print( i,row )
                if row.Iter==-1: ms = 20; fc='green'
                else:            ms = 10; fc='red'
                plt.plot( [row.Utm.easting], [row.Utm.northing],
                        marker="^", markersize=ms, mfc=fc, mec=fc )
                plt.text( row.Utm.easting,row.Utm.northing, row.Name )
        for i,row in df_Dis.iterrows():
	        #print( i, row )
	        FR,TO = row.Name.split(',')
	        fr = df_Sta[df_Sta.Name==FR].iloc[-1].Utm
	        to = df_Sta[df_Sta.Name==TO].iloc[-1].Utm
	        plt.plot( [fr.easting,to.easting] , [fr.northing,to.northing], '-r' )
        plt.ticklabel_format(useOffset = False, style = 'plain')
        plt.xticks(rotation=90)
        plt.gca().set_aspect("equal")
        plt.grid()
        plt.tight_layout()
        plt.title( TITLE )
        #plt.show()
        plt.savefig( TITLE )

###########################################################
###########################################################
###########################################################
pd.set_option('display.max_rows', None)
pd.set_option('display.max_columns', None)
pd.set_option('display.width', None)

if len(sys.argv)==2:
    YAMLFILE = Path(sys.argv[1])
else:
    YAMLFILE = Path( './Dec17_Travese_Compute/P4_Trav.yaml' )
trav = UTMTraverse( YAMLFILE )
### Case closed traverse , adjust interior angles #########
if 'JUNCTION' in trav.YAML.keys():
    trav.CloseTrav()

###########################################################
for i in range(trav.YAML['ITER']):
    print(f'========================= Iter {i} ============================')
    df_Sta,df_Dis,df_Ang = trav.CalcTraverse(ITER=i)
trav.Clousure()
###########################################################
def _toUtmStr(row):
    if row.UTMSTR is None:
        return row.Utm.toStr(prec=-3)
    else:
        return row.UTMSTR
df_Sta['UTMSTR'] = df_Sta.apply( _toUtmStr, axis='columns' )
df_Dis['ScalFac'] = df_Dis['ScalFac'].map( '{:.9f}'.format )
if 1:
    print( df_Sta )
    print( df_Dis )
    print( df_Ang )
if trav.YAML['ITER']>=2:
    dE=list(); dN=list()
    for grp,row in df_Sta.groupby('Name'):
        MAX = row.Iter.max()
        if MAX==trav.YAML['ITER']-1:
            dE.append( row.iloc[MAX].Utm.easting- row.iloc[MAX-1].Utm.easting )
            dN.append( row.iloc[MAX].Utm.northing- row.iloc[MAX-1].Utm.northing )
    print( 'Converting@{} dE : {}'.format(trav.YAML['ITER'], np.array( dE ).round(3) ) )
    print( 'Converting@{} dN : {}'.format(trav.YAML['ITER'], np.array( dN ).round(3) ) )
for fmt in ('png','pdf','svg'):
    PLOT = f'./CACHE/{YAMLFILE.stem}.{fmt}'
    print( f'Plotting {PLOT}...')
    trav.Plot( PLOT )
#trav.Plot( (YAMLFILE.stem + '.pdf' ) )
#############################################################
trav.AdjustCompassRule()
print( '@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ End of UtmTrav.py @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@')
#import pdb; pdb.set_trace()


#!/bin/python3

__author__ = "Adam Karl"
"""Find the sum of all numbers below N which divide the sum of the factorial of their digits"""
#https://projecteuler.net/problem=34

digitFactorials = []

def sumCuriousNumbersUnderN(n):
    """Return a sum of all numbers that evenly divide the sum of the factorial of their digits"""
    sumCurious = 0
    for i in range(10, n): #no numbers under 10 are curious, even though 1!=1 and 2!=2
        string = str(i)
        digitFactorialSum = 0
        for c in string:
            digitFactorialSum += digitFactorials[int(c)]
        if digitFactorialSum == i:
            sumCurious += i
    return sumCurious

def main():
    print("Find the sum of all numbers which are equal to the sum of the factorial of their digits")
    global digitFactorials
    digitFactorials = [1] #0! = 1
    for i in range(9): #1-9
        digitFactorials.append(digitFactorials[-1] * (i+1))

    #since 7 * 9! = 2540160, 2540160 is the last number that can be potentially curious
    result = sumCuriousNumbersUnderN(2540161)
    print("Sum = %d" % result)

if __name__ == "__main__":
    main()


import copy
import os

import numpy as np

from metadrive.manager.base_manager import BaseManager
from metadrive.scenario.scenario_description import ScenarioDescription as SD, MetaDriveType
from metadrive.scenario.utils import read_scenario_data, read_dataset_summary

class ScenarioDataManager(BaseManager):
    DEFAULT_DATA_BUFFER_SIZE = 100
    PRIORITY = -10

    def __init__(self):
        super(ScenarioDataManager, self).__init__()
        from metadrive.engine.engine_utils import get_engine
        engine = get_engine()

        self.store_data = engine.global_config["store_data"]
        self.directory = engine.global_config["data_directory"]
        self.num_scenarios = engine.global_config["num_scenarios"]
        self.start_scenario_index = engine.global_config["start_scenario_index"]

        # for multi-worker
        self.worker_index = self.engine.global_config["worker_index"]
        self.available_scenario_indices = [
            i for i in range(
                self.start_scenario_index + self.worker_index, self.start_scenario_index +
                self.num_scenarios, self.engine.global_config["num_workers"]
            )
        ]
        self._scenarios = {}

        # Read summary file first:
        self.summary_dict, self.summary_lookup, self.mapping = read_dataset_summary(self.directory)
        self.summary_lookup[:self.start_scenario_index] = [None] * self.start_scenario_index
        end_idx = self.start_scenario_index + self.num_scenarios
        self.summary_lookup[end_idx:] = [None] * (len(self.summary_lookup) - end_idx)

        # sort scenario for curriculum training
        self.scenario_difficulty = None
        self.sort_scenarios()

        # existence check
        assert self.start_scenario_index < len(self.summary_lookup), "Insufficient scenarios!"
        assert self.start_scenario_index + self.num_scenarios <= len(self.summary_lookup), \
            "Insufficient scenarios! Need: {} Has: {}".format(self.num_scenarios,
                                                              len(self.summary_lookup) - self.start_scenario_index)

        for p in self.summary_lookup[self.start_scenario_index:end_idx]:
            p = os.path.join(self.directory, self.mapping[p], p)
            assert os.path.exists(p), "No Data at path: {}".format(p)

        # stat
        self.coverage = [0 for _ in range(self.num_scenarios)]

    @property
    def current_scenario_summary(self):
        return self.current_scenario[SD.METADATA]

    def _get_scenario(self, i):
        assert i in self.available_scenario_indices, \
            "scenario index exceeds range, scenario index: {}, worker_index: {}".format(i, self.worker_index)
        assert i < len(self.summary_lookup)
        scenario_id = self.summary_lookup[i]
        file_path = os.path.join(self.directory, self.mapping[scenario_id], scenario_id)
        ret = read_scenario_data(file_path)
        assert isinstance(ret, SD)
        self.coverage[i - self.start_scenario_index] = 1
        return ret

    def before_reset(self):
        if not self.store_data:
            assert len(self._scenarios) <= 1, "It seems you access multiple scenarios in one episode"
            self._scenarios = {}

    def get_scenario(self, i, should_copy=False):

        _debug_memory_leak = False

        if i not in self._scenarios:

            if _debug_memory_leak:
                # inner psutil function
                def process_memory():
                    import psutil
                    import os
                    process = psutil.Process(os.getpid())
                    mem_info = process.memory_info()
                    return mem_info.rss

                cm = process_memory()

            # self._scenarios.clear_if_necessary()

            if _debug_memory_leak:
                lm = process_memory()
                print("{}:  Reset! Mem Change {:.3f}MB".format("data manager clear scenario", (lm - cm) / 1e6))
                cm = lm

            ret = self._get_scenario(i)
            self._scenarios[i] = ret

            if _debug_memory_leak:
                lm = process_memory()
                print("{}:  Reset! Mem Change {:.3f}MB".format("data manager read scenario", (lm - cm) / 1e6))
                cm = lm

        else:
            ret = self._scenarios[i]
            # print("===Don't need to get new scenario. Just return: ", i)

        if should_copy:
            return copy.deepcopy(self._scenarios[i])

        # Data Manager is the first manager that accesses  data.
        # It is proper to let it validate the metadata and change the global config if needed.

        return ret

    def get_metadata(self):
        state = super(ScenarioDataManager, self).get_metadata()
        raw_data = self.current_scenario
        state["raw_data"] = raw_data
        return state

    def transform_coordinate(self, scenario):
        raise ValueError("Deprecated now as all coordinates is right-handed now")
        if not self.engine.global_config["allow_coordinate_transform"]:
            assert scenario[SD.METADATA][SD.COORDINATE] == MetaDriveType.COORDINATE_METADRIVE, \
                "Only support MetaDrive coordinate!"
        else:
            # It supports loading WaymoData or exported data in two coordinates
            if scenario[SD.METADATA][SD.COORDINATE] == MetaDriveType.COORDINATE_WAYMO:
                self._coordinate_transform = True
            elif scenario[SD.METADATA][SD.COORDINATE] == MetaDriveType.COORDINATE_METADRIVE:
                self._coordinate_transform = False
            else:
                raise ValueError()

    @property
    def current_scenario_length(self):
        return self.current_scenario[SD.LENGTH]

    @property
    def current_scenario(self):
        return self.get_scenario(self.engine.global_random_seed)

    def sort_scenarios(self):
        """
        TODO(LQY): consider exposing this API to config
        Sort scenarios to support curriculum training. You are encouraged to customize your own sort method
        :return: sorted scenario list
        """
        if self.engine.max_level == 0:
            raise ValueError("Curriculum Level should be greater than 1")
        elif self.engine.max_level == 1:
            return

        def _score(scenario_id):
            file_path = os.path.join(self.directory, self.mapping[scenario_id], scenario_id)
            scenario = read_scenario_data(file_path)
            obj_weight = 0

            # calculate curvature
            ego_car_id = scenario[SD.METADATA][SD.SDC_ID]
            state_dict = scenario["tracks"][ego_car_id]["state"]
            valid_track = state_dict["position"][np.where(state_dict["valid"].astype(int))][..., :2]

            dir = valid_track[1:] - valid_track[:-1]
            dir = np.arctan2(dir[..., 1], dir[..., 0])
            curvature = sum(abs(dir[1:] - dir[:-1]) / np.pi) + 1

            sdc_moving_dist = SD.sdc_moving_dist(scenario)
            num_moving_objs = SD.num_moving_object(scenario, object_type=MetaDriveType.VEHICLE)
            return sdc_moving_dist * curvature + num_moving_objs * obj_weight

        start = self.start_scenario_index
        end = self.start_scenario_index + self.num_scenarios
        id_scores = [(s_id, _score(s_id)) for s_id in self.summary_lookup[start:end]]
        id_scores = sorted(id_scores, key=lambda scenario: scenario[-1])
        self.summary_lookup[start:end] = [id_score[0] for id_score in id_scores]
        self.scenario_difficulty = {id_score[0]: id_score[1] for id_score in id_scores}

    def clear_stored_scenarios(self):
        self._scenarios = {}

    @property
    def current_scenario_difficulty(self):
        return self.scenario_difficulty[self.summary_lookup[self.engine.global_random_seed]
                                        ] if self.scenario_difficulty is not None else 0

    @property
    def current_scenario_id(self):
        return self.current_scenario_summary["scenario_id"]

    @property
    def current_scenario_file_name(self):
        return self.summary_lookup[self.engine.global_random_seed]

    @property
    def data_coverage(self):
        return sum(self.coverage) / len(self.coverage) * self.engine.global_config["num_workers"]


# --------------
#Importing header files
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

#Reading the file
data=pd.read_csv(path)

#Code starts here

# Step 1
#Reading the file

#Creating a new variable to store the value counts
loan_status=data["Loan_Status"].value_counts()

#Plotting bar plot
loan_status.plot(kind="bar")

# Step 2
#Plotting an unstacked bar plot
property_and_loan=data.groupby(["Property_Area","Loan_Status"]).size().unstack()
property_and_loan.plot(kind="bar",stacked=False)
plt.xlabel("Property Area")
plt.ylabel("Loan Status")
plt.xticks(rotation=45)

#Changing the x-axis label

#Changing the y-axis label

#Rotating the ticks of X-axis

# Step 3
#Plotting a stacked bar plot
education_and_loan=data.groupby(["Education","Loan_Status"]).size().unstack()
education_and_loan.plot(kind="bar",stacked=True)
plt.xlabel("Education Status")
plt.ylabel("Loan Status")
plt.xticks(rotation=45)

#Changing the x-axis label

#Changing the y-axis label

#Rotating the ticks of X-axis

# Step 4
#Subsetting the dataframe based on 'Education' column
graduate=data[data["Education"]=="Graduate"]
not_graduate=data[data["Education"]=="Not Graduate"]
dd=graduate["LoanAmount"]
dd=dd.fillna(0)
dd.plot(kind="density",label="Graduate")
ss=not_graduate["LoanAmount"]
ss=ss.fillna(0)
ss.plot(kind="density",label="not_Graduate")

#Subsetting the dataframe based on 'Education' column

#Plotting density plot for 'Graduate'

#Plotting density plot for 'Graduate'

#For automatic legend display

# Step 5
#Setting up the subplots
fig ,(ax_1,ax_2,ax_3)=plt.subplots(nrows = 3 , ncols = 1)

#Plotting scatter plot
data.plot.scatter(x='ApplicantIncome',y="LoanAmount",ax=ax_1,title="Applicant Income")
data.plot.scatter(x='CoapplicantIncome',y="LoanAmount",ax=ax_2,title="Coapplicant Income")

#Setting the subplot axis title
data["TotalIncome"]=data["ApplicantIncome"]+data["CoapplicantIncome"]

#Plotting scatter plot
data.plot.scatter(x="TotalIncome",y="LoanAmount",ax=ax_3,title="Total Income")

#Setting the subplot axis title

#Creating a new column 'TotalIncome'

#Plotting scatter plot

#Setting the subplot axis title


import random
import sys

sys.tracebacklimit = -1
errors = (AttributeError, ValueError)

#let's make it a class so it's easier to modify
class TicTacToe():
	def __init__ (self, gameboard = None):
		self.gameboard = gameboard
		self.gameboard = []
		self.player = None
#let's build the Gameboard
	def build_board(self):
		for i in range (3):
			row = []
			for j in range (3):
				row.append('-')
			self.gameboard.append(row)

#let's define how the board is going to look
#everytime the user plays.

	def show_board(self):
		for row in self.gameboard:
			for item in row:
				print(item, end = " ")
			print ()
#let's check if the board if full
	def is_board_full(self):
		for row in self.gameboard:
			for item in row:
				if item == '-':
					return False
		return True

#user input
	def user_input(self):

		try:
			self.row, self.col = list (map(int, input("\n Enter row and column number: ").split()))
			print()
		except errors:
			print ("Please enter a valid input")

#check if place is taken
	def check_place(self):
		if self.gameboard [self.row -1][self.col -1] != '-':
			print("This position is taken, choose a different position player:", self.player)
			self.row, self.col = (list (map(int, input("\n Enter row and column number: ").split())))
			print()
		else:
			pass

#function to fill the board
	def fill_board(self):
		if self.gameboard [self.row -1][self.col -1] == '-':
			self.gameboard [self.row -1][self.col -1] = self.player
			print()

#first player
	def first(self, player):
		if random.randint (0,1) == 0:
			return ('X','O')
		return ('O','X')

#let's define the win variable
	def win_variable(self):
		win = None

#let's check the winner
	def check_winner(self):
		n = len(self.gameboard)
		win = True
		for i in range (3):
			if all ((self.gameboard[i][j] == self.player for j in range(3))):
				return True
			if all ((self.gameboard[j][i] == self.player for j in range(3))):
				return True
			if all ((self.gameboard[j][j] == self.player for j in range(3))):
				return True
			if all ((self.gameboard[n - 1 -i][i]  == self.player for j in range(3))):
				return True
			else:
				return False


from tkinter import *
from tkinter import messagebox
from PIL import ImageTk, Image
import pygame
import time
import threading
import serial

pygame.mixer.init()
arduinoData = serial.Serial('COM3', 9600)
test = False
musica = False
'''
Esta función se encarga de crear la ventana principal, donde se muestra la consola de comandos y el robot virtual.
Parámetros:
    None
Retorna:
    No retorna
'''
def ventana():
    ventana = Tk()
    ventana.title('Robot Virtual')
    ventana.minsize(1280, 720)
    ventana.resizable(width=NO, height=NO)
    ventana.configure(background='#C3DFF4')

    canvas = Canvas(ventana, width=450, height=400, background='#7293C4')
    canvas.place(x=40, y=150)
    canvas2 = Canvas(ventana, width=650, height=600, background='White')
    canvas2.place(x=600, y=50)

    '''
    Objeto Robot
    atributos: ventana, nombre (str), imagen, fecha_creacion (str), felicidad (int), contador (int), anim (boolean)
    metódos:
    hello_ani(): se encarga de crear la animación del robot saludando
    sayhello(): muestra en un messagebox  el nombre del robot y llama a la función hello_ani
    forward_ani(): se encarga de crear la animación del robot caminando hacia adelante
    forward(): cambia la imagen del robot para que este mire hacia adelante
    backward_ani(): se encarga de crear la animación del robot caminando hacia atrás
    backward(): cambia la imagen del robot para que este mire hacia atrás
    stop(): se encarga de detener cualquier animación
    backflip_ani(): se encarga de crear la animación del robot haciendo un backflip
    backflip(): se encarga de llamar a la funcion backflip_ani
    music_on(): carga un archivo mp3 y lo reproduce
    music_off(): detiene la reproducción del archivo mp3 y lo cierra
    '''
    class Robot:
        def __init__(self, ventana, nombre, imagen, contador, anim):
            self.ventana = ventana
            self.nombre = nombre
            self.imagen = imagen
            self.contador = contador
            self.anim = anim

        def hello_ani(self):
            self.anim = True
            canvas2.delete('imgR')
            while self.contador != 12:
                if self.anim == False:
                    break
                nombre = 'Imagenes/wave/wave'+str(self.contador)+'.jpg'
                imgR = ImageTk.PhotoImage(Image.open(nombre).resize((700,400)))
                canvas2.create_image(0,100, image=imgR, anchor=NW, tag='imgR')
                self.contador+=1
                time.sleep(0.15)
                canvas2.delete('imgR')
            self.contador = 1
            canvas2.create_image(0, 100, image=self.imagen, anchor=NW)

        def sayhello(self):
            messagebox.showinfo(title='Nombre:', message='Hola! Mi nombre es ' + str(self.nombre))
            threading.Thread(target=self.hello_ani).start()

        def forward_ani(self):
            self.anim = True
            canvas2.delete('imgR')
            while self.contador != 22:
                if self.anim == False:
                    break
                nombre = 'Imagenes/walkF/walkF'+str(self.contador)+'.jpg'
                imgR = ImageTk.PhotoImage(Image.open(nombre).resize((700,400)))
                canvas2.create_image(0, 100, image=imgR, anchor=NW, tag='imgR')
                self.contador+=1
                time.sleep(0.15)
                canvas2.delete('imgR')
            self.contador = 1
            canvas2.create_image(0, 100, image=self.imagen, anchor=NW)

        def forward(self):
            canvas2.delete('imgR')
            self.imagen = ImageTk.PhotoImage(Image.open('Imagenes/defaultF.png').resize((700,400)))
            canvas2.create_image(0,100, image=self.imagen, anchor=NW, tag='imgR')
            threading.Thread(target=self.forward_ani).start()

        def backward_ani(self):
            self.anim = True
            canvas2.delete('imgR')
            while self.contador != 22:
                if self.anim == False:
                    break
                nombre = 'Imagenes/walkB/walkB'+str(self.contador)+'.jpg'
                imgR = ImageTk.PhotoImage(Image.open(nombre).resize((700,400)))
                canvas2.create_image(0, 100, image=imgR, anchor=NW, tag='imgR')
                self.contador+=1
                time.sleep(0.15)
                canvas2.delete('imgR')
            self.contador = 1
            canvas2.create_image(0, 100, image=self.imagen, anchor=NW)

        def backward(self):
            canvas2.delete('imgR')
            self.imagen = ImageTk.PhotoImage(Image.open('Imagenes/defaultB.png').resize((700,400)))
            canvas2.create_image(0,100, image=self.imagen, anchor=NW, tag='imgR')
            threading.Thread(target=self.backward_ani).start()

        def stop(self):
            self.anim = False

        def backflip_ani(self):
            self.anim = True
            canvas2.delete('imgR')
            while self.contador != 38:
                if self.anim == False:
                    break
                nombre = 'Imagenes/backflip/backflip ('+str(self.contador)+').jpg'
                imgR = ImageTk.PhotoImage(Image.open(nombre).resize((700,400)))
                canvas2.create_image(0, 100, image=imgR, anchor=NW, tag='imgR')
                self.contador+=1
                time.sleep(0.15)
                canvas2.delete('imgR')
            self.contador = 1
            canvas2.create_image(0, 100, image=self.imagen, anchor=NW)

        def backflip(self):
            threading.Thread(target=self.backflip_ani).start()

        def music_on(self):
            pygame.mixer.music.load('aespa.mp3')
            pygame.mixer.music.play()

        def music_off(self):
            pygame.mixer.music.stop()
            pygame.mixer.music.unload()

    '''
    Esta función se encarga de recibir un dato por medio de una entry y devuelve uno de los métodos del objeto robot segun lo que se introdujo en la entry
    Parámetros:
        None
    Retorna:
        No retorna
    '''
    def aceptar():
        comando = consola.get()
        if comando == 'sayhello':
            R.sayhello()
        if comando == 'forward':
            R.forward()
        if comando == 'backward':
            R.backward()
        if comando == 'stop':
            R.stop()
        if comando == 'backflip':
            R.backflip()
        if comando == 'music_on':
            R.music_on()
        if comando == 'music_off':
            R.music_off()

    '''
    Esta funcion se encarga de cambiar la variable test a un valor boolean True
    Parametros:
        None
    Retorna:
        wait_for_button(test)
    '''
    def test1():
        global test
        test = True
        return wait_for_button(test)

    '''
    Esta funcion se encarga de cambiar la variable test a un valor boolean True
    Parametros:
        test: se encarga de
    Retorna:
        Segun el boton que se presiona, retorna uno de los comandos del robot
    '''
    def wait_for_button(test):
        global musica
        while test == True:
            command = arduinoData.read(2)
            if len(command) == 2:
                if command == b'B1':
                    test = False
                    R.sayhello()
                elif command == b'B2':
                    test = False
                    R.forward()
                elif command == b'B3':
                    test = False
                    R.backward()
                elif command == b'B4':
                    test = False
                    R.backflip()
                elif command == b'B5':
                    if musica == True:
                        test = False
                        musica = False
                        R.music_off()
                    else:
                        test = False
                        musica = True
                        R.music_on()

    Label0 = Label(canvas, text='Consola de comandos', bg='#A2C4EC', fg='#383A3D', font=('Baskerville Old Face', 10))
    Label0.place(x=20, y=10)
    Label1 = Label(canvas, text='Los comandos disponibles son: ', bg='#A2C4EC', fg='#383A3D', font=('Baskerville Old Face', 10), width=25)
    Label2 = Label(canvas, text='- sayhello\n- forward\n- backward\n- stop\n- dance\n- music_on\n- music_off', bg='#A2C4EC', width=10, justify=LEFT, fg='#383A3D', font=('Baskerville Old Face', 10), anchor='w')
    Label1.place(x=20, y=120)
    Label2.place(x=110, y=150)

    consola = Entry(canvas, bg='#A2C4EC')
    consola.place(x=20, y=35)

    btn = Button(canvas, text='Aceptar', bg='#A2C4EC', fg='#383A3D', font=('Baskerville Old Face', 10), command=test1)
    btn.place(x=60, y=60)

    imgR = ImageTk.PhotoImage(Image.open('Imagenes/defaultF.png').resize((700,400)))
    canvas2.create_image(0,100, image=imgR, anchor=NW, tag='imgR')
    R = Robot(ventana, 'Linaria', imgR, 1, False)
    ventana.mainloop()

ventana()


import os
import requests
import time
import pyfiglet
req = requests.get('https://google.com')
os.system('clear')
print ("Fuck Dunia Percintaan")
time.sleep(1)
os.system('clear')
print ("\33[36;1m")
text = pyfiglet.figlet_format("SantriXploiter")
print (text)
print
print ("[1] Tentang SantriXploiter")
print ("[2] Blog SantriXploiter")
print ("[3] Web SantriXploiter")
print ("[4] Instagram SntriXploiter")
print ("[5] Group Telegram ")
print ("[6] Channel Youtube ")
print ("[7] Fanspage Facebook")
print
pilih = int(input("pilih : "))
if pilih == 1:
  time.sleep(1)
  print("Komunitas dari Pelajar Biasa yang tertarik Pada Dunia IT Dan Blog Yang Membahas Seputar Dunia IT dan Agama")
elif pilih == 2:
  time.sleep(1)
  os.system('xdg-open https://santrixploiter.blogspot.com')
elif pilih == 3:
  time.sleep(1)
  os.system('xdg-open http://aliyajnck.000webhostapp.com')
elif pilih == 4:
  time.sleep(1)
  os.system('xdg-open https://www.instagram.com/santri.xploiter/')
elif pilih == 5:
  time.sleep(1)
  os.system('xdg-open http://t.me/santrixploiter')
elif pilih == 6:
  time.sleep(1)
  os.system('xdg-open https://www.youtube.com/channel/UCG8EAqQdcDzphpluzrJWt3Q')
elif pilih == 7:
  time.sleep(1)
  os.system('xdg-open https://www.facebook.com/santrixploiter/')
else:
  print ("Tidak Tersedia")


# import the required modules and types for this example...
from typing import Any, Dict, List

# import the paginator and modal...
from discord.ext.modal_paginator import ModalPaginator, PaginatorModal

# import the discord.py module
import discord

from discord.ext import commands

bot = commands.Bot(command_prefix=commands.when_mentioned, intents=discord.Intents(guilds=True, messages=True))

# pre-defined questions and title for each modal
# this could be loaded from a json file or database
# each "modal" is a dict with the following keys:
# title: str - the title of the modal
# required: bool - whether the modal is required to be filled out before the paginator can be finished
# questions: List[str] - a list of questions to ask the user. Max 5 questions per modal.
personal_questions = {
    "title": "Personal Questions",
    "required": True,
    "questions": [
        "What is your name?",
        "What is your age?",
        "Any hobbies?",
        "Dad's name?",
        "Mom's name?",
    ],
}
misc_questions = {
    "title": "Miscellaneous Questions",
    "required": False,
    "questions": [
        "What is your favorite color?",
        "What is your favorite food?",
        "What is your favorite animal?",
        "What is your favorite movie?",
        "What is your favorite game?",
    ],
}
reason_questions = {
    "title": "Why Questions",
    "required": True,
    "questions": [
        "Why do you want to join?",
        "Why should we let you in?",
        "What do you like about the server?",
        "What do you like about the bot?",
        "What do you like about the community?",
    ],
}
# want even more control over the inputs?
# you can use the TextInput class directly
# from discord.ui import TextInput
# and then add the TextInput to the modal
# modal.add_item(TextInput(...)) instead of looping.
# see the discord.py docs for more info on the TextInput class.

# more dict like? use a dict for the questions too
# questions = [
#   {
#     "label": "What is your name?",
#     "min_length": 2,
#     "max_length": 200,
#     "required": False,
#     ...
#   },
#   ...
# ]
# etc...

# subclass the paginator to define our own on_finish method
# and to add the modals to the paginator via a custom __init__
class VerifyModal(ModalPaginator):
    def __init__(self, questions_inputs: List[Dict[str, Any]], *, author_id: int, **kwargs: Any) -> None:
        # initialize the paginator with the the author_id kwarg
        # and any other kwargs we passed to the constructor.
        # possible kwargs are as follows:
        # timeout: Optional[int] = None - the timeout for the paginator (view)
        # disable_after: bool = True - whether to disable all buttons after the paginator is finished or cancelled.
        # can_go_back: bool = True - whether the user can go back to previous modals using the "Previous" button.
        # sort_modals: bool = True - whether to sort the modals by the required kwarg.
        # See more on the class.
        super().__init__(author_id=author_id, **kwargs)
        # iterate over the questions_inputs list
        for data in questions_inputs:
            # unpack the data from the dict
            title: str = data["title"]
            required: bool = data["required"]
            questions: List[str] = data["questions"]
            # create a new modal with the title and required kwarg
            modal = PaginatorModal(title=title, required=required)
            # add the questions to the modal
            for question in questions:
                modal.add_input(
                    label=question,  # the label of the text input
                    min_length=2,  # the minimum length of the text input
                    max_length=200,  # the maximum length of the text input
                    # see the discord.py docs for more info on the other kwargs
                )

            # add the modal to the paginator
            self.add_modal(modal)

    # override the on_finish method to send the answers to the channel when the paginator is finished.
    async def on_finish(self, interaction: discord.Interaction[Any]) -> None:
        # create a list of answers
        # default format: **Modal Title**\nQuestion: Answer\nQuestion: Answer\n... etc
        answers: list[str] = []
        for modal in self.modals:
            prefix = f"**{modal.title}**\n"
            field: discord.ui.TextInput[Any]
            for field in modal.children:  # type: ignore
                prefix += f"{field.label}: {field.value}\n"

            answers.append(prefix)

        await interaction.response.send_message(f"Answers from {interaction.user.mention}:\n\n" + "\n\n".join(answers))

# define the prefix command.
@bot.command()
async def verify(ctx: commands.Context[commands.Bot]):
    # initialize the paginator with all the questions data we defined above in a list
    # and the author_id so that only the command invoker can use the paginator.
    questions_inputs = [personal_questions, misc_questions, reason_questions]
    paginator = VerifyModal(questions_inputs, author_id=ctx.author.id)

    # send the paginator to the current channel
    await paginator.send(ctx)

# run the bot
bot.run("...")

# run @bot_name verify in a channel with the bot to test the paginator


"""
    1. Imports and definitions
"""

# i) imports

import matplotlib.pyplot as plt
import torch
import pyro
import pyro.distributions as dist
from pyro.infer import SVI, Trace_ELBO
from pyro.optim import Adam
import seaborn as sns

"""
    2. Data & stochastic model
"""

# i) Generate data

true_mean = torch.tensor(4.0)
data = dist.Normal(true_mean, 1).sample([100])

# ii) Construct stochastic model

prior_dist = dist.Normal(0., 3.)
def model(data = None):
    # Prior over the mean
    mu = pyro.sample("mu", prior_dist)
    # Likelihood
    len_data = len(data) if data is not None else 1
    with pyro.plate("data", len_data):
        obs = pyro.sample("obs", dist.Normal(mu, 1), obs=data)
    return obs

# iii) construct variational distribution

def guide(data = None):
    # Variational parameters
    mu_loc = pyro.param("mu_loc", torch.tensor(0.))
    mu_scale = pyro.param("mu_scale", torch.tensor(1.), constraint=dist.constraints.positive)
    # Variational distribution over the mean
    mu_sample = pyro.sample("mu", dist.Normal(mu_loc, mu_scale))
    return mu_sample

"""
    3. SVI inference
"""

# i) Perform inference

pyro.clear_param_store()
svi = SVI(model, guide, Adam({"lr": 0.01}), loss=Trace_ELBO())
for _ in range(1000):
    svi.step(data)

# ii) Print out parameters and uncertainties

for name, param in pyro.get_param_store().items():
    print('{} : {}'.format(name, param))

"""
    4. Plots and illustrations
"""

# i) Sample from the prior and the posterior

prior_samples = torch.tensor([prior_dist.sample() for _ in range(1000)])
posterior_samples = torch.tensor([guide().detach() for _ in range(1000)])

#  ii) Sample from the model, from model with posterior swapped in

model_samples = torch.tensor([model().detach() for _ in range(1000)])

def model_with_posterior():
    # Posterior over the mean
    mu = guide()
    # Likelihood
    obs = pyro.sample("obs", dist.Normal(mu, 1))
    return obs
ppd_samples = torch.tensor([model_with_posterior().detach() for _ in range(1000)])

# iv) Plotting empirical distributions

fig, axs = plt.subplots(3, 2, figsize=(10, 15))  # 3x2 grid of Axes

# data
sns.histplot(data, ax=axs[0, 0], kde=True)
axs[0, 0].set_title('Actual data')
# model
sns.histplot(model_samples, ax=axs[1, 0], kde=True)
axs[1, 0].set_title('Model samples of data using prior distribution')
# ppd
sns.histplot(ppd_samples, ax=axs[1, 1], kde=True)
axs[1, 1].set_title('Model samples of data using posterior distribution (ppd)')
# mu prior
sns.histplot(prior_samples, ax=axs[2, 0], kde=True)
axs[2, 0].set_title('Prior samples of mu')
# mu posterior
sns.histplot(posterior_samples, ax=axs[2, 1], kde=True)
axs[2, 1].set_title('Posterior samples of mu given data')

plt.tight_layout()
plt.show()

predictive = pyro.infer.Predictive(model = model, guide = guide, num_samples = 1000)
predictive_samples = predictive(())

# v) Use predictive functionality to produce values for all sample sites based
# on the posterior density encoded in guide i.e. p(z|x) (samples of mu) and
# p(x'|x) = int p(x'|z)p(z|x) dz (samples of new obs).

fig, axs = plt.subplots(2, 2, figsize=(10, 10))  # 3x2 grid of Axes

# predictive mu
sns.histplot(predictive_samples['mu'], ax=axs[0, 0], kde=True)
axs[0, 0].set_title('Predictive samples mu')
# predictive obs
sns.histplot(predictive_samples['obs'], ax=axs[0, 1], kde=True)
axs[0, 1].set_title('Predictive samples obs')

plt.tight_layout()
plt.show()


import os
import shutil

declare_namespace_template = """
import pkg_resources
pkg_resources.declare_namespace(__name__)
"""

pkgutil_extend_path_template = """
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
"""

module_template = """
print ('this is module %s' % __name__)
"""

setup_template = """
from setuptools import setup, find_packages

setup(
    name='%(pkgname)s',
    version='0.1',
    description='A test package for name-spaces',
    zip_safe=%(zip_safe)r,
    packages=find_packages(),
    namespace_packages = %(namespace_packages)r
    )
"""

workdir = os.getcwd()
OLDPWD = os.getcwd()

def make_package(
    pkgname, namespace_packages, modules, zip_safe=False, declare_namespace_template=declare_namespace_template
):
    base = os.path.join(workdir, pkgname)
    if os.path.exists(base):
        shutil.rmtree(base)
    os.mkdir(base)
    os.chdir(base)
    # write __init__-files for each namespaced package
    for ns in namespace_packages:
        ns = os.path.join(*ns.split('.'))
        if not os.path.exists(ns):
            os.mkdir(ns)
        ns = os.path.join(ns, '__init__.py')
        with open(ns, 'w') as outfh:
            outfh.write(declare_namespace_template)
    # write the module itself
    for mod in modules:
        mod = os.path.join(*mod.split('/'))
        ns = os.path.dirname(mod)
        if not os.path.exists(ns):
            os.mkdir(ns)
        with open(mod, 'w') as outfh:
            outfh.write(module_template)
    # write the setup.py
    with open('setup.py', 'w') as outfh:
        outfh.write(setup_template % locals())

    os.chdir(OLDPWD)

# Collection of packages to be installed using
#   PYTHONPATH=. python setup.py install --install-lib .
# This will keep the __init__files of the namespace-packages.
make_package('nspkg1-aaa', ['nspkg1'], ['nspkg1/aaa/__init__.py'])
make_package('nspkg1-bbb', ['nspkg1', 'nspkg1.bbb'], ['nspkg1/bbb/zzz/__init__.py'], zip_safe=True)
make_package('nspkg1-ccc', ['nspkg1'], ['nspkg1/ccc.py'])
make_package('nspkg1-empty', ['nspkg1'], [], zip_safe=True)

# Collection of packages to be installed using
#   python setup.py install --install-lib . \
#     --single-version-externally-managed --record ./install.log
# This will omit the __init__files of the namespace-packages, but generate a -nspkg.pth file.
make_package('nspkg2-aaa', ['nspkg2'], ['nspkg2/aaa/__init__.py'])
make_package('nspkg2-bbb', ['nspkg2', 'nspkg2.bbb'], ['nspkg2/bbb/zzz/__init__.py'], zip_safe=True)
make_package('nspkg2-ccc', ['nspkg2'], ['nspkg2/ccc.py'])
make_package('nspkg2-empty', ['nspkg2'], [], zip_safe=True)

# Collection of packages to be installed using
#   PYTHONPATH=. python setup.py install --install-lib .
# This will keep the __init__files of the namespace-packages.
make_package(
    'nspkg3-a',
    # zipped egg in front of nspkg3-aaa!
    ['nspkg3', 'nspkg3.a'],
    ['nspkg3/a/__init__.py'],
    zip_safe=True,
    declare_namespace_template=pkgutil_extend_path_template,
)
make_package(
    'nspkg3-aaa',
    ['nspkg3'],
    ['nspkg3/aaa/__init__.py'],
    declare_namespace_template=pkgutil_extend_path_template,
)
make_package(
    'nspkg3-bbb',
    ['nspkg3', 'nspkg3.bbb'],
    ['nspkg3/bbb/zzz/__init__.py'],
    zip_safe=True,
    declare_namespace_template=pkgutil_extend_path_template,
)
make_package(
    'nspkg3-ccc',
    ['nspkg3'],
    ['nspkg3/ccc.py'],
    declare_namespace_template=pkgutil_extend_path_template,
)
make_package(
    'nspkg3-empty',
    ['nspkg3'],
    [],
    zip_safe=True,
    declare_namespace_template=pkgutil_extend_path_template,
)


import logging
import os
from pprint import pprint
from gensim import corpora, models, similarities
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)

if (os.path.exists("tmp/mesh.dict")):
   dictionary = corpora.Dictionary.load('tmp/mesh.dict')
   corpus = corpora.MmCorpus('tmp/corpus.mm')
   print("Used files generated from dict_corpus")
else:
   print("Please run dict_corpus.py to generate data set")

if (os.path.exists("tmp/model.tfidf")):
    tfidf = models.TfidfModel.load('tmp/model.tfidf')
    lsi = models.LsiModel.load('tmp/model.lsi')
    lda = models.LdaModel.load('tmp/model.lda')

new_doc = open('sample.txt', 'r', encoding = 'utf-8').read()
print(new_doc)

new_vec = dictionary.doc2bow(new_doc.lower().split())
pprint(new_vec)

vec_tfidf = tfidf[new_vec]
pprint(vec_tfidf)

vec_lsi = lsi[new_vec]
pprint(vec_lsi)

vec_lda = lda[new_vec]
pprint(vec_lda)


import core.constants as const
from core.aiml_sample import aiml_processing

def processing(data: str, kernel, input_memory, is_empty: bool, ans: str='') -> dict:

    if not is_empty:
        id_current, is_user_msg, answer = data.split(sep=';', maxsplit=2)
        id_current = int(id_current)
        is_user_msg = int(is_user_msg)
        if not is_user_msg:
            print(id_current, int(answer))
            number = const.MESSAGES[id_current]['buttons'][int(answer)-1]['text']
            ans, input_memory = aiml_processing(kernel, input_memory,
                                                number, ans=ans)
            id_next = const.TRANSITION_MAP[id_current][int(answer)]
            return ans, input_memory, const.MESSAGES[id_next]
        else:
            ans, input_memory  = aiml_processing(kernel, input_memory, answer, ans)
            id_message = int(ans.split(';')[0])

            return ans, input_memory, const.MESSAGES[id_message]
    else:
        id_current = int(data)
        id_next = const.TRANSITION_MAP[id_current][1]
        return ans, input_memory, const.MESSAGES[id_next]


from flask import request, jsonify, Blueprint
from flask_jwt_extended import jwt_required, current_user
from models.Containers_model import Container
from models.Image_model import Image
from models.db import db
from controller import Containers_controller as containerctl
from utils import check_authorization

container_bp = Blueprint('container_bp', __name__)

@container_bp.route('/images', methods=["GET"])
@jwt_required()
def get_images_by_user():
    images = Image.query.filter_by(author_id=current_user.id).all()
    print(images)
    responses = [{
            "docker_id": image.docker_id,
            "name": image.name,
            "createdAt": image.createdAt,
            "author": image.author.name
        } for image in images]

    return jsonify(responses), 200

@container_bp.route('/image/<id>', methods=["GET"])
@jwt_required()
def get_image_by_id(id):
    image = Image.query.filter_by(docker_id=id).first()
    response = {
            "docker_id": image.docker_id,
            "name": image.name,
            "createdAt": image.createdAt,
            "author": image.author.name
        }
    return jsonify(response), 200

@container_bp.route('/image/<id>', methods=["DELETE"])
@jwt_required()
def delete_img(id):
    image = Image.query.filter_by(docker_id=id).first()
    if not image:
        return jsonify({"message": "Image with the given ID is not found"}), 404
    auth_check = check_authorization(image.author_id, current_user.id)
    if not auth_check:
        return jsonify({"success": False, "message": "Unauthorized"}), 401
    ok, message = containerctl.delete_image(id, True)
    if not ok:
        return jsonify({"success": ok, "message": message}), 500
    db.session.delete(image)
    db.session.commit()
    return jsonify({"success": ok, "message": message}), 200

@container_bp.route('/image', methods=["POST"])
@jwt_required()
def create_image():
    body = request.get_json()
    print(current_user)
    result = containerctl.build_image(
        current_user.email.split("@")[0].lower(),
        body['baseImg'],
        body['packages'],
        body['desktop'],
        body['username'],
        body['password'],
        body['timezone'],
        body.get('memory', 4096)
    )
    # Insert into Image Table
    new_image = Image(
        name=result[1],
        docker_id=result[2],
        author_id=current_user.id
    )
    try:
        db.session.add(new_image)
        db.session.commit()
        return jsonify({"success": result[0], "image_id": result[2], "image_name": result[1]}), 201
    except Exception as e:
        db.session.rollback()
        return jsonify({"success": False, "message": str(e)}), 400

@container_bp.route('/containers', methods=["GET"])
@jwt_required()
def get_container_by_user():
    containers = Container.query.filter_by(author_id=current_user.id).all()
    responses = [{
        "container_name": container.container_name,
        "docker_id": container.docker_id,
        "author": container.author.name,
        "subdomain": container.subdomain,
        "vnc_port": container.vnc_port,
        "vnc_host": container.vnc_host,
        "vnc_url": container.vnc_url,
        "ssh_port": container.ssh_port,
        "ws_port": container.ws_port,
        "state": container.state,
        "image_name": container.image.name,
        "image_author": container.image.author.name,
        "createdAt": container.createdAt
    } for container in containers]
    print(containers)
    return jsonify(responses), 200

@container_bp.route('/container', methods=["POST"])
@jwt_required()
def create_and_run_container():
    body = request.get_json()
    image_name = body['image_name']
    result = containerctl.run_container(
        current_user.email.split("@")[0].lower(),
        image_name
    )

    hostname = request.host.split(":")[0]
    subdomain = f"{result[3]}.{hostname}"

    # Insert into Container Table
    new_container = Container(
        docker_id=result[1],
        vnc_port=result[2],
        author_id=current_user.id,
        state=result[4],
        container_name=result[3],
        vnc_host=hostname,
        subdomain=subdomain,
        image_id=result[5],
        ssh_port=result[7],
        ws_port=result[6],
        vnc_url=f"ws://{hostname}:{result[6]}"
    )
    try:
        db.session.add(new_container)
        db.session.commit()
        return jsonify({"success": result[0], "container_id": result[1], "vnc_host": request.host_url.replace("http://", "").replace("/:5000", ""), "vnc_port": result[2], "container_name": result[3], "status": result[4], "ssh_port":result[7],
        "ws_port":result[6]}), 201
    except Exception as e:
        db.session.rollback()
        return jsonify({"success": False, "message": str(e)}), 400

@container_bp.route('/container/<container_id>', methods=["GET"])
@jwt_required()
def get_container_by_id(container_id):
    container = Container.query.filter_by(docker_id=container_id).first()
    if not container:
        return jsonify({"message": "Container with the given ID is not found"}), 404
    check = check_authorization(container.author_id, current_user.id)
    if not check:
        return jsonify({"message": "Unauthorized"}), 401

    logs = containerctl.get_logs(container.docker_id)
    response = {
        "container_name": container.container_name,
        "docker_id": container.docker_id,
        "author": container.author.name,
        "subdomain": container.subdomain,
        "vnc_port": container.vnc_port,
        "vnc_host": container.vnc_host,
        "vnc_url": container.vnc_url,
        "ssh_port": container.ssh_port,
        "ws_port": container.ws_port,
        "state": container.state,
        "image_name": container.image.name,
        "image_author": container.image.author.name,
        "createdAt": container.createdAt,
        "logs": logs.decode()
    }
    return jsonify(response), 200

@container_bp.route('/container/<container_id>', methods=["PUT"])
@jwt_required()
def stop_container(container_id):
    container_check = Container.query.filter_by(docker_id=container_id).first()
    if not container_check:
        return jsonify({"message": "Container with the given ID is not found"}), 404
    auth_check = check_authorization(container_check.author_id, current_user.id)
    if not auth_check:
        return jsonify({"message": "Unauthorized"}), 401
    ok = containerctl.stop_container(container_id)
    container = Container.query.filter_by(docker_id=container_id).first()
    container.state = "Stopped"
    db.session.commit()
    return jsonify({"success": ok}), 200

@container_bp.route('/container/<container_id>', methods=["DELETE"])
@jwt_required()
def delete_container(container_id):
    container_check = Container.query.filter_by(docker_id=container_id).first()
    if not container_check:
        return jsonify({"message": "Container with the given ID is not found"}), 404
    auth_check = check_authorization(container_check.author_id, current_user.id)
    if not auth_check:
        return jsonify({"message": "Unauthorized"}), 401
    ok = containerctl.delete_container(container_id)
    container = Container.query.filter_by(docker_id=container_id).first()
    db.session.delete(container)
    db.session.commit()
    return jsonify({"success": ok}), 200


"""
Microdeploy Configuration manager.
"""

import yaml
import glob
import re
import os

class Config(object):

    # config = None

    def __init__(self, config_filename=None, default_baudrate=115200, override={}):

        if config_filename:
            try:
                with open(config_filename) as config_file:
                    config_yaml = yaml.load(config_file.read(), yaml.Loader)
            except FileNotFoundError as e:
                raise FileNotFoundError(f"Config file not found: '{config_filename}'")
            except yaml.scanner.ScannerError as e:
                message = str(f'\t{e}').replace('\n', '\n\t')  # indent message from ScannerError
                raise yaml.scanner.ScannerError(f"Error in config file: {config_filename}\n\n{message}")
        else:
            config_file = None
            config_yaml = {}

        self.config = {
            'packages': config_yaml.get('packages', {}),
            'device': config_yaml.get('device', {}),
            'default': {
                # 'destination': config_yaml.get('default', {}).get('destination') or '/' or None,  # default path of destination for put files to MCU
                'baudrate': config_yaml.get('default', {}).get('baudrate') or default_baudrate}}

        dict_update(self.config, override)
        self.config_filename = config_filename

    def device(self) -> dict:
        """Return device configuration, applying `*overrides`."""
        device = self.config.get('device', {})
        device = dict(device)
        return {
            'port': device.get('port'),
            'baudrate': device.get('baudrate', self.config['default']['baudrate'])}

    def package(self, name: str) -> dict:
        """Return package configuration dict."""
        try:
            package = self.config.get('packages', {})[name]
        except KeyError as e:
            KeyError(f"Package not found: {name}  - packages available: {', '.join(self.config.get('packages', {}).keys())}")

        try:
            ignore = set()
            for package_to_include in package.get('include', []):
                to_ignore = self.config.get('packages', {})[package_to_include].get('ignore', [])
                if to_ignore:
                    ignore.update(to_ignore)
                    package['ignore'] = list(ignore)
        except KeyError as e:
            raise KeyError(f'Package not found: {package_to_include} - in {name}.include')

        return package

    def package_files(self, name: str) -> list:
        """Return list of files in package having `name` (processing includes)."""
        try:
            package_config = self.package(name)
        except KeyError as e:
            raise ValueError(f"Package not found: {name}  - packages available: {', '.join(self.config.get('packages', {}).keys())}")

        def ignore(filename):
            return any(re.search(ignored, filename) for ignored in package_config.get('ignore', []))

        package_files = []
        for file_desc in package_config.get('files', []):
            if type(file_desc) is not str and not len(file_desc) == 2:
                raise ValueError("File definition must be string or tuple, eg. 'main.py' or ('source.py', 'destination.py')")
            source, destination = [file_desc, None] if type(file_desc) is str else file_desc
            source_relative = self.make_relative_to_configfile(source)
            if ignore(source_relative):
                continue
            if '*' not in source_relative:
                package_files.append((source_relative, destination or source))
            else:
                for source_file in glob.iglob(source_relative, recursive=True):  # Note: this allows wildcards in source files, eg. 'tests/*.py' or 'tests/**/*.py``
                    if os.path.isdir(source_file):
                        continue
                    if ignore(source_file):
                        continue
                    if destination is not None:
                        destination_file = destination
                    else:
                        relative_path = os.path.relpath(os.path.dirname(self.config_filename) or '.')
                        destination_file = source_file[1+len(relative_path):]
                    package_files.append((source_file, destination_file))

        for package_to_include in package_config.get('include', []):
            try:
                package_files = self.package_files(package_to_include) + package_files
            except ValueError as e:
                raise KeyError(f'Package not found: {package_to_include} - in {name}.include')

        return package_files

    def make_relative_to_configfile(self, filename):
        """Return `filename` made relative to config file path."""
        return os.path.join(os.path.relpath(os.path.dirname(self.config_filename) or '.'), filename)

class Configurable(object):
    """
    Abstract class base for components (ie. device, package).
    """
    def __init__(self, config: Config):
        self.config = config

# Helpers

def dict_update(d1, d2):
    """
    Update `d1` with `d2` deeply.
    """
    for k in d2:
        if k in d1 and isinstance(d1[k], dict) and isinstance(d2[k], dict):
            dict_update(d1[k], d2[k])
        else:
            d1[k] = d2[k]


from setuptools import setup

with open("README.md", "r") as fh:
    long_description = fh.read()

setup(name='immudb-py',
      version='1.4.0',
      license="Apache License Version 2.0",
      description='Python SDK for Immudb',
      long_description=long_description,
      long_description_content_type="text/markdown",
      author='Codenotary',
      url='https://github.com/codenotary/immudb-py',
      # download_url='',
      packages=['immudb', 'immudb.database', 'immudb.embedded',
                'immudb.embedded.ahtree', 'immudb.embedded.htree', 'immudb.embedded.store',
                'immudb.grpc', 'immudb.handler', 'immudb.schema'],
      keywords=['immudb', 'immutable'],
      install_requires=[
          'grpcio>=1.31.0',
          'dataclasses>=0.6',
          'protobuf>=3.13.0,<4.0.0',
          'google-api>=0.1.12',
          'google-api-core>=1.22.1',
          'ecdsa>=0.16.1'
      ],
      classifiers=[
          'Intended Audience :: Developers',
          'Topic :: Software Development :: Build Tools',
          "License :: OSI Approved :: Apache Software License",
          "Operating System :: OS Independent",
          'Programming Language :: Python :: 3',
          'Programming Language :: Python :: 3.6',
          'Programming Language :: Python :: 3.7',
          'Programming Language :: Python :: 3.8',
      ],
      python_requires='>=3.6',
      )


# -*-coding: utf-8 -*-
# Python 3.6
# Author:Zhang Haitao
# Email:13163385579@163.com
# TIME:2018-08-15  16:04
# NAME:zht-wind_api.py
import datetime
import multiprocessing
import pickle

from WindPy import w
from utils.dateu import get_today
import numpy as np

DIR=r'e:\tmp_wind'

w.start()

import pandas as pd
import os

DATE_FORMAT='%Y-%m-%d'

def multi_task(func, args_iter, n=8):
    pool=multiprocessing.Pool(n)
    results=pool.map(func, args_iter)
    pool.close()#trick: close the processing every time the pool has finished its task, and pool.close() must be called before pool.join()
    pool.join()
    #refer to https://stackoverflow.com/questions/38271547/when-should-we-call-multiprocessing-pool-join
    return results

def get_stkcd_list():
    today = get_today(DATE_FORMAT)
    path=os.path.join(DIR,'stkcd_list_{}.pkl'.format(today))
    if os.path.exists(path):
        return pickle.load(open(path,'rb'))
    else:
        codes=w.wset("SectorConstituent",u"date={};sector=全部A股".format(today)).Data[1]
        f=open(path,'wb')
        pickle.dump(codes,f)
        return codes

def get_ann_dt():
    today=get_today(DATE_FORMAT)
    path=os.path.join(DIR,'ann_dt_{}.pkl'.format(today))
    if os.path.exists(path):
        return pickle.load(open(path,'rb'))
    else:
        codes=get_stkcd_list()
        result=w.wsd(','.join(codes), "stm_issuingdate", "ED-5Q", get_today(), "Period=Q;Days=Alldays")
        ann_dt=pd.DataFrame(result.Data,index=result.Codes,columns=result.Times).T
        f=open(path,'wb')
        pickle.dump(ann_dt,f)
        return ann_dt

def get_last_ann_dt():
    anns=[an for an in os.listdir(os.path.join(DIR)) if an.startswith('ann_dt') and an[7:-4] !=get_today()]
    anns=sorted(anns,key=lambda x:pd.to_datetime(x[7:-4]))
    return pd.read_pickle(os.path.join(DIR,anns[-1]))

def get_data_for_one_stk(args):
    stkcd, rpdate, indicators=args
    data = w.wsd(stkcd, ','.join(indicators), rpdate, rpdate, "unit=1;rptType=1;Period=Q;Days=Alldays").Data
    s = pd.Series([d[0] for d in data], index=indicators)
    s.fillna(value=np.nan, inplace=True)
    s.name=stkcd
    print('Getting data for {}-----{}'.format(rpdate,stkcd))
    return s

def get_new_df():
    ann_dt=get_ann_dt()
    last_ann_dt=get_last_ann_dt()

    new_index=ann_dt.index.union(last_ann_dt.index)
    new_column=ann_dt.columns.union(last_ann_dt.columns)

    ann_dt=ann_dt.reindex(index=new_index,columns=new_column)
    last_ann_dt=last_ann_dt.reindex(index=new_index,columns=new_column)
    new_df=ann_dt[ann_dt!=last_ann_dt]
    return new_df

def get_new_data(ann_dt_df):
    name_df = pd.read_excel(os.path.join(DIR,r'indicators_name.xlsx'), sheet_name='financial')
    dfs = []
    for date, row in ann_dt_df.iterrows():
        row = row.dropna()
        row = row.apply(lambda x: x.strftime(DATE_FORMAT))
        rpdate = row.name.strftime(DATE_FORMAT)
        stkcds = row.index.tolist()
        indicators = name_df['wind_name'].tolist()
        # _df=pd.concat(multi_task(get_data_for_one_stk,((stkcd, rpdate, indicators) for stkcd in stkcds)),axis=1).T
        _df = pd.concat([get_data_for_one_stk((stkcd, rpdate, indicators)) for stkcd in stkcds], axis=1).T
        _df['rpdate'] = rpdate
        _df.set_index('rpdate', append=True, inplace=True)
        _df.index.names = ['stkcd', 'rpdate']
        # df.to_pickle(os.path.join(DIR,'data_{}.pkl'.format(get_today())))
        dfs.append(_df)
    df = pd.concat(dfs)
    return df

# today_updated.to_pickle(os.path.join(DIR, 'today_updated_{}.pkl'.format(get_today())))

#TODO: how to combine the old and the new dataframe

# codes = get_stkcd_list()
# result = w.wsd(','.join(codes), "stm_issuingdate", "ED-5Q", get_today(), "Period=Q;Days=Alldays")
# ann_dt = pd.DataFrame(result.Data, index=result.Codes, columns=result.Times).T
#
# ann_dt.to_pickle(os.path.join(DIR,'ann_dt_{}.pkl'.format(get_today())))
#
# test_ann_dt=ann_dt.iloc[:,30]
#
# df=get_new_data(ann_dt)

def test():
    ann_dt=pd.read_pickle(os.path.join(DIR,'ann_dt_{}.pkl'.format(get_today())))
    test_ann_dt=ann_dt.iloc[:,:30]
    df=get_new_data(test_ann_dt)
    df.to_pickle('test_df.pkl')

if __name__ == '__main__':
    test()

#TODO: logger and multiprocessing