import os

from extension.constants import ENV_OPTION_PREFIX
from extension.interface import ExtensionModules

class MockExtensionModules(ExtensionModules):
    def inputs(self):
        return []

    def outputs(self):
        return []

    def generate_inputs(self, data):
        pass

    def generate_outputs(self, data):
        pass

def test_properties(tmp_path):
    name = 'foo'
    root = str(tmp_path)
    metadata = {}
    config = {}

    builder = MockExtensionModules(name, root, metadata, config)

    assert builder.name is name
    assert builder.root is root
    assert builder.metadata is metadata
    assert builder.config is config

def test_get_env_option(tmp_path):
    builder = MockExtensionModules('foo', str(tmp_path), {}, {})
    option = os.urandom(4).hex()
    value = '9000'
    env_var = f'{ENV_OPTION_PREFIX}{builder.name}_{option}'.upper()

    assert builder.get_env_option(option) == ''

    os.environ[env_var] = value
    try:
        assert builder.get_env_option(option) == value
    finally:
        del os.environ[env_var]

    assert builder.get_env_option(option) == ''

def test_required_methods(tmp_path):
    # This is just for code coverage
    builder = MockExtensionModules('foo', str(tmp_path), {}, {})

    builder.inputs()
    builder.outputs()
    builder.generate_inputs({})
    builder.generate_outputs({})


import keras
import keras_cv
import keras_core as keras
import tensorflow as tf

images = tf.ones(shape=(1, 512, 512, 3))
labels = {
    "boxes": [
        [
            [0, 0, 100, 100],
            [100, 100, 200, 200],
            [300, 300, 100, 100],
        ]
    ],
    "classes": [[1, 1, 1]],
}
model = keras_cv.models.YOLOV8Detector(
    num_classes=20,
    bounding_box_format="xywh",
    backbone=keras_cv.models.YOLOV8Backbone.from_preset(
        "yolo_v8_m_backbone_coco"
    ),
    fpn_depth=2.
)

# Evaluate model
model(images)

# Get predictions using the model
model.predict(images)

# Train model
model.compile(
    classification_loss='binary_crossentropy',
    box_loss='ciou',
    optimizer=tf.optimizers.SGD(global_clipnorm=10.0),
    jit_compile=False,
)
model.fit(images, labels)


import tests.hakoblog  # noqa: F401

from hakoblog.db import DB
from hakoblog.loader.user import UserLoader
from hakoblog.action.blog import BlogAction
from hakoblog.loader.blog import BlogLoader

from tests.util import random_string, create_user, global_user

def test_create():
    db = DB()

    user = create_user()
    title = random_string(10)
    blog_id = BlogAction.create(
        db,
        owner_id=user.id,
        title=title,
    )
    found_blog = BlogLoader.find_by_id(db, blog_id)

    assert found_blog.id == blog_id
    assert found_blog.owner_id == user.id
    assert found_blog.title == title

def test_ensure_global_blog_created():
    db = DB()

    with global_user(random_string(10)) as global_user_name:
        assert UserLoader.find_by_name(db, global_user_name) is None

        blog = BlogAction.ensure_global_blog_created(db)

        found_user = UserLoader.find_by_name(db, global_user_name)
        assert blog.owner_id == found_user.id

        # Check no exeception raises
        blog_again = BlogAction.ensure_global_blog_created(db)
        assert blog_again.id == blog.id


import heapq
from typing import List

def topKFrequent(nums: List[int], k: int) -> List[int]: # Verified on Leetcode
    frequencies = {}
    for num in nums:
        if num not in frequencies:
            frequencies[num] = 1
        else:
            frequencies[num] += 1

    temp = []
    for num, f in frequencies.items():
        temp.append((f, num))

    min_heap = temp[:k]
    heapq.heapify(min_heap)

    for item in temp[k:]:
        if item[0] > min_heap[0][0]:
            heapq.heapreplace(min_heap, item)

    return list(map(lambda x: x[1], min_heap))

if __name__ == "__main__":
    print(topKFrequent([1, 1, 1, 2, 2, 3], 2))


from typing import Iterable, Tuple, TypeVar, Callable, Any, List, Dict, Union
import math
import numpy as np
import os.path
import torch
import torchaudio
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader
import warnings
import pandas as pd

import plots
from utils import validate_audio

# Useful references for the dataloading using iterable datasets:
# https://medium.com/speechmatics/how-to-build-a-streaming-dataloader-with-pytorch-a66dd891d9dd
# https://pytorch.org/docs/stable/data.html#torch.utils.data.IterableDataset
# https://discuss.pytorch.org/t/example-for-torch-utils-data-iterabledataset/101175/13
# https://github.com/pytorch/pytorch/issues/13246#issuecomment-905703662

# https://discuss.pytorch.org/t/implementing-an-infinite-loop-dataset-dataloader-combo/35567/4

def interpolate(x, ratio):
    '''
    Interpolate the x to have equal time steps as targets
    Input:
        x: (batch_size, time_steps, class_num)
    Output:
        out: (batch_size, time_steps*ratio, class_num)
    '''
    (batch_size, time_steps, classes_num) = x.shape
    upsampled = x[:, :, None, :].repeat(1, 1, ratio, 1)
    upsampled = upsampled.reshape(batch_size, time_steps * ratio, classes_num)

    return upsampled

def convert_output_format_polar_to_cartesian(in_dict):
    out_dict = {}
    for frame_cnt in in_dict.keys():
        if frame_cnt not in out_dict:
            out_dict[frame_cnt] = []
            for tmp_val in in_dict[frame_cnt]:
                ele_rad = tmp_val[3] * np.pi / 180.
                azi_rad = tmp_val[2] * np.pi / 180

                tmp_label = np.cos(ele_rad)
                x = np.cos(azi_rad) * tmp_label
                y = np.sin(azi_rad) * tmp_label
                z = np.sin(ele_rad)
                out_dict[frame_cnt].append([tmp_val[0], tmp_val[1], x, y, z])
    return out_dict

def _read_audio(fname: str, directory_root: str, resampler: Union[torch.nn.Sequential, None], trim_seconds: int = -1) -> Tuple[torch.Tensor, int, float]:
    ''' trim_seconds = to limit how many seconds to load '''
    fpath = os.path.join(directory_root,
                         fname)
    metadata = torchaudio.info(fpath)
    num_frames = trim_seconds if trim_seconds == -1 else trim_seconds * metadata.sample_rate
    this_audio, fs = torchaudio.load(fpath, num_frames=num_frames)
    duration_seconds = this_audio.shape[-1] / fs
    assert validate_audio(this_audio), f'ERROR: {fname}  audio is not valid.'

    if resampler is not None:
        this_audio = resampler(this_audio)

    return torch.tensor(this_audio, dtype=torch.float), fs, duration_seconds

def _read_time_array(fname: str, directory_root: str) -> List:
    ''' Time arrays are the full list of events for a whole audio file.
    This is before any parsing'''
    fpath = os.path.join(directory_root,
                         fname)
    fpath_csv = fpath.replace('mic', 'metadata').replace('foa', 'metadata').replace('wav', 'csv')
    this_time_array = pd.read_csv(fpath_csv, header=None).values

    return this_time_array

def load_output_format_file(fname: str, directory_root: str):
    """
    Adapted from the official baseline.
    Loads DCASE output format csv file and returns it in dictionary format

    :param _output_format_file: DCASE output format CSV
    :return: _output_dict: dictionary
    """

    fpath = os.path.join(directory_root,
                         fname)
    fpath_csv = fpath.replace('mic', 'metadata').replace('foa', 'metadata').replace('wav', 'csv')

    _output_dict = {}
    _fid = open(fpath_csv, 'r')
    # next(_fid)
    for _line in _fid:
        _words = _line.strip().split(',')
        _frame_ind = int(_words[0])
        if _frame_ind not in _output_dict:
            _output_dict[_frame_ind] = []
        if len(_words) == 5: #polar coordinates
            _output_dict[_frame_ind].append([int(_words[1]), int(_words[2]), float(_words[3]), float(_words[4])])
        elif len(_words) == 6: # cartesian coordinates
            _output_dict[_frame_ind].append([int(_words[1]), int(_words[2]), float(_words[3]), float(_words[4]), float(_words[5])])
    _fid.close()
    return _output_dict

def _add_rotated_label_each_frame(label, time_array4frame_event, start_frame, rotation_pattern=None):
    """ From Sony """
    event_class = time_array4frame_event[1]

    azi_rad = time_array4frame_event[3] / 180 * np.pi
    ele_rad = time_array4frame_event[4] / 180 * np.pi
    if rotation_pattern:
        azi_reflection, azi_rotation, ele_reflection = rotation_pattern
    else:
        azi_reflection, azi_rotation, ele_reflection = [1, 0, 1]  # if None, no rotation
    rotated_azi_rad = azi_reflection * azi_rad + azi_rotation
    rotated_ele_rad = ele_reflection * ele_rad
    x_axis = 1 * np.cos(rotated_ele_rad) * np.cos(rotated_azi_rad)
    y_axis = 1 * np.cos(rotated_ele_rad) * np.sin(rotated_azi_rad)
    z_axis = 1 * np.sin(rotated_ele_rad)

    label[0, event_class, start_frame: start_frame + 10] = x_axis
    label[1, event_class, start_frame: start_frame + 10] = y_axis
    label[2, event_class, start_frame: start_frame + 10] = z_axis

    return (label)

def _get_labels(time_array, start_sec, fs, chunk_size_audio, rotation_pattern=None, multi_track=False, num_classes=12):
    """
    [frame number (int)], [active class index (int)], [track number index (int)], [azimuth (int)], [elevation (int)]
    Frame, class, and track enumeration begins at 0. Frames correspond to a temporal resolution of 100msec.
    Azimuth and elevation angles are given in degrees, rounded to the closest integer value, with azimuth and
    elevation being zero at the front, azimuth ϕ∈[−180∘,180∘], and elevation θ∈[−90∘,90∘]. Note that the azimuth
    angle is increasing counter-clockwise (ϕ=90∘ at the left).
    """

    # This 100 is the sampling frequency of the labels
    # And the 10 for index_diff stuff, is the desired sampling frequency, to match the spectrograms.
    # So the spectrograms use a step_size = 240, with fs = 24000, which is 10 ms
    # Therefore, here they have 100 / 10 = 10
    # My intuition is that a different step_size, would require to change this

    # TODO Is this really ok? Needs verification
    num_axis = 3  # X, Y, Z
    num_class = num_classes
    num_frame = round(chunk_size_audio / fs * 100) + 1  # Each frame == 100 ms (0.1 seconds)
    label = np.zeros([num_axis, num_class, num_frame])

    end_sec = start_sec + chunk_size_audio / fs

    index_diff = int(math.modf(start_sec * 10)[0] * 10)  # get second decimal place
    num_frame_wide = (int(np.ceil(end_sec * 10)) - int(np.floor(start_sec * 10)) + 1) * 10
    # "+ 1" is buffer for numerical error, such as index_diff=3 and num_frame_wide=130

    if not multi_track:
        label_wide = np.zeros([num_axis, num_class, num_frame_wide])
        for index, frame in enumerate(range(int(np.floor(start_sec * 10)), int(np.ceil(end_sec * 10)))):
            time_array4frame = time_array[time_array[:, 0] == frame]
            if time_array4frame.shape == (1, 5):
                label_wide = _add_rotated_label_each_frame(label_wide, time_array4frame[0], index * 10,
                                                           rotation_pattern)
            elif time_array4frame.shape == (2, 5):
                label_wide = _add_rotated_label_each_frame(label_wide, time_array4frame[0], index * 10,
                                                           rotation_pattern)
                label_wide = _add_rotated_label_each_frame(label_wide, time_array4frame[1], index * 10,
                                                           rotation_pattern)
            elif time_array4frame.shape == (3, 5):
                label_wide = _add_rotated_label_each_frame(label_wide, time_array4frame[0], index * 10,
                                                           rotation_pattern)
                label_wide = _add_rotated_label_each_frame(label_wide, time_array4frame[1], index * 10,
                                                           rotation_pattern)
                label_wide = _add_rotated_label_each_frame(label_wide, time_array4frame[2], index * 10,
                                                           rotation_pattern)
        label = label_wide[:, :, index_diff: index_diff + num_frame]
    else:
        # TODO This is not ready
        label_wide_1 = np.zeros([num_axis, num_class, num_frame_wide])
        label_wide_2 = np.zeros([num_axis, num_class, num_frame_wide])
        label_wide_3 = np.zeros([num_axis, num_class, num_frame_wide])
        for index, frame in enumerate(range(int(np.floor(start_sec * 10)), int(np.ceil(end_sec * 10)))):
            time_array4frame = time_array[time_array[:, 0] == frame]
            if time_array4frame.shape == (1, 5):
                label_wide_1 = _add_rotated_label_each_frame(label_wide_1, time_array4frame[0], index * 10,
                                                             rotation_pattern)
            elif time_array4frame.shape == (2, 5):
                label_wide_1 = _add_rotated_label_each_frame(label_wide_1, time_array4frame[0], index * 10,
                                                             rotation_pattern)
                label_wide_2 = _add_rotated_label_each_frame(label_wide_2, time_array4frame[1], index * 10,
                                                             rotation_pattern)
            elif time_array4frame.shape == (3, 5):
                label_wide_1 = _add_rotated_label_each_frame(label_wide_1, time_array4frame[0], index * 10,
                                                             rotation_pattern)
                label_wide_2 = _add_rotated_label_each_frame(label_wide_2, time_array4frame[1], index * 10,
                                                             rotation_pattern)
                label_wide_3 = _add_rotated_label_each_frame(label_wide_3, time_array4frame[2], index * 10,
                                                             rotation_pattern)
        label = np.stack((
            label_wide_1[:, :, index_diff: index_diff + num_frame],
            label_wide_2[:, :, index_diff: index_diff + num_frame],
            label_wide_3[:, :, index_diff: index_diff + num_frame]
        ))
    return (label)

def _read_fnames(directory_root: str, list_dataset: str) -> List:
    """Reads the fnames in the list_dataset.
    Each fname corresponds to a single wav file in the dataset.
    This to prepare the dataset, before loading any audio or labels."""
    fnames = []
    fpath = os.path.join(directory_root,
                         'list_dataset',
                         list_dataset)
    for fname in pd.read_table(fpath, header=None).values.tolist():
        if isinstance(fname, List): fname = fname[0]

        parent_dir = directory_root.split('/')[-1] + '/'
        if parent_dir in fname:
            fname = fname.replace(parent_dir, '')
        fnames.append(fname)
    return fnames

def get_adpit_labels_for_file(_desc_file: Dict, _nb_label_frames: int, num_classes: int = 13) -> np.ndarray:
    """

    ADAPATED from csl_feature_class from the baseline, with modifications to remove the dependcy to the class.

    Reads description file and returns classification based SED labels and regression based DOA labels
    for multi-ACCDOA with Auxiliary Duplicating Permutation Invariant Training (ADPIT)

    :param _desc_file: metadata description file
    :return: label_mat: of dimension [nb_frames, 6, 4(=act+XYZ), max_classes]
    """

    se_label = np.zeros((_nb_label_frames, 6, num_classes))  # [nb_frames, 6, max_classes]
    x_label = np.zeros((_nb_label_frames, 6, num_classes))
    y_label = np.zeros((_nb_label_frames, 6, num_classes))
    z_label = np.zeros((_nb_label_frames, 6, num_classes))

    for frame_ind, active_event_list in _desc_file.items():
        if frame_ind < _nb_label_frames:
            active_event_list.sort(key=lambda x: x[0])  # sort for ov from the same class
            active_event_list_per_class = []
            for i, active_event in enumerate(active_event_list):
                active_event_list_per_class.append(active_event)
                if i == len(active_event_list) - 1:  # if the last
                    if len(active_event_list_per_class) == 1:  # if no ov from the same class
                        # a0----
                        active_event_a0 = active_event_list_per_class[0]
                        se_label[frame_ind, 0, active_event_a0[0]] = 1
                        x_label[frame_ind, 0, active_event_a0[0]] = active_event_a0[2]
                        y_label[frame_ind, 0, active_event_a0[0]] = active_event_a0[3]
                        z_label[frame_ind, 0, active_event_a0[0]] = active_event_a0[4]
                    elif len(active_event_list_per_class) == 2:  # if ov with 2 sources from the same class
                        # --b0--
                        active_event_b0 = active_event_list_per_class[0]
                        se_label[frame_ind, 1, active_event_b0[0]] = 1
                        x_label[frame_ind, 1, active_event_b0[0]] = active_event_b0[2]
                        y_label[frame_ind, 1, active_event_b0[0]] = active_event_b0[3]
                        z_label[frame_ind, 1, active_event_b0[0]] = active_event_b0[4]
                        # --b1--
                        active_event_b1 = active_event_list_per_class[1]
                        se_label[frame_ind, 2, active_event_b1[0]] = 1
                        x_label[frame_ind, 2, active_event_b1[0]] = active_event_b1[2]
                        y_label[frame_ind, 2, active_event_b1[0]] = active_event_b1[3]
                        z_label[frame_ind, 2, active_event_b1[0]] = active_event_b1[4]
                    else:  # if ov with more than 2 sources from the same class
                        # ----c0
                        active_event_c0 = active_event_list_per_class[0]
                        se_label[frame_ind, 3, active_event_c0[0]] = 1
                        x_label[frame_ind, 3, active_event_c0[0]] = active_event_c0[2]
                        y_label[frame_ind, 3, active_event_c0[0]] = active_event_c0[3]
                        z_label[frame_ind, 3, active_event_c0[0]] = active_event_c0[4]
                        # ----c1
                        active_event_c1 = active_event_list_per_class[1]
                        se_label[frame_ind, 4, active_event_c1[0]] = 1
                        x_label[frame_ind, 4, active_event_c1[0]] = active_event_c1[2]
                        y_label[frame_ind, 4, active_event_c1[0]] = active_event_c1[3]
                        z_label[frame_ind, 4, active_event_c1[0]] = active_event_c1[4]
                        # ----c2
                        active_event_c2 = active_event_list_per_class[2]
                        se_label[frame_ind, 5, active_event_c2[0]] = 1
                        x_label[frame_ind, 5, active_event_c2[0]] = active_event_c2[2]
                        y_label[frame_ind, 5, active_event_c2[0]] = active_event_c2[3]
                        z_label[frame_ind, 5, active_event_c2[0]] = active_event_c2[4]

                elif active_event[0] != active_event_list[i + 1][0]:  # if the next is not the same class
                    if len(active_event_list_per_class) == 1:  # if no ov from the same class
                        # a0----
                        active_event_a0 = active_event_list_per_class[0]
                        se_label[frame_ind, 0, active_event_a0[0]] = 1
                        x_label[frame_ind, 0, active_event_a0[0]] = active_event_a0[2]
                        y_label[frame_ind, 0, active_event_a0[0]] = active_event_a0[3]
                        z_label[frame_ind, 0, active_event_a0[0]] = active_event_a0[4]
                    elif len(active_event_list_per_class) == 2:  # if ov with 2 sources from the same class
                        # --b0--
                        active_event_b0 = active_event_list_per_class[0]
                        se_label[frame_ind, 1, active_event_b0[0]] = 1
                        x_label[frame_ind, 1, active_event_b0[0]] = active_event_b0[2]
                        y_label[frame_ind, 1, active_event_b0[0]] = active_event_b0[3]
                        z_label[frame_ind, 1, active_event_b0[0]] = active_event_b0[4]
                        # --b1--
                        active_event_b1 = active_event_list_per_class[1]
                        se_label[frame_ind, 2, active_event_b1[0]] = 1
                        x_label[frame_ind, 2, active_event_b1[0]] = active_event_b1[2]
                        y_label[frame_ind, 2, active_event_b1[0]] = active_event_b1[3]
                        z_label[frame_ind, 2, active_event_b1[0]] = active_event_b1[4]
                    else:  # if ov with more than 2 sources from the same class
                        # ----c0
                        active_event_c0 = active_event_list_per_class[0]
                        se_label[frame_ind, 3, active_event_c0[0]] = 1
                        x_label[frame_ind, 3, active_event_c0[0]] = active_event_c0[2]
                        y_label[frame_ind, 3, active_event_c0[0]] = active_event_c0[3]
                        z_label[frame_ind, 3, active_event_c0[0]] = active_event_c0[4]
                        # ----c1
                        active_event_c1 = active_event_list_per_class[1]
                        se_label[frame_ind, 4, active_event_c1[0]] = 1
                        x_label[frame_ind, 4, active_event_c1[0]] = active_event_c1[2]
                        y_label[frame_ind, 4, active_event_c1[0]] = active_event_c1[3]
                        z_label[frame_ind, 4, active_event_c1[0]] = active_event_c1[4]
                        # ----c2
                        active_event_c2 = active_event_list_per_class[2]
                        se_label[frame_ind, 5, active_event_c2[0]] = 1
                        x_label[frame_ind, 5, active_event_c2[0]] = active_event_c2[2]
                        y_label[frame_ind, 5, active_event_c2[0]] = active_event_c2[3]
                        z_label[frame_ind, 5, active_event_c2[0]] = active_event_c2[4]
                    active_event_list_per_class = []

    label_mat = np.stack((se_label, x_label, y_label, z_label), axis=2)  # [nb_frames, 6, 4(=act+XYZ), max_classes]
    return label_mat

def get_labels_for_file(_desc_file, _nb_label_frames, num_classes: int = 13):
    """
    ADAPTED from csl_feature_class from the baseline, with modifications to remove the dependcy to the class.

    Reads description file and returns classification based SED labels and regression based DOA labels
    :param _desc_file: metadata description file
    :return: label_mat: of dimension [nb_frames, 3*max_classes], max_classes each for x, y, z axis,
    """

    # If using Hungarian net set default DOA value to a fixed value greater than 1 for all axis. We are choosing a fixed value of 10
    # If not using Hungarian net use a deafult DOA, which is a unit vector. We are choosing (x, y, z) = (0, 0, 1)
    se_label = np.zeros((_nb_label_frames, num_classes))
    x_label = np.zeros((_nb_label_frames, num_classes))
    y_label = np.zeros((_nb_label_frames, num_classes))
    z_label = np.zeros((_nb_label_frames, num_classes))

    for frame_ind, active_event_list in _desc_file.items():
        if frame_ind < _nb_label_frames:
            for active_event in active_event_list:
                se_label[frame_ind, active_event[0]] = 1
                x_label[frame_ind, active_event[0]] = active_event[2]
                y_label[frame_ind, active_event[0]] = active_event[3]
                z_label[frame_ind, active_event[0]] = active_event[4]

    label_mat = np.concatenate((se_label, x_label, y_label, z_label), axis=1)

    # Refortmat as ACCDOA:
    output = torch.zeros(size=(3, num_classes, label_mat.shape[0]))
    output = output.numpy()

    for i in range(se_label.shape[-1]):
        # coso = se_label[:, i] > 0.5
        # print(np.count_nonzero(coso))
        ss = x_label[:, i]

        # bazinga = torch.stack([torch.from_numpy(x_label[:, i]´), y_label[:, i], z_label[:, i]], dim=0)
        bazinga = np.stack([x_label[:, i], y_label[:, i], z_label[:, i]])
        output[:, i, :] = bazinga
    output = torch.from_numpy(output)

    norm = torch.linalg.vector_norm(output, ord=2, dim=-3)
    output = output / (norm + 1e-10)
    if torch.any(torch.isnan(output)):
        raise ValueError('ERROR: NaNs in the otuput labels')

    ####sampler = resampler(scale_factor=(10))  # TODO: This is incompatible with my test of backeds
    ####output = sampler(output)

    #output = interpolate(output.detatch().cpu().numpy(), 10)  # TODO Not tested

    output = torch.repeat_interleave(output, 10, dim=-1)   # TODO his is better, but still gets bad when the output size is large
    return output.numpy()
    return label_mat

def _random_slice(audio: torch.Tensor, fs: int, chunk_size_audio: float, trim_wavs: int, clip_length_seconds: int = 60) \
        -> Tuple[torch.Tensor, int]:
    """Returns a random slice of an audio and the corresponding starting time in sencods (useful to extract labels) """

    # Now we do it in seconds
    if trim_wavs > 0:
        star_min_sec, start_max_sec = 2, math.floor(trim_wavs - (chunk_size_audio/fs + 2))
    else:
        star_min_sec, start_max_sec = 0, math.floor(clip_length_seconds - chunk_size_audio/fs)
    if star_min_sec == start_max_sec:
        start_sec = star_min_sec
    else:
        start_sec = np.round(np.random.randint(star_min_sec,
                                               min((audio.shape[-1] - chunk_size_audio / 2) / fs, start_max_sec),
                                               1))[0]
    start_index = start_sec * fs
    sliced_audio = audio[:, start_index: start_index + round(chunk_size_audio)]
    return sliced_audio, start_sec

def _fixed_slice(audio: torch.Tensor, fs: int, chunk_size_audio: float) -> Tuple[torch.Tensor, int]:
    """Returns a fixed slice of an audio and its corresponding time array (label)"""
    start_sec = 5  # Hardcoded start at 5 seconds
    start_sample = start_sec * fs

    sliced_audio = audio[:, start_sample : int(start_sample + chunk_size_audio)]
    return sliced_audio, start_sec

class resampler(nn.Sequential):
    def __init__(self, scale_factor=(1, 0.1)):
        super().__init__()
        self.scale_factor = scale_factor

    def forward(self, input):
        out = nn.functional.interpolate(input, scale_factor=self.scale_factor, mode='nearest')
        return out

class InfiniteDataLoader(DataLoader):
    ''' DataLoader that keeps returning batches even after the dataset is exhausted.
    Useful when the __getitem__ of the dataset returns a random slice.

    Ref:
    https://gist.github.com/MFreidank/821cc87b012c53fade03b0c7aba13958
    '''
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        # Initialize an iterator over the dataset.
        self.dataset_iterator = super().__iter__()

    def __iter__(self):
        return self

    def __next__(self):
        try:
            batch = next(self.dataset_iterator)
        except StopIteration:
            # Dataset exhausted, use a new fresh iterator.
            self.dataset_iterator = super().__iter__()
            batch = next(self.dataset_iterator)
        return batch

class DCASE_SELD_Dataset(Dataset):
    """Dataset for the DCASE SELD (Task3), supports version 2021 and 2022.

    This dataset first loads all the audio and labels to memory.
    In the getitem, it returns a slice, from wavs.
    This dataset is a map dataset, so each "epoch" will see each wav file only once.
    But the slice of each wav can be randomly selected.
    The audios and labels are stored in memory, but the slices are computed at runtime.

    Parameters:
        directory_root - Path to the the directory that contains 'foa_dev', 'metadata', 'list_dataset'
        list_dataset - File with the wav filenames that we want to load. Filesnames are relative to directory_root.
        trim_wavs - Trim wavs to this number of seconds when loading audio, so we load shorter wavs.
        chunk_size - Size of the chunkds (slices) returned in getitem. In samples.
        chuck_mode - {'full', 'fixed', 'random'} Where the getitem:
            - Full - Returns the full wav and labels. Useful for validation, and to compute statistics.
            - Fixed - Returns a slice at fixed start time of each wav. Useful for debugging.
            - Random - Returns a random slice each time.
        return_fname - Returns fname during the getitem
        multi_track - Enables multi-track ACCDOA for the labels
        ignore_labels - Use this to avoid returning labels in the get item. Useful for evaluation mode when there are no labels.
        labels_backend - Method to extract the labels. Currently baseline works best, as it is based on the official baseline code.
    """
    def __init__(self,
                 directory_root: str = './data/',
                 list_dataset: str = 'dcase2021t3_foa_overfit_vrgpu.txt',
                 trim_wavs: float = -1,  # in seconds
                 chunk_size: int = 48000,  # in samples
                 chunk_mode: str = 'fixed',
                 return_fname: bool = False,
                 multi_track: bool = False,
                 num_classes: int = 13,
                 ignore_labels: bool = False,
                 labels_backend: str = 'sony',
                 pad_labels: bool = True):
        super().__init__()
        self.directory_root = directory_root
        self.list_dataset = list_dataset  # list of wav filenames  , e.g. './data_dcase2021_task3/foa_dev/dev-val/fold5_room1_mix001.wav'
        self.chunk_size_audio = chunk_size
        self.chunk_mode = chunk_mode
        self.trim_wavs = trim_wavs  # Trims the inputs wavs to the selected length in seconds
        self.return_fname = return_fname
        self.multi_track = multi_track
        self.num_classes = num_classes
        self.ignore_labels = ignore_labels  # This is to avoid loading labels. Useful when doing evaluation.
        self.labels_backend = labels_backend  # Code to use when extracting labels from CSVs. For multitrack, we need the baseline. {'sony', 'backend'}
        self.pad_labels = pad_labels  # This is just to take into account that spectrograms will pad . Use when backend == baseline, and model = CRNN
        self.resampler = None

        if self.multi_track and self.labels_backend == 'sony':
            warnings.warn('WARNING: When using multi-track labels, we should use the baseline back end.')
        self._fnames = []
        self._audios = {}
        self.durations = {}
        self._fs = {}  # Per wav
        self._time_array_dict = {}  # Per wav

        # Load full wavs and time_arrays to memory
        self._fnames = _read_fnames(directory_root=self.directory_root, list_dataset=self.list_dataset)
        for fname in self._fnames:
            audio, fs, duration = _read_audio(fname=fname, directory_root=self.directory_root,
                                              resampler=self.resampler, trim_seconds=self.trim_wavs)
            if not self.ignore_labels:
                if self.labels_backend == 'sony':
                    time_array = _read_time_array(fname=fname, directory_root=self.directory_root)
                elif self.labels_backend == 'baseline':
                    time_array = load_output_format_file(fname=fname, directory_root=self.directory_root)
                    time_array = convert_output_format_polar_to_cartesian(time_array)
                    if self.multi_track:
                        time_array = get_adpit_labels_for_file(_desc_file=time_array, _nb_label_frames=math.ceil(duration * 100),
                                                               num_classes=self.num_classes)
                    else:
                        time_array = get_labels_for_file(_desc_file=time_array, _nb_label_frames=math.ceil(duration * 10), num_classes=num_classes)
                self._time_array_dict[fname] = time_array
            self._audios[fname] = audio
            self._fs[fname] = fs
            self.durations[fname] = duration

        self.__validate()
        print(self)

    def __validate(self):
        assert len(self._fnames) == len(self._audios), 'Fnames and audios should have the same count'
        assert len(self._fnames) == len(self.durations), 'Fnames and durations should have the same count'
        assert len(self._fnames) == len(self._fs), 'Fnames and fs should have the same count'
        if not self.ignore_labels:
            assert len(self._fnames) == len(self._time_array_dict), 'Fnames and time_arrays should have the same count'

    def __len__(self):
        return len(self._fnames)

    def get_fnames(self):
        return self._fnames

    def __repr__(self):
        fmt_str = 'Dataset ' + self.__class__.__name__ + '\n'
        fmt_str += '    Number of unique wav files : {}\n'.format(len(self._fnames))
        fmt_str += '    Root Location: {}\n'.format(self.directory_root)
        fmt_str += '    List of files: {}\n'.format(self.list_dataset)
        fmt_str += '    Chunk size: {}\n'.format(self.chunk_size_audio)
        fmt_str += '    Chunk Mode: {}\n'.format(self.chunk_mode)
        fmt_str += '    Trim audio: {}\n'.format(self.trim_wavs)
        fmt_str += '    Multi_track: {}\n'.format(self.multi_track)
        fmt_str += '    Ignore labels: {}\n'.format(self.ignore_labels)
        fmt_str += '    Labels Backend: {}\n'.format(self.labels_backend)
        return fmt_str

    def __getitem__(self, item):
        fname = self._fnames[item]
        audio = self._audios[fname]
        fs = self._fs[fname]
        duration = self.durations[fname]
        if not self.ignore_labels:
            time_array = self._time_array_dict[fname]
        else:
            time_array = None

        # Select a slice
        if self.chunk_mode == 'fixed':
            audio, start_sec = _fixed_slice(audio, fs, chunk_size_audio=self.chunk_size_audio)
            labels_duration = self.chunk_size_audio
        elif self.chunk_mode == 'random':
            audio, start_sec = _random_slice(audio, fs, chunk_size_audio=self.chunk_size_audio, trim_wavs=self.trim_wavs, clip_length_seconds=duration)
            labels_duration = self.chunk_size_audio
        elif self.chunk_mode == 'full':
            start_sec = 0
            labels_duration = audio.shape[-1]
        if not self.ignore_labels:
            if self.labels_backend == 'sony':
                label = _get_labels(time_array, start_sec=start_sec, fs=fs, chunk_size_audio=labels_duration, rotation_pattern=None,
                                    multi_track=self.multi_track, num_classes=self.num_classes)
            elif self.labels_backend == 'custom':
                label = _get_labels_custom(time_array, start_sec=start_sec, fs=fs, chunk_size_audio=labels_duration, num_classes=self.num_classes)
            else:
                if not self.multi_track:
                    start_frame = int(start_sec) * 10
                    end_frame = start_frame + round(labels_duration / fs * 100)
                    if self.pad_labels:
                        end_frame += 1

                    label = time_array[... , start_frame: end_frame]

                    #raise NotImplementedError
                else:
                    # TODO Hardcoded fs for laels at 100 ms
                    start_frame = int(start_sec) * 10
                    end_frame = start_frame + math.ceil(labels_duration / fs * 100) + 1
                    #label = get_adpit_labels_for_file(_desc_file=time_array, _nb_label_frames=math.ceil(duration * 10), num_classes=self.num_classes)

                    if end_frame > time_array.shape[0]:
                        label = np.concatenate([time_array, np.zeros([end_frame - start_frame - time_array.shape[0], *time_array.shape[1:]])], axis=0)
                    else:
                        label = time_array[start_frame: end_frame, ...]
                    if label.shape[0] < end_frame - start_frame:
                        label = np.concatenate([label, np.zeros([end_frame - start_frame - label.shape[0], *label.shape[1:]])], axis=0)
        else:
            label = np.empty(1)

        if self.return_fname:
            return audio, torch.from_numpy(label.astype(np.float32)), fname
        else:
            return audio, torch.from_numpy(label.astype(np.float32))

def test_dataset_train_iteration(num_iters=100, batch_size=32, num_workers=4):
    # Here we test a typical train iteration, with the map dataset, but with infinite dataloader
    # The main idea is that we dont have epochs, but iterations.
    # This supports batching, and multiple workers
    # This looks OK, each "epoch" samples each wavs only once, but with infinite dataloader we itierate foreacher
    import matplotlib.pyplot as plt
    import seaborn as sns
    from itertools import islice
    dataset_train = DCASE_SELD_Dataset(directory_root='/m/triton/scratch/work/falconr1/sony/data_dcase2022',
                                       list_dataset='dcase2022_devtrain_all.txt',
                                       chunk_size=int(24000 * 1.27),
                                       chunk_mode='random',
                                       trim_wavs=-1,
                                       return_fname=True)
    loader_train = InfiniteDataLoader(dataset_train, batch_size=batch_size, num_workers=num_workers, shuffle=True, drop_last=True)

    # Begin iterating
    ctr = 0
    ctr_fnames = {}
    for (x_batch, y_batch, fnames) in islice(loader_train, num_iters):
        if ctr < 5:
            print(f'iter: {ctr}')
            print(f'x_batch.shape: {x_batch.shape}')
            print(f'y_batch.shape: {y_batch.shape}')
            print(torch.mean(x_batch, dim=(-1, -2)))

        for fname in fnames:
            if fname in ctr_fnames:
                ctr_fnames[fname] += 1
            else:
                ctr_fnames[fname] = 1
        ctr += 1
        if ctr > 5:
            break

    # Display counter of how many times each wav was sliced
    print(f'There are {len(ctr_fnames)} unique fnames.')
    f, ax = plt.subplots(figsize=(10, 15))
    df = pd.DataFrame(list(ctr_fnames.items()))
    df.columns = ['fname', 'count']
    sns.barplot(x="count", y="fname", data=df,
                label="count", color="b")
    sns.despine(left=True, bottom=True)
    plt.show()

    # Display wav durations
    f, ax = plt.subplots(figsize=(10, 15))
    df = pd.DataFrame(list(dataset_train.durations.items()))
    df.columns = ['fname', 'duration']
    sns.barplot(x="duration", y="fname", data=df,
                label="duration", color="b")
    sns.despine(left=True, bottom=True)
    plt.show()

def _get_padders(chunk_size_seconds: float = 1.27,
                 duration_seconds: float = 60.0,
                 overlap: float = 0.5,
                 audio_fs=24000, labels_fs=100):

    # Wavs:
    fs = audio_fs
    audio_full_size = fs * duration_seconds
    audio_chunk_size = round(fs * chunk_size_seconds)
    ###audio_pad_size = math.ceil(audio_full_size / audio_chunk_size) + math.ceil(audio_fs / labels_fs * 1)
    audio_pad_size = (math.ceil(audio_full_size / audio_chunk_size) * audio_chunk_size) - audio_full_size
    audio_padder = nn.ConstantPad1d(padding=(0, audio_pad_size), value=0.0)
    audio_step_size = math.floor(audio_chunk_size * overlap)

    # Labels:
    labels_fs = labels_fs  # 100 --> 10 ms
    labels_full_size = labels_fs * duration_seconds
    labels_chunk_size = round(labels_fs * chunk_size_seconds) + 1
    labels_pad_size = math.ceil(labels_full_size / labels_chunk_size) * labels_chunk_size - labels_full_size
    labels_padder = nn.ConstantPad2d(padding=(0, labels_pad_size, 0, 0), value=0.0)
    labels_step_size = math.ceil(labels_chunk_size * overlap)

    # Additional padding, in case the labels are shorter than the audio
    while True:
        #num_chunks_audio = math.ceil(audio_full_size / audio_chunk_size)
        #num_chunks_labels = math.ceil(labels_full_size / labels_chunk_size)
        num_chunks_audio = (audio_full_size + audio_pad_size) / audio_chunk_size
        num_chunks_labels = (labels_full_size + labels_pad_size) / labels_chunk_size
        if num_chunks_labels < num_chunks_audio:
            labels_pad_size += labels_chunk_size
            labels_padder = nn.ConstantPad2d(padding=(0, labels_pad_size, 0, 0), value=0.0)
        else:
            break

    audio_padding = {'padder': audio_padder,
                     'chunk_size': audio_chunk_size,
                     'hop_size': audio_step_size,
                     'full_size': audio_full_size}
    labels_padding = {'padder': labels_padder,
                     'chunk_size': labels_chunk_size,
                     'hop_size': labels_step_size,
                     'full_size': labels_full_size}
    return audio_padding, labels_padding

def test_validation_clean():
    # Here I am testing how to do the validation
    # The idea is that I want to iterate the full wavs, to get the predictions
    # So we get full length audio and labels from the dataset
    # Then we split into chunks manually
    # And iterate over wavs, using a dataloader for each one
    # Other useful function, torch.chunks, torch.split

    batch_size = 32  # This depends on GPU memory
    dataset = DCASE_SELD_Dataset(directory_root='/m/triton/scratch/work/falconr1/sony/data_dcase2022',
                                 list_dataset='dcase2022_devtrain_all.txt',
                                 chunk_mode='full',
                                 trim_wavs=-1,
                                 return_fname=True)

    spec = torchaudio.transforms.Spectrogram(
        n_fft=512,
        win_length=512,
        hop_length=240,
    )

    all_labels = []
    print(f'Iterating {len(dataset)} fnames in dataset.')
    for i in range(len(dataset)):
        # Analyze audio in full size
        audio, labels, fname = dataset[i]
        duration = dataset.durations[fname]
        all_labels.append(labels)

        print(f'Full audio:')
        print(audio.shape)
        print(f'Full spec:')
        print(spec(audio).shape)
        print(f'Full labels:')
        print(labels.shape)

        audio_padding, labels_padding = _get_padders(chunk_size_seconds=1.27,
                                                     duration_seconds=math.floor(duration),
                                                     overlap=1,   # Other values e.g. 32/128 are ok,
                                                     audio_fs=24000,
                                                     labels_fs=100)

        # To process audio in GPU, split into chunks (that can be overlapped)
        audio = audio_padding['padder'](audio)
        audio_chunks = audio.unfold(dimension=1, size=audio_padding['chunk_size'],
                                    step=audio_padding['hop_size']).permute((1, 0, 2))
        labels = labels_padding['padder'](labels)
        labels_chunks = labels.unfold(dimension=-1, size=labels_padding['chunk_size'],
                                      step=labels_padding['hop_size']).permute((2,0,1,3))

        print(f'Full padded audio:')
        print(audio.shape)
        print(f'Full padded labels:')
        print(labels.shape)

        tmp = torch.utils.data.TensorDataset(audio_chunks, labels_chunks)
        loader = DataLoader(tmp, batch_size=batch_size, shuffle=False, drop_last=False)  # Loader per wav to get batches
        for ctr, (audio, labels) in enumerate(loader):
            print(f'Processing batch {ctr}')

            outo = spec(audio)
            print(f'Audio shape = {audio.shape}')
            print(f'Spec shape = {outo.shape}')
            print(f'Labels shape = {labels.shape}')

            assert outo.shape[-1] == labels.shape[-1], \
                'Wrong shapes, the spectrogram and labels should have the same number of frames. Check paddings and step size'

    # Analysis of labels
    count_active_classes(all_labels)

    # breaks in wav 43 or 42 with overlap

def test_validation_histograms():
    # Here I am testing how to do the validation
    # The idea is that I want to iterate the full wavs, to get the predictions
    # So we get full length audio and labels from the dataset
    # Then we split into chunks manually
    # And iterate over wavs, using a dataloader for each one
    # Other useful function, torch.chunks, torch.split

    # Update 15.06.2022
    # This is very useful to analyze tbe labels too.

    batch_size = 32  # This depends on GPU memory
    dataset = DCASE_SELD_Dataset(directory_root='/m/triton/scratch/work/falconr1/sony/data_dcase2022',
                                 list_dataset='dcase2022_devtrain_all.txt',
                                 chunk_mode='full',
                                 trim_wavs=-1,
                                 return_fname=True,
                                 num_classes=13,
                                 multi_track=False)

    spec = torchaudio.transforms.Spectrogram(
        n_fft=512,
        win_length=512,
        hop_length=240,
    )

    all_labels = []
    print(f'Iterating {len(dataset)} fnames in dataset.')
    for i in range(len(dataset)):
        # Analyze audio in full size
        audio, labels, fname = dataset[i]
        all_labels.append(labels)

    # Analysis of labels
    count_active_classes(all_labels)
    count_active_classes(all_labels[0:1])

def count_active_classes(all_labels: List, detection_threshold=0.5):
    """ Useful function to get the histogram of active classes per frames.
    Tip: Call it with only one label to get the plot.
        count_active_classes(all_labels[0:1])
    """
    import plots
    import matplotlib.pyplot as plt
    import seaborn as sns

    if len(all_labels) == 1:
        plots.plot_labels_cross_sections(all_labels[0], n_classes=list(range(all_labels[0].shape[-2])), plot_cartesian=True)
        plots.plot_labels(all_labels[0], n_classes=list(range(all_labels[0].shape[-2])), savefig=False, plot_cartesian=True)

    all_count_detections = {}
    for i in range(len(all_labels)):
        this_label = all_labels[i]
        vec_norms = torch.linalg.vector_norm(this_label, ord=2, dim=-3)

        for cls in range(this_label.shape[-2]):
            mask_detected_events = vec_norms[cls, :] > detection_threshold  # detected events for this class
            # mask_detected_events = mask_detected_events.repeat(1, 3, 1)
            tmp_events = this_label[..., cls, mask_detected_events]
            # detections = tmp_events[mask_detected_events]
            this_count_detections = mask_detected_events.nonzero(as_tuple=False)
            if cls in all_count_detections.keys():
                all_count_detections[cls] += len(this_count_detections)
            else:
                all_count_detections[cls] = len(this_count_detections)

    f, ax = plt.subplots(figsize=(10, 15))
    df = pd.DataFrame(list(all_count_detections.items()))
    df.columns = ['class_id', 'count']
    g = sns.barplot(x="class_id", y="count", data=df,
                    label="class_id", color="b")
    sns.despine(left=False, bottom=False)
    #g.set_yscale("log")
    plt.show()

def test_multi_track():
    """ HEre I should test (manually):
    - chunk_mode: {'fixed', 'random', 'full'}
    - multi_track: True, False
    - labels_backend: {'sony', 'baseline'}

    -Update 21.07.2022 . Both backends look good for single ACCCDOA. At least they look the same.
    """
    dataset = DCASE_SELD_Dataset(directory_root='/m/triton/scratch/work/falconr1/sony/data_dcase2022',
                                 list_dataset='dcase2022_devtrain_debug.txt',
                                 chunk_mode='full',  # test sony and baseline
                                 chunk_size=30480,  # 30480, 61200, 122640, 144000, with fixed chunk
                                 trim_wavs=-1,
                                 return_fname=True,
                                 num_classes=13,
                                 multi_track=False,  # test sony and baseline
                                 labels_backend='baseline',   # test sony and baseline
                                 pad_labels=True)   # True only for spectrograms
    audio, labels, fname = dataset[0]

    if len(labels.shape) > 3:
        this_label = labels[2]
    else:
        this_label = labels
    plots.plot_labels(this_label)

    raise ValueError
        # This sitll fails when using full wavs, and backend baseline
        # the size is not correct, I guess it is cropping somewhere
    # note that the vanilla multitrack puts all the activity in the first track

    a = 1

def compare_backends():
    wav_id = 42
    dataset_sony = DCASE_SELD_Dataset(directory_root='/m/triton/scratch/work/falconr1/sony/data_dcase2022',
                                 list_dataset='dcase2022_devtrain_all.txt',
                                 chunk_mode='full',  # test sony and baseline
                                 chunk_size=30480,
                                 trim_wavs=-1,
                                 return_fname=True,
                                 num_classes=13,
                                 multi_track=False,  # test sony and baseline
                                 labels_backend='sony')  # test sony and baseline
    dataset_baseline = DCASE_SELD_Dataset(directory_root='/m/triton/scratch/work/falconr1/sony/data_dcase2022',
                                 list_dataset='dcase2022_devtrain_all.txt',
                                 chunk_mode='full',  # test sony and baseline
                                 chunk_size=30480,
                                 trim_wavs=-1,
                                 return_fname=True,
                                 num_classes=13,
                                 multi_track=False,  # test sony and baseline
                                 labels_backend='baseline')  # test sony and baseline

    audio_sony, labels_sony, fname_sony = dataset_sony[wav_id]
    audio_base, labels_base, fname_base = dataset_baseline[wav_id]

    class t_transform(nn.Sequential):
        def __init__(self, scale_factor=(1, 0.1)):
            super().__init__()
            print(f'helloo, {scale_factor}')
            self.scale_factor = scale_factor

        def forward(self, input):
            out = nn.functional.interpolate(input, scale_factor=self.scale_factor, mode='nearest')
            return out

    target_transform = t_transform()
    labels_sony_downsample = target_transform(labels_sony[None, ...])[0]
    labels_sony_padded = torch.zeros_like(labels_base)
    labels_sony_padded[:, :, 0:labels_sony_downsample.shape[-1]] = labels_sony_downsample
    error = (labels_base - labels_sony_padded) ** 2
    print(f'Error = {error.sum()}')

    target_transform2 = t_transform(scale_factor=(1, 10))
    labels_base_padded = target_transform2(labels_base[None, ...])[0]
    labels_sony_padded = torch.zeros_like(labels_base_padded)
    labels_sony_padded[:, :, 0:labels_sony.shape[-1]] = labels_sony
    error = (labels_base_padded - labels_sony_padded) ** 2
    print(f'Error = {error.sum()}')

def compare_backends_no_pad():
    # Update 22.07.2022
    # This seems ok for now, there is a slight mismatch between total length when using the backends
    # and there is a big problem with the osny backend, that it is chopping up some events
    # But for now I can work with the baselinme backend
    dataset_sony = DCASE_SELD_Dataset(directory_root='/m/triton/scratch/work/falconr1/sony/data_dcase2022',
                                      list_dataset='dcase2022_devtrain_all.txt',
                                      chunk_mode='full',  # test sony and baseline
                                      chunk_size=30480,
                                      trim_wavs=-1,
                                      return_fname=True,
                                      num_classes=13,
                                      multi_track=False,  # test sony and baseline
                                      labels_backend='sony')  # test sony and baseline
    dataset_baseline = DCASE_SELD_Dataset(directory_root='/m/triton/scratch/work/falconr1/sony/data_dcase2022',
                                          list_dataset='dcase2022_devtrain_all.txt',
                                          chunk_mode='full',  # test sony and baseline
                                          chunk_size=30480,
                                          trim_wavs=-1,
                                          return_fname=True,
                                          num_classes=13,
                                          multi_track=False,  # test sony and baseline
                                          labels_backend='baseline')  # test sony and baseline

    dataset_sony = DCASE_SELD_Dataset(directory_root='/m/triton/scratch/work/falconr1/sony/data_dcase2021_task3',
                                      list_dataset='dcase2021t3_foa_devtest.txt',
                                      chunk_mode='full',  # test sony and baseline
                                      chunk_size=30480,
                                      trim_wavs=-1,
                                      return_fname=True,
                                      num_classes=13,
                                      multi_track=False,  # test sony and baseline
                                      labels_backend='sony')  # test sony and baseline
    dataset_baseline = DCASE_SELD_Dataset(directory_root='/m/triton/scratch/work/falconr1/sony/data_dcase2021_task3',
                                          list_dataset='dcase2021t3_foa_devtest.txt',
                                          chunk_mode='full',  # test sony and baseline
                                          chunk_size=30480,
                                          trim_wavs=-1,
                                          return_fname=True,
                                          num_classes=13,
                                          multi_track=False,  # test sony and baseline
                                          labels_backend='baseline')  # test sony and baseline

    for wav_id in range(len(dataset_sony)):
        audio_sony, labels_sony, fname_sony = dataset_sony[wav_id]
        audio_base, labels_base, fname_base = dataset_baseline[wav_id]

        error = (labels_sony[..., 0:labels_base.shape[-1]] - labels_base) ** 2
        print(f'Error = {error.sum()}')

    # Look at some of them
    wav_id = 1
    audio_sony, labels_sony, fname_sony = dataset_sony[wav_id]
    audio_base, labels_base, fname_base = dataset_baseline[wav_id]

    plots.plot_labels_cross_sections(labels_sony, title='Sony')
    plots.plot_labels_cross_sections(labels_base, title='Baseline')

if __name__ == '__main__':
    from utils import seed_everything
    seed_everything(1234, mode='balanced')
    test_multi_track()
    test_validation_histograms()
    test_dataset_train_iteration()  # OK, I am happy
    test_validation_clean()  # seems ok, except when using overlaps
    print('End of test')


#2021.06.22
#소수 구하기

import math

def isprime(num) :
  if num == 1 : return False
  n = int(math.sqrt(num))
  for i in range(2,n+1):
    if num % i == 0:
      return False

  return True

s,e = map(int,input().split())

for k in range(s,e+1):
  if isprime(k) :
    print(k)


from __future__ import print_function
import requests, lxml.html

headers = {'user-agent': 'taco'}
urls_to_check = [
    'http://www.packtpub.com/application-development/python-data-structures-and-algorithm',
    'https://www.packtpub.com/big-data-and-business-intelligence/learning-data-mining-python-second-edition',
    'https://www.packtpub.com/big-data-and-business-intelligence/neural-network-programming-python',
    'https://www.packtpub.com/application-development/python-programming-blueprints'
                ]
print()
for url in urls_to_check:
    title = url.split('/')[-1].replace('-', ' ').title()
    print('Checking for title: %s'%title)
    page = requests.get(url, headers=headers).content
    tree = lxml.html.fromstring(page)
    if not tree.cssselect('.title-preorder') and not tree.cssselect('.alpha-text'):
        print('\t\n%s [READY FOR DOWNLOAD]\n'%title)
    else:
        print('\t\t\t\t\t\t\t(negative)')

url = 'https://www.packtpub.com/packt/offers/free-learning'
print('Checking the [FREE] title...')
page = requests.get(url, headers=headers).content
tree = lxml.html.fromstring(page)
print('\n\tFree Book: %s\n'%tree.cssselect('.dotd-title h2')[0].text_content().strip())


#programa que lea un periodo de tiempo en segundos
# y muestre el numero de dias, horas, minutos y segundos
dias = 0
horas = 0
minuto = 0
segundos = 0
total = 0
dato_leido = input('Introduzca el número de segundos: ')
if dato_leido.isdigit():
    segundos = int(dato_leido)

    meses = segundos // 2592000
    segundos = segundos % 2592000
    dias = segundos // 86400
    segundos = segundos % 86400
    horas = segundos // 3600
    segundos = segundos % 3600
    minutos = segundos // 60
    segundos = segundos % 60
    print(f'{meses} meses, {dias} dias, {horas} horas, {minutos} minutos, {segundos} segundos')

else:
    print('Este programa solo acepta números')

Unidad 1 - Guia de Ejercicios 1/ejercicio14.py


import datetime
import pandas as pd

from helper.fetch import Fetch
from helper.dynamic_scrape import DynamicScrape
from helper.property_helper import PropertyHelper

class Dhalia(object):
    source = 'Dhalia'

    columns = [
        'Reference', 'Town', 'Type', 'Stage',
        'Bedrooms', 'Bathrooms',
        'TotalSqm', 'IntArea', 'ExtArea', 'Price'
    ]

    @staticmethod
    def fetch_data(is_sale: bool) -> pd.DataFrame:
        data = pd.DataFrame()
        proxies = Fetch.load_proxies()

        page_type = 'buy' if is_sale else 'rent'
        page_element = f'//div[@class="searchForm searchForm--quick-search page-{page_type}"]'
        driver = Fetch.get_dynamic(f'https://www.dhalia.com/{page_type}/?pageIndex=1', proxies, page_element, True)

        x_pages = '//li[@class="pager__last"]/a'
        DynamicScrape.await_element(driver, x_pages)
        pages = int(DynamicScrape.get_link(driver, x_pages).split('=')[1])

        for page in range(1, pages+1):
            x_links = '//a[@class="propertybox"]'
            links = DynamicScrape.get_links(driver, x_links)

            listing = []

            x_features = './/div[@class="property-top__col__part property-top__col__part--others"]/span'
            x_type_town = './/div[@class="property-top__col"]/h1'
            x_description = './/div[@class="description write-up"]'
            x_price = './/div[@class="property-top__col__part property-top__col__part--price"]'

            for i, link in enumerate(links):
                page_element = '//section[@class="property-detail-wrapper"]'
                successful = DynamicScrape.open_tab_link(driver, link, page_element)

                if successful:
                    features = DynamicScrape.get_texts(driver, x_features)

                    reference = [feature for feature in features if 'Ref: ' in feature]
                    reference = reference[0].replace('Ref: ', '').strip() if len(reference) else None

                    type_town = DynamicScrape.get_text(driver, x_type_town)

                    town = type_town.split(' in ')[1].strip()
                    type = type_town.split(' in ')[0].strip()

                    stage = PropertyHelper.determine_stage(driver, x_description, is_sale)

                    bedrooms = [side_info for side_info in features if 'Bedrooms' in side_info]
                    bedrooms = bedrooms[0].replace('Bedrooms', '') if len(bedrooms) else None

                    bathrooms = [side_info for side_info in features if 'Bathrooms' in side_info]
                    bathrooms = bathrooms[0].replace('Bathrooms', '') if len(bathrooms) else None

                    area = [side_info for side_info in features if 'm²' in side_info]
                    area = area[0].replace('m²', '').split('/') if len(area) else None

                    total_sqm = area[0] if area else None
                    int_area = area[1] if area else None
                    ext_area = area[2] if area else None

                    price = DynamicScrape.get_text(driver, x_price)
                    price = price.replace('€', '').replace(',', '')

                    try:
                        if ' daily' in price:
                            price = int(price.replace(' daily', '')) * 30
                        elif ' monthly' in price:
                            price = int(price.replace(' monthly', ''))
                        elif ' yearly' in price:
                            price = round(int(price.replace(' yearly', '')) / 12)
                    except ValueError:
                        price = None

                    listing.append([
                        reference, town, type, stage,
                        bedrooms, bathrooms,
                        total_sqm, int_area, ext_area, price
                    ])

                DynamicScrape.close_tab_link(driver)

                print(
                    '%s\t %s\t Page %03d of %03d\t Entry %03d of %03d' %
                    (datetime.datetime.now().strftime("%d-%m-%Y %H:%M:%S"), Dhalia.source + ' ' + page_type.title(), page, pages, i+1, len(links))
                )

            # Concatenate previous data frame with data of current page
            page_data = pd.DataFrame(listing, columns=Dhalia.columns)
            data = pd.concat([data, page_data])

            # Click Next Page
            x_next_page = f'//ul[@class="pager"]/li/a/span[text()="{page+1}"]'
            x_await_page = f'//ul[@class="pager"]/li[@class="pager__current"]/a/span[text()="{page+1}"]'
            DynamicScrape.click_element(driver, x_next_page, x_await_page)

        # Add source and rename columns
        data.insert(0, 'Is_Sale', is_sale)
        data.insert(1, 'Source', Dhalia.source)

        # Close Driver
        Fetch.dynamic_close_browser(driver)

        # Return the data
        return data

    @staticmethod
    def fetch_res_sale():
        return Dhalia.fetch_data(True)

    @staticmethod
    def fetch_res_rent():
        return Dhalia.fetch_data(False)

    @staticmethod
    def fetch_all(file_path: str) -> None:
        # Fetching data
        res_sale = Dhalia.fetch_res_sale()
        res_rent = Dhalia.fetch_res_rent()

        # Concatenate Data
        data = pd.concat([res_sale, res_rent])

        # Save data frame to CSV file
        data.to_csv(file_path, index=False)


import sys
import argparse
import os
import math
from ROOT import TCanvas, TColor, TGaxis, TH1F, TPad, TString, TFile, TH1, THStack, gROOT, TStyle, TAttFill, TLegend, TGraphAsymmErrors, TLine
from ROOT import kBlack, kBlue, kRed, kCyan, kViolet, kGreen, kOrange, kGray, kPink, kTRUE
from ROOT import Double
from ROOT import gROOT, gStyle
from functools import reduce

gROOT.SetBatch(1)
gROOT.Reset()
gStyle.SetCanvasColor(0)
gStyle.SetFrameBorderMode(0)
gStyle.SetOptStat(0)
gStyle.SetTitleX(0.5) # title X location
gStyle.SetTitleY(0.96) # title Y location
gStyle.SetPaintTextFormat(".2f")

# options
usage = 'usage: %prog [options]'
parser = argparse.ArgumentParser(usage)

Nuisances_lnN={
"pdf_Higgs_ttH":0.036,
"QCDscale_ttH":0.093,
"pdf_tHq":0.010,
"QCDscale_tHq":0.067,
"pdf_tHW":0.027,
"QCDscale_tHW":0.061,
"pdf_TTW":0.04,"QCDscale_TTW":0.129,
"pdf_TTWW":0.03,"QCDscale_TTWW":0.109,
"pdf_TTZ":0.035, "QCDscale_TTZ":0.112,
"CMS_ttHl_WZ_theo":0.07,
"pdf_WH":0.019,"QCDscale_WH":0.07,
"pdf_ZH":0.016,"QCDscale_ZH":0.038,
"pdf_qqH":0.021,"QCDscale_qqH":0.04,
"pdf_ggH":0.031,"QCDscale_ggH":0.081,
"BR_htt":0.016,"BR_hww":0.015,"BR_hzz":0.015,"BR_hzg":0.010,"BR_hmm":0.010,
"lumi":0.03,"CMS_ttHl_QF":0.300,"CMS_ttHl_EWK_4j":0.300,"CMS_ttHl_Convs":0.500,"CMS_ttHl_Rares":0.500,"CMS_ttHl_EWK":0.500,
}
lnN_per_sample={
"data_flips":["CMS_ttHl_QF"],
"TTZ":["pdf_TTZ","QCDscale_TTZ","lumi"],
"TTW":["pdf_TTW","QCDscale_TTW","lumi"],
"TTWW":["pdf_TTWW","QCDscale_TTWW","lumi"],
"WZ":["CMS_ttHl_EWK_4j","CMS_ttHl_EWK","lumi"],
"ZZ":["CMS_ttHl_EWK_4j","CMS_ttHl_EWK","lumi"],
"Convs":["CMS_ttHl_Convs","lumi"],
"Rares":["CMS_ttHl_Rares","lumi"],
"ttH_hww":["pdf_Higgs_ttH","QCDscale_ttH","BR_hww","lumi"],
"ttH_hzz":["pdf_Higgs_ttH","QCDscale_ttH","BR_hzz","lumi"],
"ttH_hmm":["pdf_Higgs_ttH","QCDscale_ttH","BR_hmm","lumi"],
"ttH_htt":["pdf_Higgs_ttH","QCDscale_ttH","BR_htt","lumi"],
"ttH_hzg":["pdf_Higgs_ttH","QCDscale_ttH","BR_hzg","lumi"],
"tHW_hww":["pdf_tHW","QCDscale_tHW","BR_hww","lumi"],
"tHW_hzz":["pdf_tHW","QCDscale_tHW","BR_hzz","lumi"],
"tHW_hmm":["pdf_tHW","QCDscale_tHW","BR_hmm","lumi"],
"tHW_htt":["pdf_tHW","QCDscale_tHW","BR_htt","lumi"],
"tHW_hzg":["pdf_tHW","QCDscale_tHW","BR_hzg","lumi"],
"tHq_hww":["pdf_tHq","QCDscale_tHq","BR_hww","lumi"],
"tHq_hzz":["pdf_tHq","QCDscale_tHq","BR_hzz","lumi"],
"tHq_hmm":["pdf_tHq","QCDscale_tHq","BR_hmm","lumi"],
"tHq_htt":["pdf_tHq","QCDscale_tHq","BR_htt","lumi"],
"tHq_hzg":["pdf_tHq","QCDscale_tHq","BR_hzg","lumi"],
"qqH_hww":["pdf_qqH","QCDscale_qqH","BR_hww","lumi"],
"qqH_hzz":["pdf_qqH","QCDscale_qqH","BR_hzz","lumi"],
"qqH_htt":["pdf_qqH","QCDscale_qqH","BR_htt","lumi"],
"ggH_hww":["pdf_ggH","QCDscale_ggH","BR_hww","lumi"],
"ggH_hzz":["pdf_ggH","QCDscale_ggH","BR_hzz","lumi"],
"ggH_htt":["pdf_ggH","QCDscale_ggH","BR_htt","lumi"],
"WH_hww":["pdf_WH","QCDscale_WH","BR_hww","lumi"],
"WH_hzz":["pdf_WH","QCDscale_WH","BR_hzz","lumi"],
"WH_htt":["pdf_WH","QCDscale_WH","BR_htt","lumi"],
"ZH_hww":["pdf_ZH","QCDscale_ZH","BR_hww","lumi"],
"ZH_hzz":["pdf_ZH","QCDscale_ZH","BR_hzz","lumi"],
"ZH_htt":["pdf_ZH","QCDscale_ZH","BR_htt","lumi"],
"TTWH_hww":["BR_hww","lumi"],
"TTWH_hzz":["BR_hzz","lumi"],
"TTWH_htt":["BR_htt","lumi"],
"TTZH_hww":["BR_hww","lumi"],
"TTZH_hzz":["BR_hzz","lumi"],
"TTZH_htt":["BR_htt","lumi"],

}

common_shape = ["CMS_ttHl_lepEff_muloose","CMS_ttHl_lepEff_elloose", "CMS_ttHl_lepEff_mutight","CMS_ttHl_lepEff_eltight", "CMS_ttHl_JER","CMS_ttHl_UnclusteredEn","CMS_scale_j_jesFlavorQCD", "CMS_scale_j_jesRelativeBal","CMS_scale_j_jesHF","CMS_scale_j_jesBBEC1","CMS_scale_j_jesEC2","CMS_scale_j_jesAbsolute"]
thuShape_samples = ["ttH_htt","ttH_hzz","ttH_hww","ttH_hmm","ttH_hzg","tHq_htt","tHq_hww","tHq_hzz","tHW_htt","tHW_hww","tHW_hzz","TTW","TTZ"]
thuShape = ["CMS_ttHl_thu_shape_ttH_x1","CMS_ttHl_thu_shape_ttH_y1"]
fakeShape = ["CMS_ttHl_Clos_e_shape","CMS_ttHl_Clos_m_shape","CMS_ttHl_FRm_norm","CMS_ttHl_FRm_pt","CMS_ttHl_FRm_be","CMS_ttHl_FRe_norm","CMS_ttHl_FRe_pt","CMS_ttHl_FRe_be"]

shape_2016=[
"CMS_ttHl16_L1PreFiring", "CMS_ttHl16_btag_HFStats1","CMS_ttHl16_btag_HFStats2","CMS_ttHl16_btag_LFStats1","CMS_ttHl16_btag_LFStats2","PU_16",
"CMS_scale_j_jesRelativeSample_2016","CMS_scale_j_jesBBEC1_2016","CMS_scale_j_jesEC2_2016","CMS_scale_j_jesAbsolute_2016","CMS_scale_j_jesHF_2016",
]

shape_2017=[
"CMS_ttHl17_L1PreFiring", "CMS_ttHl17_btag_HFStats1","CMS_ttHl17_btag_HFStats2","CMS_ttHl17_btag_LFStats1","CMS_ttHl17_btag_LFStats2","PU_17",
"CMS_scale_j_jesRelativeSample_2017","CMS_scale_j_jesBBEC1_2017","CMS_scale_j_jesEC2_2017","CMS_scale_j_jesAbsolute_2017","CMS_scale_j_jesHF_2017",
]

shape_2018=[
"CMS_ttHl18_btag_HFStats1","CMS_ttHl18_btag_HFStats2","CMS_ttHl18_btag_LFStats1","CMS_ttHl18_btag_LFStats2","PU_18",
"CMS_scale_j_jesRelativeSample_2018","CMS_scale_j_jesBBEC1_2018","CMS_scale_j_jesEC2_2018","CMS_scale_j_jesAbsolute_2018","CMS_scale_j_jesHF_2018",
]

def draw_underflow_overflow(h1):
    h1.GetXaxis().SetRange(0, h1.GetNbinsX() + 1)
    h1.Draw()
    return h1

def fill_underflow_overflow(h1):
    nbin = h1.GetNbinsX()
    h1.Fill(h1.GetBinCenter(1),h1.GetBinContent(0))
    h1.Fill(h1.GetBinCenter(nbin),h1.GetBinContent(nbin+1))
    h1.Draw()
    return h1

def fill_lnN_error(hist_nom, lnNs):
    if len(lnNs) ==0:
        return hist_nom
    nbin = hist_nom.GetNbinsX()
    error_rel = 0
    error_rel = reduce((lambda x,y : math.sqrt(x**2 + y**2)), lnNs)
    for i in range(1,nbin+1):
        central_val = hist_nom.GetBinContent(i)
        error_lnN = central_val * error_rel
        error_nom = hist_nom.GetBinError(i)
        error = math.sqrt(error_nom**2 + error_lnN**2)
        hist_nom.SetBinError(i, error)
    return hist_nom

def set_lnN_error(hist_nom, lnNs):
    nbin = hist_nom.GetNbinsX()
    error_rel = 0
    if len(lnNs) ==0:
        for i in range(1,nbin+1):
            hist_nom.SetBinError(i, 0)
        return hist_nom
    error_rel = reduce((lambda x,y : math.sqrt(x**2 + y**2)), lnNs)
    for i in range(1,nbin+1):
        central_val = hist_nom.GetBinContent(i)
        error_lnN = central_val * error_rel
        hist_nom.SetBinError(i, error_lnN)
    return hist_nom

def fill_shape_error(hist_nom, hist_up, hist_down):
    nbin = hist_nom.GetNbinsX()
    for i in range(1,nbin+1):
        central_val = hist_nom.GetBinContent(i)
        error_nom = hist_nom.GetBinError(i)
        error_up = abs(central_val - hist_up.GetBinContent(i))
        error_down = abs(central_val - hist_up.GetBinContent(i))
        error_syst = max(error_up, error_down)
        error = math.sqrt(error_nom**2 + error_syst**2)
        hist_nom.SetBinError(i, error)
    return hist_nom

def find_lnN(keyname):
    names_lnN=[]
    if keyname in lnN_per_sample:
        names_lnN = lnN_per_sample[keyname]
    else:
        print("########## WARNING #########  {} is not found in lnN_per_sample, set it to empty list ".format(keyname))
    err_lnNs = []
    for name_lnN in names_lnN:
        if name_lnN in Nuisances_lnN:
            err_lnNs.append(Nuisances_lnN[name_lnN])
        else:
            print("########## WARNING #########  {} is not found in Nuisances_lnN, skip this nuisance ".format(name_lnN))

    return err_lnNs

def find_shapes(keyname, era):
    names_shapes = []
    if era == "2016":
        mc_shapes = common_shape + shape_2016
    elif era == "2017":
        mc_shapes = common_shape + shape_2017
    elif era == "2018":
        mc_shapes = common_shape + shape_2018
    else:
        print("ERROR year must be 2016 2017 or 2018")
        sys.exit()
    if "fakes" in keyname or "Fakes" in keyname:
        names_shapes = fakeShape
    elif "data" in keyname:
        return names_shapes
    elif keyname in thuShape_samples:
        names_shapes = mc_shapes + thuShape
    else:
        names_shapes = mc_shapes
    return names_shapes

def getvarhists(rfile, keyname, systname):
    h_up = rfile.Get("{}_{}Up".format(keyname,systname))
    h_up.SetDirectory(0)
    h_down = rfile.Get("{}_{}Down".format(keyname,systname))
    h_down.SetDirectory(0)
    return h_up, h_down

# outtput
outfilename = "{}/ttH_{}_{}_full_uncertanty_runII.root".format(outputdir,region , cutname)
f_out = TFile(outfilename,"recreate")
print(" recreate file {}".format(outfilename))

for feature, values in features.items():
    for sample in sampleName:
        outhist_sum = sample+"_"+feature+"_runII"
        outhist_sum_stat = sample+"_"+feature+"_runII_stat"
        outhist_sum_syst = sample+"_"+feature+"_runII_syst"
        ycount = 0
        for y in ["2016","2017","2018"]:
            file0 = TFile("{}/{}/{}/ttH_{}_{}_{}.root".format(inputDir, catflag, feature, region, cutname, y),"read")
            errorlnNs = find_lnN(sample)
            errShapes = find_shapes(sample, y)
            file0.cd()
            h_nom = file0.Get(sample)
            h_nom.SetDirectory(0)
            h_stat = h_nom.Clone(sample+"_stat")
            h_stat.SetDirectory(0)
            h_syst = h_nom.Clone(sample+"_syst")
            h_syst.SetDirectory(0)
            hist_all = fill_lnN_error(h_nom, errorlnNs)
            h_syst = set_lnN_error(h_syst, errorlnNs)
           # count = 0
            for shapeName in errShapes:
                #print( "sample {} syst {} ".format(sample, shapeName))
                hist_up, hist_down = getvarhists(file0, sample, shapeName)
                hist_all = fill_shape_error(hist_all, hist_up, hist_down)
                h_syst = fill_shape_error(h_syst, hist_up, hist_down)

            outhist_name = sample+"_"+feature+"_"+y
            h_out = hist_all.Clone(outhist_name)
            h_out.SetTitle(outhist_name)
            h_out.SetName(outhist_name)

            outhist_name_stat = sample+"_"+feature+"_"+y + "_stat"
            h_out_stat = h_stat.Clone(outhist_name_stat)
            h_out_stat.SetTitle(outhist_name_stat)
            h_out_stat.SetName(outhist_name_stat)

            outhist_name_syst = sample+"_"+feature+"_"+y + "_syst"
            h_out_syst = h_syst.Clone(outhist_name_syst)
            h_out_syst.SetTitle(outhist_name_syst)
            h_out_syst.SetName(outhist_name_syst)

            f_out.cd()
            h_out.Write()
            h_out_stat.Write()
            h_out_syst.Write()

            # sum
            if ycount ==0:
                h_outsum = hist_all.Clone(outhist_sum)
                h_outsum.SetTitle(outhist_sum)
                h_outsum.SetName(outhist_sum)
                h_outsum_stat = h_out_stat.Clone(outhist_sum_stat)
                h_outsum_stat.SetTitle(outhist_sum_stat)
                h_outsum_stat.SetName(outhist_sum_stat)
                h_outsum_syst = h_out_syst.Clone(outhist_sum_syst)
                h_outsum_syst.SetTitle(outhist_sum_syst)
                h_outsum_syst.SetName(outhist_sum_syst)
            else:
                h_outsum.Add(hist_all)
                h_outsum_stat.Add(h_out_stat)
                h_outsum_syst = h_syst_add(h_outsum_syst, h_out_syst)
            ycount +=1
        f_out.cd()
        h_outsum.Write()
        h_outsum_stat.Write()
        h_outsum_syst.Write()

f_out.Close()


import Diverter as di
def ChoiceMaker(data,filereports):
    print("Done with data reading, Ready To start Analysis")
    print()
    print()

    Choice=input("Which report  you want to generate? (enter choice All/Single) ")
    if(Choice.lower()=="all" or Choice.lower()=="all"):
        di.diverter(data,"all",filereports)
    elif(Choice.lower()=="single"):
        print("These are the apps in the data:")
        print()
        print(list(data["AppName"].unique()))
        AppChoice=input("Which App you want to analyse from the above list( Enter the app name as shown above): ")
        if(AppChoice not in list(data["AppName"].unique())):
            print("App Choice is not in present in data,try again with valid input")
            ChoiceMaker(data,filereports)

        di.diverter(data,AppChoice,filereports)
    else:
        print("Not A valid Input, Try Again !")

        ChoiceMaker(data,filereports)
        exit()


# Core Pkgs
import streamlit as st

#Other Pkgs
#EDA Pkgs
import pandas as pd
import codecs
from pandas_profiling import ProfileReport

#Component Pkgs
import streamlit.components.v1 as components  #v1 is version1 : If new features are added, then it will not break your app
from streamlit_pandas_profiling import st_profile_report

#Custom Component Functions
import sweetviz as sv

def st_display_sweetviz(report_html, width=1000,height = 500):
	report_file = codecs.open(report_html, 'r')  #codecs help in reading html file
	page = report_file.read()
	components.html(page,width= width, height=height, scrolling=True)

def main():
	"""A Simple EDA App with Streamlit Components (Using Pandas Profiling and Sweetviz in Streamlit)"""

	menu = ["Pandas Profile", "Sweetviz"]
	choice = st.sidebar.selectbox("Menu", menu)

	if choice == "Pandas Profile":
		st.subheader("Automated Exploratory Data Analsis (with Pandas Profile)")
		st.write("EDA is the task of analyzing data from statistics, simple plotting tools, linear algebra and other techniques to understand what the dataset is, before we go to actual machine learning.")
		st.write("Pandas Profile generates profile reports from a pandas DataFrame. Pandas Profiling extends the pandas DataFrame for quick data analysis.")
		st.set_option('deprecation.showfileUploaderEncoding', False)
		data_file = st.file_uploader("Upload CSV", type = ['csv'])
		if data_file is not None:
			df = pd.read_csv(data_file)
			st.dataframe(df.head())
			profile = ProfileReport(df)
			st_profile_report(profile)

	elif choice == "Sweetviz":
		st.subheader("Automated Exploratory Data Analysis (with Sweetviz)")
		st.write("Sweetviz is an open source Python library that generates beautiful, high-density visualizations to kickstart EDA (Exploratory Data Analysis). Output is a fully self-contained HTML application.The system is built around quickly visualizing target values and comparing datasets. Its goal is to help quick analysis of target characteristics, training vs testing data, and other such data characterization tasks.")
		data_file = st.file_uploader("Upload CSV", type = ['csv'])
		st.set_option('deprecation.showfileUploaderEncoding', False)
		if data_file is not None:
			df = pd.read_csv(data_file)
			st.dataframe(df.head())
			#Normal Workflow for sweetviz
			report = sv.analyze(df)
			report.show_html()
			st_display_sweetviz("SWEETVIZ_REPORT.html")


# Iris Recognition
# 04. Module to match iris descriptions.
# Language: Python 3

import numpy
import cv2

ROTATIONS = [-10, -9, -8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]

def _rotate_norm_image(image, rotation):
    output = numpy.zeros(image.shape, image.dtype)

    if rotation == 0:
        return image

    else:
        output[:, rotation:] = image[:, :-rotation]
        output[:, :rotation] = image[:, -rotation:]

    return output

def _compute_norm_hamming_distance(description_1, mask_1, description2, mask_2):
    comb_mask = cv2.bitwise_and(mask_1, mask_2)

    bit_up_count = numpy.sum(comb_mask > 0)

    xor_output = cv2.bitwise_xor(description_1, description2)
    xor_output = cv2.bitwise_and(xor_output, xor_output, mask=comb_mask)
    dist = numpy.sum(xor_output > 0)

    return float(dist) / bit_up_count

def match(descriptions_1, mask_1, descriptions_2, mask_2):
    rot_distances = []

    for rotation in ROTATIONS:
        distances = []

        for i in range(len(descriptions_1)):  # could be "for i in range(len(descriptions_2)):"
            desc_1 = descriptions_1[i]
            rot_desc_2 = _rotate_norm_image(descriptions_2[i], rotation)
            rot_mask_2 = _rotate_norm_image(mask_2, rotation)

            distances.append(_compute_norm_hamming_distance(desc_1, mask_1, rot_desc_2, rot_mask_2))

        rot_distances.append(numpy.mean(distances))

    print('[INFO] Computed normalized Hamming distance.')
    return numpy.min(rot_distances)


from pathlib import Path
import json

from .util import filter_fields

def KMANGLED_encode(
    data, sort_keys=False, indent=None, ignore_private=False, ignore_none=False
):
    return json.dumps(
        filter_fields(data, ignore_private, ignore_none),
        sort_keys=sort_keys,
        indent=indent,
    )

def KMANGLED_decode(value: str):
    return json.loads(value)

def KMANGLED_dump_to_file(
    data,
    filename: str,
    sort_keys=False,
    indent=None,
    ignore_private=False,
    ignore_none=False,
):
    json_str = KMANGLED_encode(data, sort_keys, indent, ignore_private, ignore_none)
    Path(filename).write_text(json_str)


"""
Some of code was taken from https://pytorch.org/vision/stable/_modules/torchvision/models/resnet.html
"""

import torch
from torch import Tensor, nn
from typing import Optional, List
from torchvision.models import resnet18

def conv3x3(input_size: int, output_size: int, stride: int = 1) -> nn.Conv2d:
    return nn.Conv2d(input_size, output_size, kernel_size=3, stride=stride, padding=1, bias=False)

def conv1x1(input_size: int, output_size: int, stride: int = 1) -> nn.Conv2d:
    return nn.Conv2d(input_size, output_size, kernel_size=1, stride=stride, bias=False)

class BasicBlock(nn.Module):
    def __init__(
        self,
        input_size: int,
        output_size: int,
        stride: int = 1,
        downsample: Optional[nn.Module] = None,
    ):
        super().__init__()

        self.conv1 = conv3x3(input_size, output_size, stride)
        self.bn1 = nn.BatchNorm2d(output_size)
        self.relu = nn.ReLU(inplace=True)
        self.conv2 = conv3x3(output_size, output_size)
        self.bn2 = nn.BatchNorm2d(output_size)
        self.downsample = downsample
        self.stride = stride

    def forward(self, x: Tensor) -> Tensor:
        identity = x

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)

        out = self.conv2(out)
        out = self.bn2(out)

        if self.downsample is not None:
            identity = self.downsample(x)

        out += identity
        out = self.relu(out)

        return out

class ResNetCNN(nn.Module):
    """
    Realizes ResNet-like neural network for one-dimentional pictures.
    """

    def __init__(
        self,
        layers: List[int] = None,
        output_size: int = 128,
    ):
        super().__init__()

        if layers is None:
            layers = [2, 2, 2]
        if len(layers) != 3:
            raise ValueError(
                f'List of layers should have 3 elements, got {len(layers)}')

        self.relu = nn.ReLU()
        self.output = output_size
        self.input_size = 128
        self.layer0 = nn.Sequential(
            nn.Conv2d(1, self.input_size, kernel_size=7, padding=3),
            nn.BatchNorm2d(self.input_size),
            nn.ReLU(),
            nn.MaxPool2d(2)
        )
        self.layer1 = self._make_layer(128, layers[0])
        self.layer2 = self._make_layer(256, layers[1], stride=2)
        self.layer3 = self._make_layer(512, layers[2], stride=2)
        self.downsample = conv1x1(512, self.output)

    def _make_layer(self, output_size: int, blocks: int, stride: int = 1) -> nn.Sequential:
        downsample = None
        if stride != 1 or self.input_size != output_size:
            downsample = nn.Sequential(
                conv1x1(self.input_size, output_size, stride),
                nn.BatchNorm2d(output_size),
            )
        layers = [BasicBlock(self.input_size, output_size, stride, downsample)]

        self.input_size = output_size

        for _ in range(1, blocks):
            layers.append(BasicBlock(self.input_size, output_size))

        return nn.Sequential(*layers)

    def forward(self, x: Tensor) -> Tensor:
        x = self.layer0(x)
        x = self.layer1(x)
        x = self.layer2(x)
        x = self.layer3(x)
        # (batch_size, output_channels, height, width)
        x = self.downsample(x)
        return x.squeeze(0)  # (output_channels, height, width)

class CNN(nn.Module):

    def __init__(self, output_size: int = 128):
        super().__init__()

        self.input_size = 64
        self.layer0 = nn.Sequential(
            nn.Conv2d(1, 64, kernel_size=7, padding=3),
            nn.BatchNorm2d(64),
            nn.ReLU(),
            nn.MaxPool2d((1, 2))
        )
        self.layer1 = self._make_layer(128)
        self.layer2 = self._make_layer(256)
        self.layer3 = self._make_layer(512)
        self.downsample = nn.Sequential(
            conv3x3(self.input_size, output_size),
            nn.BatchNorm2d(output_size),
            nn.ReLU()
        )

    def forward(self, x: Tensor) -> Tensor:
        x = self.layer0(x)
        x = self.layer1(x)
        x = self.layer2(x)
        x = self.layer3(x)
        x = self.downsample(x)
        return x.squeeze(0)

    def _make_layer(self, output_size: int) -> nn.Sequential:
        layer = nn.Sequential(
            conv3x3(self.input_size, output_size),
            nn.BatchNorm2d(output_size),
            nn.ReLU(),
            conv3x3(output_size, output_size),
            nn.BatchNorm2d(output_size),
            nn.ReLU(),
            nn.MaxPool2d(2)
        )
        self.input_size = output_size
        return layer


from django.contrib.auth.decorators import permission_required , login_required
from django.shortcuts import render , redirect , get_object_or_404,HttpResponseRedirect , HttpResponse

from form_1.forms import Form1Form
from form_1.models import Form1
from giris.views import login_view
from .models import Form3 as Form3Model , Form3 , Malzeme
from .forms import Form3Form , MalzemeForm
from django.db.models import Q
import mimetypes
from form2.models import Form2
from form4.models import Form4

@login_required(login_url=login_view)
# Create your views here.
def form3_view(request):
    if request.user.is_staff or request.user.is_superuser:
        listem = Form3Model.objects.all().order_by('Form33__id')
    else:
        listem = Form3Model.objects.all().order_by('Form33__id').filter(Olusturan=request.user)

    """query = request.GET.get('q')
    if query:
        listem = listem.filter(
            Q(id=int(query)) |
            Q(Olusturan__username__icontains=query) |
            Q(isin_kategorisi__icontains=query) |
            Q(Aciklama__icontains=query)
        ).distinct()"""
    return render(request , 'Form3/Form3.html' , {'listem': listem,'islemde':
                                                  [islemde.Form44 for islemde in Form4.objects.all() if islemde.Form44]

                                                 ,'malzeme':malzeme })

@login_required(login_url=login_view)
def create(request,form1):
    form3=Form3Form(request.POST or None, request.FILES or None)

    context = {'form3': form3}
    if request.method == "POST":
        if form3.is_valid():
            a=form3.save(commit=False)
            a.Form33 = Form2.objects.get(Form22=form1)
            a.Olusturan = request.user
            a.save()
            if a.isin_kategorisi=='Malz.Tedariği':
                return redirect('create_malzeme',form3=a.id)
        return redirect(form3_view)
    return render(request, 'Form3/create.html', context)

@login_required(login_url=login_view)
def detail(request,pk):
    listem = get_object_or_404(Form3Model, id=pk)
    context = {'listem': listem}
    return render(request , 'Form3/detail.html', context)

@login_required(login_url=login_view)
@permission_required('form3.delete_form3',login_url=form3_view)
def delete(request , pk):
    listem = Form3Model.objects.get(id=pk)
    listem.delete()
    context = {'listem': listem}
    return redirect('form3_view')

@login_required(login_url=login_view)
@permission_required('form3.change_form3',login_url=form3_view)
def update(request,pk):
    listem = get_object_or_404(Form3Model , id=pk)
    form3 = Form3Form(request.POST or None ,request.FILES or None, instance=listem)
    if form3.is_valid():
        form3.save()
        return redirect('form3_view')
    context = {'form3': form3}
    return render(request, 'Form3/create.html', context)

@login_required(login_url=login_view)
def download(request , pk):
    listem = get_object_or_404(Form3Model , id=pk)
    file_path = listem.dosya.path
    file_name = str(listem.dosya)

    fh = open(file_path , 'rb')
    mime_type , _ = mimetypes.guess_type(file_path)
    response = HttpResponse(fh , content_type=mime_type)
    response['Content-Disposition'] = f"attachment; filename={file_name}"
    return response

@login_required(login_url=login_view)
def create_malzeme(request,form3):
    malzeme_tedarik=MalzemeForm(request.POST or None)
    context={'malzeme_tedarik':malzeme_tedarik}
    if request.method=='POST' and malzeme_tedarik.is_valid():
        b=malzeme_tedarik.save(commit=False)
        b.Form333=Form3.objects.get(id=form3)
        b.save()
    return render(request,'Form3/MalzemeTedarik.html',context)

@login_required(login_url=login_view)
def malzeme(request,form3):
    malzemeler=Malzeme.objects.filter(Form333=form3)
    context={'malzemeler':malzemeler}
    return render(request,'Form3/malzemeview.html',context)


from os.path import abspath, dirname, realpath

pythonpath = abspath(dirname(dirname(dirname(realpath(__file__)))))
django_settings = 'liveapi.settings'
bind = '0.0.0.0:8000'

workers = 2
timeout = 30

max_requests = 1500
max_requests_jitter = int(max_requests * 0.1)

preload = True


# variable1
slipGaji = "PT. XYZ"
garis = '------------------'
namaPegawai1 = "Ahmad"
agama = "Islam"
jumlahAnak = 2
gaji_pokok = 4000000
tunjangan_jabatan = 20/100
tunjangan_keluarga = 10/100
gaji_kotor = gaji_pokok + 800000 + 400000
zakat_profesi = gaji_kotor - 2.5/100
take_home_pay = gaji_pokok + 800000 + 400000
# variable2
namaPegawai2 = "Alex"
agama2 = "Kristen Protestan"
jumlahAnak2 = 5
gaji_Pokok2 = 6000000
tunjangan_jabatan2 = 20/100
tunjangan_keluarga2 = 20/100
gaji_kotor2 = gaji_Pokok2 + 1200000 + 1200000
zakat_profesi2 = gaji_kotor2 - 0
take_home_pay2 = gaji_Pokok2 + 1200000 + 1200000 - 0
# pegawai1
if gaji_pokok < 400000:
    tunjangan_jabatan = gaji_pokok * (20/100)
else:
    tunjangan_jabatan = gaji_pokok * (20/100)

if gaji_pokok < 400000:
    tunjangan_keluarga = gaji_pokok * (10/100)
else:
    tunjangan_keluarga = gaji_pokok * (10/100)

if gaji_kotor < 5200000:
    zakat_profesi = gaji_kotor * (2.5/100)
else:
    zakat_profesi = gaji_kotor * (2.5/100)
# pegawai2
if gaji_Pokok2 < 600000:
    tunjangan_jabatan2 = gaji_Pokok2 * (20/100)
else:
    tunjangan_jabatan2 = gaji_Pokok2 * (20/100)

if gaji_Pokok2 < 600000:
    tunjangan_keluarga2 = gaji_Pokok2 * (20/100)
else:
    tunjangan_keluarga2 = gaji_Pokok2 * (20/100)

if gaji_kotor2 < 6400000:
    zakat_profesi2 =  0
else:
    zakat_profesi2 =  0

# print pegawai1
print("SLIP GAJI",slipGaji,
"\n",garis,
"\nNama Pegawai\t\t\t:",namaPegawai1,
"\nAgama\t\t\t\t:",agama,
"\nJumblah Anak\t\t\t:",jumlahAnak,
"\nGaji Pokok\t\t\t:", gaji_pokok)

tunjangan_jabatan = gaji_pokok + tunjangan_jabatan
print('tunjangan jabatan               : {}'.format(int(tunjangan_jabatan)))

tunjangan_keluarga = gaji_pokok + tunjangan_keluarga
print('tunjangan keluarga              : {}'.format(int(tunjangan_keluarga)))

gaji_kotor = gaji_pokok + 400000 + 800000
print('gaji kotor                      : {}'.format(int(gaji_kotor)))

zakat_profesi = gaji_kotor - zakat_profesi
print('zakat profesi                   : {}'.format(int(zakat_profesi)))

take_home_pay = gaji_pokok + 400000 + 800000 - 130000
print('take_home_pay                   : {}'.format(int(take_home_pay)))
# print pegawai2
print("\nSLIP GAJI",slipGaji,
"\n",garis,
"\nNama Pegawai\t\t\t:",namaPegawai2,
"\nAgama\t\t\t\t:",agama2,
"\nJumlah Anak\t\t\t:",jumlahAnak2,
"\nGaji Pokok\t\t\t:",gaji_Pokok2)

tunjangan_jabatan2 = gaji_Pokok2 + tunjangan_jabatan2
print('tunjangan jabatan               : {}'.format(int(tunjangan_jabatan2)))

tunjangan_keluarga2 = gaji_Pokok2 + tunjangan_keluarga2
print('tunjangan keluarga              : {}'.format(int(tunjangan_keluarga2)))

gaji_kotor2 = gaji_Pokok2 + 1200000 + 1200000
print('gaji kotor                      : {}'.format(int(gaji_kotor2)))

zakat_profesi2 =  0
print('zakat profesi                   : {}'.format(int(zakat_profesi2)))

take_home_pay2 = gaji_Pokok2 + 1200000 + 1200000 - 0
print('take_home_pay                   : {}'.format(int(take_home_pay2)))


#!/usr/bin/env python3.7

import argparse
import json
import sys

def matches(parts, subject):
    if len(parts) == 0:
        yield subject
        return

    part, *rest = parts

    # If we're extracting something from `subject`, and `subject` is neither a
    # list nor a dict, then there's nothing to extract.  Whether this is an
    # error or just a no-op was part of how my original solution was wrong.
    if type(subject) not in [list, dict]:
        return

    if type(subject) is list:
        if part == '*':
            for child in subject:
                yield from matches(rest, child)
            return

        try:
            index = int(part)
        except ValueError:
            return  # can't extract a property name from a list

        yield from matches(rest, subject[index])
    else:
        assert type(subject) is dict

        if part == '*':
            for child in subject.values():
                yield from matches(rest, child)
        elif part in subject:
            yield from matches(rest, subject[part])

def parse(pattern):
    # Corner case:  If the pattern is empty, then splitting on "." would yield
    # `[""]` instead of `[]`.
    if len(pattern) == 0:
        return []
    else:
        return pattern.split('.')

def extract(pattern, subject):
    parts = parse(pattern)
    results = list(matches(parts, subject))

    # If there were no wildcards in the query, then at most one thing can be
    # matched.  Avoid the redundant outer list when possible.
    if '*' in parts:
        return results  # list of results

    if len(results) == 0:
        return None

    assert len(results) == 1
    return results[0]

def parse_command_line(args):
    parser = argparse.ArgumentParser(description='Extract values from JSON.')

    parser.add_argument('pattern',
                        help='JSON query (path) to extract from input')

    return parser.parse_args()

if __name__ == '__main__':
    options = parse_command_line(sys.argv[1:])
    result = extract(options.pattern, json.load(sys.stdin))

    if result is not None:
        json.dump(result, sys.stdout, indent=4, sort_keys=True)

    print()  # for the newline


class newNode:
    def __init__(self, data):
        self.data = data
        self.left = self.right = None

# Function to find height of a tree
def height(root, ans):
    if (root == None):
        return 0

    left_height = height(root.left, ans)

    right_height = height(root.right, ans)

    ans[0] = max(ans[0], 1 + left_height +
                             right_height)

    return 1 + max(left_height,
                   right_height)

def diameter(root):
    if (root == None):
        return 0
    ans = [-999999999999]
    height_of_tree = height(root, ans)
    return ans[0]

# Driver code
if __name__ == '__main__':
    root = newNode(1)
    root.left = newNode(2)
    root.right = newNode(3)
    root.left.left = newNode(4)
    root.left.right = newNode(5)

    print("Diameter is", diameter(root))

advance/dyanamic programing/dp on tree/problem/diameter of tree (binary)/firstapproch.py


class Nodo:

    def __init__(self, tipo, nombre=None, isArray=None, direccion=None, siguiente=None, etiquetaTrue=None, etiquetaFalse=None):
        self.tipo = tipo  # Tipo de dato
        self.nombre = nombre  # Nombre del nodo donde retornamos el valor
        self.isArray = isArray
        self.direccion = direccion
        self.codigo = []
        self.siguiente = siguiente
        self.etiquetaTrue = etiquetaTrue
        self.etiquetaFalse = etiquetaFalse

    def __repr__(self):
        return f"<Nodo>{self.tipo} {self.nombre}"


import time

from server.decorators import with_logger
from server.rating import RatingType

from .game import Game
from .typedefs import GameType, InitMode, ValidityState

@with_logger
class CustomGame(Game):
    init_mode = InitMode.NORMAL_LOBBY
    game_type = GameType.CUSTOM

    def __init__(self, id_, *args, **kwargs):
        new_kwargs = {
            "rating_type": RatingType.GLOBAL,
            "setup_timeout": 30
        }
        new_kwargs.update(kwargs)
        super().__init__(id_, *args, **new_kwargs)

    async def _run_pre_rate_validity_checks(self):
        limit = len(self.players) * 60
        if not self.enforce_rating and time.time() - self.launched_at < limit:
            await self.mark_invalid(ValidityState.TOO_SHORT)


#
# @lc app=leetcode id=118 lang=python3
#
# [118] Pascal's Triangle
#
# https://leetcode.com/problems/pascals-triangle/description/
#
# algorithms
# Easy (58.01%)
# Likes:    11256
# Dislikes: 361
# Total Accepted:    1.4M
# Total Submissions: 1.9M
# Testcase Example:  '5'
#
# Given an integer numRows, return the first numRows of Pascal's triangle.
#
# In Pascal's triangle, each number is the sum of the two numbers directly
# above it as shown:
#
#
# Example 1:
# Input: numRows = 5
# Output: [[1],[1,1],[1,2,1],[1,3,3,1],[1,4,6,4,1]]
# Example 2:
# Input: numRows = 1
# Output: [[1]]
#
#
# Constraints:
#
#
# 1 <= numRows <= 30
#
#
#

# @lc code=start
class Solution:
    def generate(self, numRows: int) -> List[List[int]]:

        res = [[0 for _ in range(i + 1)] for i in range(numRows)]
        for row in res:
            row[0] = 1
            row[-1] = 1

        for i in range(1, numRows):
            for j in range(1, len(res[i]) - 1):
                res[i][j] = res[i - 1][j - 1] + res[i - 1][j]

        return res
# @lc code=end


import os.path
from flask import Flask
from flaskext.sqlalchemy import SQLAlchemy

CONFIG_FILEPATH = os.path.join(os.path.dirname(__file__), "../config.cfg")

def auto_register_modules(app):
    """Registers modules from :mod:`subleekr` to application."""
    import subleekr
    for modname in subleekr.__modules__:
        __import__("{0}.{1}".format(subleekr.__name__, modname))
        module = getattr(subleekr, modname)
        module.app.super_app = app
        app.register_module(module.app)

def create_app(__name__=__name__):
    app = Flask(__name__)
    try:
        app.config.from_pyfile(CONFIG_FILEPATH)
    except IOError:
        pass
    auto_register_modules(app)
    app.db = SQLAlchemy(app)
    return app