import os
import threading
from importlib_metadata import metadata
import torch
from param_file.params import *
import random
import torchaudio
import numpy as np
from torch_time_stretch import *
import torchaudio
import json

import torch.nn.functional as F
import time
import glob
import h5py
# from plot import plt_netout
# from add_beeping import beeping
torchaudio.set_audio_backend("soundfile")

"""
返回beat、downbeat、non_beat三个数组
"""

class DownbeatDataset(torch.utils.data.Dataset):
    """
        class
    """
    def __init__(self,
                 h5_dir,
                 audio_sample_rate=16000,
                 target_factor=32, # 应该和hop_length是相同的
                 dataset="ballroom",
                 folder = 0,
                 subset="train",
                 length=160000,
                 augment=False,
                 pad_mode='constant',
                 device="cpu",
                 rand = True,
                 h5_file_augment = False,
                 randseed = 16):
        self.audio_sample_rate = audio_sample_rate
        self.target_factor = target_factor
        self.target_sample_rate = audio_sample_rate / target_factor
        self.subset = subset
        self.dataset = dataset
        self.audio_dir = os.path.join(h5_dir, dataset)
        self.json_file = os.path.join(h5_dir, "dict", "8_fold_h5.json")
        self.folder = folder
        self.length = length
        self.augment = augment
        self.pad_mode = pad_mode
        self.device = device

        self.target_length = int(self.length / self.target_factor)+1
        self.inst_len = self.length/self.audio_sample_rate

        self.h5_files = []
        h5_ext = ".h5"

        # get all of the annotation files
        if dataset == "gtzan":
            self.former_h5_files = glob.glob(os.path.join(self.audio_dir, "**", "*" + h5_ext))
            if len(self.former_h5_files) == 0: # try from the root audio dir
                self.former_h5_files = glob.glob(os.path.join(self.audio_dir, "*" + h5_ext))
        else:
            train_list, val_list, test_list = self.get_files()

            # eight folder validation
            if self.subset == "train":
                self.former_h5_files = train_list
            elif self.subset == "val":
                self.former_h5_files = val_list
            elif self.subset == "test":
                self.former_h5_files = test_list
            elif self.subset in ["full-train", "full-val"]:
                self.former_h5_files = train_list + val_list + test_list

        if h5_file_augment:
            self.h5_file_augmentation()
        else:
            self.h5_files = self.former_h5_files

        if rand:
            random.seed(randseed)
            random.shuffle(self.h5_files) # shuffle them

        print(f"Selected {len(self.h5_files)} files for {self.subset} set from {self.dataset} dataset.")

    def get_files(self):
        with open(self.json_file, 'r') as jf:
            json_dict = json.load(jf)

        val_num = (self.folder+5) % 8
        test_num = (self.folder+6) % 8
        train_list = []
        val_list = []
        test_list = []
        dataset_files = json_dict[self.dataset]
        for i in range(8):
            if i == val_num:
                val_list = dataset_files[i]
            elif i == test_num:
                test_list = dataset_files[i]
            else:
                for file in dataset_files[i]:
                    train_list.append(file)
        return train_list, val_list, test_list

    def __len__(self):
        length = len(self.h5_files)
        return length

    def __getitem__(self, idx):
        # 创建迭代器
        # start_time = time.time()
        # get metadata of example
        h5_file = self.h5_files[idx % len(self.h5_files)]
        # print(h5_file)
        audio, target_beat, target_tempo, target_chord, metadata = self.load_data(h5_file)

        # do all processing in float32 not float16
        audio = audio.float()
        target_beat = target_beat.float()

        # calculate time stretch factor
        scale_factor = 1.0
        if self.augment:
            scale_factor = np.random.normal(1.0, 0.5)  # 正态分布
            scale_factor = np.clip(scale_factor, a_min=0.6, a_max=1.48)

        N_audio = audio.shape[-1]   # audio samples

        # -- remove the data if doesn't fit the condition -- #
        if N_audio < int(scale_factor*self.length):
            # del self.audio_path[index]
            new_i = torch.randint(0, self.__len__(), (1,))
            new_i = new_i[0]
            return self.__getitem__(new_i)

        # random crop of the audio and target if larger than desired
        if (N_audio > self.length*scale_factor) and self.subset not in ['val', 'test', 'full-val']:
            if int(N_audio -self.length*scale_factor) - 1== 0:
                audio_start = 0
            else:
                audio_start = random.randint(0, int(N_audio -self.length*scale_factor) -1)

            audio_stop  = int(audio_start + self.length*scale_factor)
            target_start = int(audio_start / self.target_factor)
            target_stop = int(audio_stop / self.target_factor)
            audio_cut = audio[:,audio_start:audio_stop]
            target_beat_c = target_beat[:,target_start:target_stop]
            target_beat = target_beat_c
            audio = audio_cut

        # pad the audio and target if is shorter than desired
        if audio.shape[-1] < int(self.length*scale_factor) and self.subset not in ['val', 'test', 'full-val']:
            pad_size = int(self.length*scale_factor - audio.shape[-1])
            padl = pad_size - int(pad_size / 2)
            padr = int(pad_size / 2)
            audio = F.pad(audio, (padl, padr), mode=self.pad_mode)
        if target_beat.shape[-1] < int(self.target_length*scale_factor) and self.subset not in ['val', 'test', 'full-val']:
            pad_size = int(self.target_length*scale_factor - target_beat.shape[-1])
            padl = pad_size - int(pad_size / 2)
            padr = int(pad_size / 2)
            target_beat = F.pad(target_beat, (padl, padr))

        # do augmentation if needed
        if self.augment:
            audio, target_beat, target_tempo, target_chord = self.apply_augmentations(audio, target_beat, target_tempo, target_chord, scale_factor=scale_factor)

        if audio.shape[-1] > self.length and self.subset not in ['val', 'test', 'full-val']:
            audio = audio[:,:self.length]

        ifdownbeat = 1
        no_beat = 1-target_beat[0,:]
        target_beat = torch.cat([target_beat, no_beat.unsqueeze(0)],dim=0)
        if metadata["dataset"] == "smc":
            ifdownbeat = 0
        return audio, target_beat, target_tempo, target_chord, metadata, ifdownbeat

    def h5_file_augmentation(self):
        if self.dataset == "hainsworth" or self.dataset =="hjdb" or self.dataset == "beatles" or self.dataset =="smc" or self.dataset == "beatles2":
            for path in self.former_h5_files:
                self.h5_files.extend([path]*3)
        if self.dataset == "rwc_popular":
            for path in self.former_h5_files:
                self.h5_files.extend([path]*6)
        if self.dataset == "ballroom":
            for path in self.former_h5_files:
                self.h5_files.append(path)
        print("dataset {} successfully augmented".format(self.dataset))

    def load_data(self, h5_file):
        # first load the audio file
        audio, sr, beat_samples, beat_sec, downbeat_samples, downbeat_sec, tempo, chord_samples, filename, Genre, Time_signature = self.get_h5data(h5_file)

        # transfer to tensor
        audio = torch.from_numpy(audio)

        # transfer stereo to mono if needed
        dim = audio.shape
        if dim[0] != 1:
            audio = (audio[0] + audio[1]) / 2
            audio = audio.unsqueeze(0)

        # normalize all audio inputs -1 to 1
        audio /= audio.abs().max()

        t = audio.shape[-1]/self.audio_sample_rate # audio length in sec
        N = int(t * self.target_sample_rate) + 1   # audio length in samples
        target_beat = torch.zeros(2,N)
        if tempo != None:
            if type(tempo) is bytes:
                tempo = int(float(tempo.decode('utf-8')))
                target_tempo = torch.tensor(tempo)
            else:

                target_tempo = torch.tensor(tempo)
        else:
            target_tempo = torch.zeros(1)
        # TODO: not assigned right now
        target_chord = torch.zeros(1)

        # now convert from seconds to new sample rate
        beat_samples = np.array(beat_sec * self.target_sample_rate)
        downbeat_samples = np.array(downbeat_sec * self.target_sample_rate)

        # check if there are any beats beyond the file end and cut them off
        beat_samples = beat_samples[beat_samples < N]
        downbeat_samples = downbeat_samples[downbeat_samples < N]

        beat_samples = beat_samples.astype(int)
        downbeat_samples = downbeat_samples.astype(int)

        target_beat[0,beat_samples] = 1  # first channel is beats
        target_beat[1,downbeat_samples] = 1  # second channel is downbeats

        dataset = h5_file.split("/")[-2]

        # print(threading.enumerate())
        metadata = {
            "Filename" : filename,
            "Genre" : Genre,
            "dataset" : dataset,
            "Time signature" : Time_signature
        }
        return audio, target_beat, target_tempo, target_chord, metadata

    def apply_augmentations(self, audio, target_beat, target_tempo, target_chord, scale_factor=1.0):
        # phase inversion
        if np.random.rand() < 0.5:
            audio = -audio

        # apply time stretching on GPU
        if scale_factor != 1.0:
            audio = audio.unsqueeze(0)
            audio = time_stretch(audio, float(1/scale_factor), self.audio_sample_rate)
            audio = audio.squeeze(0)

            # change tempo to stretched version
            target_tempo = target_tempo * scale_factor

            # now we update the targets beat and downbeat based on new tempo
            dbeat_ind = (target_beat[1,:] == 1).nonzero(as_tuple=False)
            dbeat_sec = dbeat_ind / self.target_sample_rate
            new_dbeat_sec = (dbeat_sec / scale_factor).squeeze()
            new_dbeat_ind = (new_dbeat_sec * self.target_sample_rate).long()

            beat_ind = (target_beat[0,:] == 1).nonzero(as_tuple=False)
            beat_sec = beat_ind / self.target_sample_rate
            new_beat_sec = (beat_sec / scale_factor).squeeze()
            new_beat_ind = (new_beat_sec * self.target_sample_rate).long()

            new_size = self.target_length
            # cut the beat index and downbeat index below new size
            new_beat_pos = torch.where(new_beat_ind < new_size)[0]
            new_downbeat_pos = torch.where(new_dbeat_ind < new_size)[0]
            if new_beat_pos.shape[-1] != 1 and new_beat_pos.shape[-1] != 0:
                new_beat_ind = new_beat_ind[:(new_beat_pos[-1]+1)]
            if new_downbeat_pos.shape[-1] != 1 and new_downbeat_pos.shape[-1] != 0:
                new_dbeat_ind = new_dbeat_ind[:(new_downbeat_pos[-1]+1)]

            # now convert indices back to target_beat vector
            stretched_target = torch.zeros(2,new_size)
            if new_beat_pos.shape[-1] != 0:
                stretched_target[0,new_beat_ind] = 1
            if new_downbeat_pos.shape[-1] != 0:
                stretched_target[1,new_dbeat_ind] = 1
            target_beat = stretched_target

        # do some paddings
        if audio.shape[-1] < self.length:
            pad_size = int(self.length - audio.shape[-1])
            padl = pad_size - (pad_size // 2)
            padr = pad_size // 2
            audio = torch.nn.functional.pad(audio,
                                             (padl, padr),
                                             mode=self.pad_mode)

        # normalize the audio
        audio /= audio.float().abs().max()

        return audio, target_beat, target_tempo, target_chord

    def get_h5data(self, h5_file):
        # initialization
        beat_samples = None
        downbeat_samples = None
        tempo = None
        chord_samples = None
        Genre = "None"
        Time_signature = "None"

        f = h5py.File(h5_file, 'r')
        audio = f["audio_16k"][()] # import audio of 16k hz
        sr = f["sr"][()]
        if "filename" in f.keys():
            filename =f["filename"][()]
        if "Filename" in f.keys():
            filename =f["Filename"][()]
        if "beat_samples" in f.keys():
            beat_samples =f["beat_samples"][()]
        if "beat_sec" in f.keys():
            beat_sec = f["beat_sec"][()]
        if "downbeat_samples" in f.keys():
            downbeat_samples = f["downbeat_samples"][()]
        if "downbeat_sec" in f.keys():
            downbeat_sec = f["downbeat_sec"][()]
        if "Genre" in f.keys():
            Genre = f["Genre"][()]
        if "Time signature" in f.keys():
            Time_signature = f["Time signature"][()]
        if "tempo" in f.keys():
            tempo = f["tempo"][()]
        if "chord_samples" in f.keys():
            chord_samples = f["chord_samples"][()]

        f.close()
        return audio, sr, beat_samples, beat_sec, downbeat_samples, downbeat_sec, tempo, chord_samples, filename, Genre, Time_signature

if __name__ == "__main__":
    # torch.multiprocessing.set_start_method('spawn')
    DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'
    INDEX = 0
    dataset = "ballroom"
    device = torch.device(DEVICE, index=INDEX)
    train_loader = DownbeatDataset(h5_dir=root_path,
                            audio_sample_rate=16000,
                            target_factor=hop_length,
                            dataset=dataset,
                            folder=1,
                            subset="train",
                            length=160000,
                            augment=True,
                            pad_mode='constant',
                            h5_file_augment=True,
                            # device=device,
                            )
    train_dataloader = torch.utils.data.DataLoader(train_loader,
                                                shuffle=True,
                                                batch_size=8,
                                                num_workers=2,
                                                pin_memory=False)

    val_loader = DownbeatDataset(h5_dir=root_path,
                            audio_sample_rate=16000,
                            target_factor=hop_length,
                            dataset=dataset,
                            subset="full-val",
                            length=160000,
                            augment=False,
                            pad_mode='constant')
    val_dataloader = torch.utils.data.DataLoader(val_loader,
                                                shuffle=True,
                                                batch_size=1,
                                                num_workers=2,
                                                pin_memory=False)

    for ii, data in enumerate(train_dataloader):
        audio, target_beat, target_tempo, target_chord, meta_data, _ = data
        # beeping(audio, target_beat=target_beat, sample_rate=hop_length,name=meta_data["Filename"], beeping_file="D:/coding/beat/BeatNet/BeatNet/src/BeatNet/beeping/beep.wav", target_path="D:/coding/beat/BeatNet/BeatNet/src/BeatNet/audio_output")
        print("{}/{}".format(ii, len(train_dataloader)))


import cupy
import numpy as np
import pytest
from pandas.util.testing import assert_series_equal

import cudf
from cudf.tests.utils import assert_eq

def test_tokenize():
    strings = cudf.Series(
        [
            "the quick fox jumped over the lazy dog",
            "the siamÃ©sÃ© cat jumped under the sofa",
            None,
            "",
        ]
    )

    expected = cudf.Series(
        [
            "the",
            "quick",
            "fox",
            "jumped",
            "over",
            "the",
            "lazy",
            "dog",
            "the",
            "siamÃ©sÃ©",
            "cat",
            "jumped",
            "under",
            "the",
            "sofa",
        ]
    )

    actual = strings.str.tokenize()

    assert type(expected) == type(actual)
    assert_series_equal(expected.to_pandas(), actual.to_pandas())

@pytest.mark.parametrize(
    "delimiter, expected_token_counts",
    [
        ("", [10, 9, 0, 0, 5]),
        ("o", [6, 3, 0, 0, 1]),
        (["a", "e", "i", "o", "u"], [13, 13, 0, 0, 6]),
        (["a", "e", "i", "o"], [12, 11, 0, 0, 6]),
    ],
)
def test_token_count(delimiter, expected_token_counts):
    strings = cudf.Series(
        [
            "the quick brown fox jumped over the lazy brown dog",
            "the sable siamÃ©sÃ© cat jumped under the brown sofa",
            None,
            "",
            "test_str\x01test_str\x02test_str\x03test_str\x04test_str\x05",
        ]
    )

    expected = cudf.Series(expected_token_counts)

    actual = strings.str.token_count(delimiter)

    assert type(expected) == type(actual)
    assert_series_equal(
        expected.to_pandas(), actual.to_pandas(), check_dtype=False
    )

def test_normalize_spaces():
    strings = cudf.Series(
        [
            " the\t quick fox  jumped over the lazy dog",
            "the siamÃ©sÃ© cat\f jumped\t\tunder the sofa  ",
            None,
            "",
        ]
    )
    expected = cudf.Series(
        [
            "the quick fox jumped over the lazy dog",
            "the siamÃ©sÃ© cat jumped under the sofa",
            None,
            "",
        ]
    )

    actual = strings.str.normalize_spaces()

    assert type(expected) == type(actual)
    assert_series_equal(expected.to_pandas(), actual.to_pandas())

@pytest.mark.parametrize(
    "n, separator, expected_values",
    [
        (
            2,
            "_",
            [
                "this_is",
                "is_my",
                "my_favorite",
                "favorite_book",
                "book_on",
                "on_my",
                "my_bookshelf",
            ],
        ),
        (
            3,
            "-",
            [
                "this-is-my",
                "is-my-favorite",
                "my-favorite-book",
                "favorite-book-on",
                "book-on-my",
                "on-my-bookshelf",
            ],
        ),
    ],
)
def test_ngrams(n, separator, expected_values):
    strings = cudf.Series(
        ["this", "is", "my", "favorite", "book", "on", "my", "bookshelf"]
    )

    expected = cudf.Series(expected_values)

    actual = strings.str.ngrams(n=n, separator=separator)

    assert type(expected) == type(actual)
    assert_series_equal(expected.to_pandas(), actual.to_pandas())

@pytest.mark.parametrize(
    "n, expected_values",
    [
        (
            2,
            [
                "th",
                "hi",
                "is",
                "is",
                "my",
                "bo",
                "oo",
                "ok",
                "he",
                "er",
                "re",
            ],
        ),
        (3, ["thi", "his", "boo", "ook", "her", "ere"]),
    ],
)
def test_character_ngrams(n, expected_values):
    strings = cudf.Series(["this", "is", "my", "book", "here", ""])

    expected = cudf.Series(expected_values)

    actual = strings.str.character_ngrams(n=n)

    assert type(expected) == type(actual)
    assert_series_equal(expected.to_pandas(), actual.to_pandas())

@pytest.mark.parametrize(
    "n, separator, expected_values",
    [
        (
            2,
            "_",
            [
                "this_is",
                "is_my",
                "my_favorite",
                "book_on",
                "on_my",
                "my_bookshelf",
            ],
        ),
        (
            3,
            "-",
            ["this-is-my", "is-my-favorite", "book-on-my", "on-my-bookshelf"],
        ),
    ],
)
def test_ngrams_tokenize(n, separator, expected_values):
    strings = cudf.Series(["this is my favorite", "book on my bookshelf"])

    expected = cudf.Series(expected_values)

    actual = strings.str.ngrams_tokenize(n=n, separator=separator)

    assert type(expected) == type(actual)
    assert_series_equal(expected.to_pandas(), actual.to_pandas())

def test_character_tokenize_series():
    sr = cudf.Series(
        [
            "hello world",
            "sdf",
            None,
            "goodbye, one-two:three~four+five_six@sev"
            "en#eight^nine heÅ’Å½â€˜â€¢â„¢Å“$ÂµÂ¾Å¤Æ Ã© Ç„",
        ]
    )
    expected = cudf.Series(
        [
            "h",
            "e",
            "l",
            "l",
            "o",
            " ",
            "w",
            "o",
            "r",
            "l",
            "d",
            "s",
            "d",
            "f",
            "g",
            "o",
            "o",
            "d",
            "b",
            "y",
            "e",
            ",",
            " ",
            "o",
            "n",
            "e",
            "-",
            "t",
            "w",
            "o",
            ":",
            "t",
            "h",
            "r",
            "e",
            "e",
            "~",
            "f",
            "o",
            "u",
            "r",
            "+",
            "f",
            "i",
            "v",
            "e",
            "_",
            "s",
            "i",
            "x",
            "@",
            "s",
            "e",
            "v",
            "e",
            "n",
            "#",
            "e",
            "i",
            "g",
            "h",
            "t",
            "^",
            "n",
            "i",
            "n",
            "e",
            " ",
            "h",
            "e",
            "Å’",
            "Å½",
            "â€˜",
            "â€¢",
            "â„¢",
            "Å“",
            "$",
            "Âµ",
            "Â¾",
            "Å¤",
            "Æ ",
            "Ã©",
            " ",
            "Ç„",
        ]
    )

    actual = sr.str.character_tokenize()
    assert_eq(expected, actual)

    sr = cudf.Series([""])
    expected = cudf.Series([], dtype="object")

    actual = sr.str.character_tokenize()
    assert_eq(expected, actual)

    sr = cudf.Series(["a"])
    expected = cudf.Series(["a"])

    actual = sr.str.character_tokenize()
    assert_eq(expected, actual)

def test_character_tokenize_index():
    sr = cudf.core.index.as_index(
        [
            "hello world",
            "sdf",
            None,
            "goodbye, one-two:three~four+five_six@sev"
            "en#eight^nine heÅ’Å½â€˜â€¢â„¢Å“$ÂµÂ¾Å¤Æ Ã© Ç„",
        ]
    )
    expected = cudf.core.index.as_index(
        [
            "h",
            "e",
            "l",
            "l",
            "o",
            " ",
            "w",
            "o",
            "r",
            "l",
            "d",
            "s",
            "d",
            "f",
            "g",
            "o",
            "o",
            "d",
            "b",
            "y",
            "e",
            ",",
            " ",
            "o",
            "n",
            "e",
            "-",
            "t",
            "w",
            "o",
            ":",
            "t",
            "h",
            "r",
            "e",
            "e",
            "~",
            "f",
            "o",
            "u",
            "r",
            "+",
            "f",
            "i",
            "v",
            "e",
            "_",
            "s",
            "i",
            "x",
            "@",
            "s",
            "e",
            "v",
            "e",
            "n",
            "#",
            "e",
            "i",
            "g",
            "h",
            "t",
            "^",
            "n",
            "i",
            "n",
            "e",
            " ",
            "h",
            "e",
            "Å’",
            "Å½",
            "â€˜",
            "â€¢",
            "â„¢",
            "Å“",
            "$",
            "Âµ",
            "Â¾",
            "Å¤",
            "Æ ",
            "Ã©",
            " ",
            "Ç„",
        ]
    )

    actual = sr.str.character_tokenize()
    assert_eq(expected, actual)

    sr = cudf.core.index.as_index([""])
    expected = cudf.core.index.StringIndex([], dtype="object")

    actual = sr.str.character_tokenize()
    assert_eq(expected, actual)

    sr = cudf.core.index.as_index(["a"])
    expected = cudf.core.index.as_index(["a"])

    actual = sr.str.character_tokenize()
    assert_eq(expected, actual)

def test_text_replace_tokens():
    sr = cudf.Series(["this is me", "theme music", ""])
    targets = cudf.Series(["is", "me"])

    expected = cudf.Series(["this _ _", "theme music", ""])
    actual = sr.str.replace_tokens(targets, "_")

    assert_eq(expected, actual)

    replacements = cudf.Series(["IS", "ME"])
    expected = cudf.Series(["this IS ME", "theme music", ""])
    actual = sr.str.replace_tokens(targets, replacements)

    assert_eq(expected, actual)

    sr = cudf.Series(
        [
            "this is a small text â˜•",
            "this \t\t is ; ; - + a looooooooooonnnnnnnggggggg text \n\t",
            "emptyme",
        ],
    )
    targets = cudf.Series(
        ["a", "â˜•", "\t", "looooooooooonnnnnnnggggggg", "emptyme"]
    )
    replacements = cudf.Series(["the", "ðŸš’", "ðŸš’ðŸš’ðŸš’ðŸš’", "ðŸ”¥ðŸ”¥", ""])

    expected = cudf.Series(
        [
            "this is the small text ðŸš’",
            "this \t\t is ; ; - + the ðŸ”¥ðŸ”¥ text \n\t",
            "",
        ]
    )
    actual = sr.str.replace_tokens(targets, replacements)

    assert_eq(expected, actual)

    sr = cudf.Series(
        ["All-we-need;is;ðŸ”¥", "\tall-we-need0is;ðŸŒŠ", "all;we:need+is;ðŸŒ¬"]
    )
    targets = cudf.Series(["ðŸŒ¬", "ðŸ”¥", "ðŸŒŠ"])
    replacements = "ðŸš°"

    expected = cudf.Series(
        ["All-we-need;is;ðŸš°", "\tall-we-need0is;ðŸš°", "all;we:need+is;ðŸš°"]
    )
    actual = sr.str.replace_tokens(targets, replacements, delimiter=";")

    assert_eq(expected, actual)
    assert_eq(sr, sr.str.replace_tokens(targets, replacements))
    assert_eq(sr, sr.str.replace_tokens([""], [""]))

def test_text_replace_tokens_error_cases():
    sr = cudf.Series(["this is me", "theme music", ""])

    with pytest.raises(
        TypeError,
        match="targets should be an array-like or a Series object, "
        "found <class 'str'>",
    ):
        sr.str.replace_tokens("me", ["a"])

    with pytest.raises(
        ValueError,
        match="targets and replacements should be same size"
        " sequences unless replacements is a string.",
    ):
        sr.str.replace_tokens(["a"], ["me", "ki"])

    with pytest.raises(
        TypeError,
        match="replacements should be an str, array-like or Series object,"
        " found <class 'set'>",
    ):
        sr.str.replace_tokens(["a"], {"s"})

    with pytest.raises(
        TypeError,
        match="Type of delimiter should be a string, found <class 'list'>",
    ):
        sr.str.replace_tokens(["a"], ["s"], delimiter=["a", "b"])

def test_text_subword_tokenize(tmpdir):
    sr = cudf.Series(
        [
            "This is a test",
            "A test this is",
            "Is test a this",
            "Test   test",
            "this   This",
        ]
    )
    hash_file = tmpdir.mkdir("nvtext").join("tmp_hashed_vocab.txt")
    content = "1\n0\n23\n"
    coefficients = [65559] * 23
    for c in coefficients:
        content = content + str(c) + " 0\n"
    # based on values from the bert_hash_table.txt file for the
    # test words used here: 'this' 'is' 'a' test'
    table = [0] * 23
    table[0] = 3015668
    table[1] = 6205475701751155871
    table[5] = 6358029
    table[16] = 451412625363
    table[20] = 6206321707968235495
    content = content + "23\n"
    for v in table:
        content = content + str(v) + "\n"
    content = content + "100\n101\n102\n\n"
    hash_file.write(content)

    tokens, masks, metadata = sr.str.subword_tokenize(str(hash_file), 8, 8)
    expected_tokens = cupy.asarray(
        [
            2023,
            2003,
            1037,
            3231,
            0,
            0,
            0,
            0,
            1037,
            3231,
            2023,
            2003,
            0,
            0,
            0,
            0,
            2003,
            3231,
            1037,
            2023,
            0,
            0,
            0,
            0,
            3231,
            3231,
            0,
            0,
            0,
            0,
            0,
            0,
            2023,
            2023,
            0,
            0,
            0,
            0,
            0,
            0,
        ],
        dtype=np.uint32,
    )
    assert_eq(expected_tokens, tokens)

    expected_masks = cupy.asarray(
        [
            1,
            1,
            1,
            1,
            0,
            0,
            0,
            0,
            1,
            1,
            1,
            1,
            0,
            0,
            0,
            0,
            1,
            1,
            1,
            1,
            0,
            0,
            0,
            0,
            1,
            1,
            0,
            0,
            0,
            0,
            0,
            0,
            1,
            1,
            0,
            0,
            0,
            0,
            0,
            0,
        ],
        dtype=np.uint32,
    )
    assert_eq(expected_masks, masks)

    expected_metadata = cupy.asarray(
        [0, 0, 3, 1, 0, 3, 2, 0, 3, 3, 0, 1, 4, 0, 1], dtype=np.uint32
    )
    assert_eq(expected_metadata, metadata)


"""
Given an array nums of n integers, are there elements a, b, c in nums such that a + b + c = 0? Find all unique triplets in the array which gives the sum of zero.

Note:

The solution set must not contain duplicate triplets.

Example:

Given array nums = [-1, 0, 1, 2, -1, -4],

A solution set is:
[
  [-1, 0, 1],
  [-1, -1, 2]
]
"""

class Solution:
    # @return a list of lists of length 3, [[val1,val2,val3]]
    def threeSum(self, num):
        num.sort()
        res = []
        for i in range(len(num)-2):
            if i == 0 or num[i] > num[i-1]:
                left = i + 1; right = len(num) - 1
                while left < right:
                    if num[left] + num[right] == -num[i]:
                        res.append([num[i], num[left], num[right]])
                        left += 1; right -= 1
                        while left < right and num[left] == num[left-1]: left +=1
                        while left < right and num[right] == num[right+1]: right -= 1
                    elif num[left] + num[right] < -num[i]:
                        while left < right:
                            left += 1
                            if num[left] > num[left-1]: break
                    else:
                        while left < right:
                            right -= 1
                            if num[right] < num[right+1]: break
        return res


import socket
import sys
import signal, os

class basic_server(object):
    def __init__(self, port):
        self._socket = socket.socket()
        self._socket.bind(("0.0.0.0", port))
        self._socket.listen(5)
        print("The server is running")
        self.__run()

    def __run(self):
        while (True):
            (new_socket, address) = self._socket.accept()
            message = new_socket.recv(1024)
            print(message)

if __name__ == '__main__':
    if len(sys.argv) < 2:
        print("error usage")
        sys.exit(1)

    port = int(sys.argv[1])
    basic_server(port)


import argparse
import copy
import os
import json

import numpy as np
from imgaug import augmenters as iaa
from tqdm import tqdm

import torch
from torch import nn
from torchvision import transforms
from torch.utils.data import DataLoader

from training_utils import (
    ImgAugTransform,
    AddRandomLines,
    TemplateClassifierDataset,
    adjust_learning_rate,
)
from models import TemplateNet

class TrainFormDetector(object):
    def __init__(
        self,
        path_2_positives,
        out_path,
        path_2_train_negatives="../data/train/negative/",
        path_2_val_negatives="../data/val/negative/",
        verbose=False,
    ):

        self.verbose = verbose
        self.out_path = out_path

        # Prep data set for training
        # List of additional augmentations not offered by pytorch
        aug_list = iaa.Sequential(
            [
                iaa.Sometimes(0.25, iaa.GaussianBlur(sigma=(0, 1.0))),
                iaa.Sometimes(0.25, iaa.AdditiveGaussianNoise(scale=(0, 0.2))),
                iaa.Sometimes(0.25, iaa.Dropout(p=(0, 0.3))),
                iaa.Sometimes(0.25, iaa.Cutout(nb_iterations=(1, 5), size=0.2, cval=1)),
            ]
        )

        # Form pixel value mean
        form_mu = (0.9094463458110517,)
        # Form pixel value std
        form_std = (0.1274794325726292,)

        # Data transformations for augmenting training data to mimic fax noise
        train_transf = transforms.Compose(
            [
                ImgAugTransform(aug_list),
                transforms.ToPILImage(),
                transforms.Resize((550, 425)),
                transforms.RandomVerticalFlip(),
                transforms.RandomHorizontalFlip(),
                transforms.RandomApply(
                    [
                        transforms.RandomAffine(
                            2.0,
                            translate=(0.03, 0.03),
                            scale=(0.5, 1.0),
                            shear=(2, 2),
                            fillcolor=1.0,
                        )
                    ]
                ),
                transforms.ToTensor(),
                transforms.RandomApply([AddRandomLines(5, 25)]),
                transforms.Normalize(form_mu, form_std),
            ]
        )

        # Load training data
        if self.verbose:
            print("Loading training data")
        self.train_data = TemplateClassifierDataset(
            path_2_positives,
            path_2_train_negatives,
            transforms=train_transf,
            verbose=self.verbose,
        )
        json.dump(self.train_data.index_2_class, open(os.path.join(self.out_path, "model_pred_key.json"), "w+"))
        # Data transformations for standard input
        val_transf = transforms.Compose(
            [
                transforms.ToPILImage(),
                transforms.Resize((550, 425)),
                transforms.ToTensor(),
                transforms.Normalize(form_mu, form_std),
            ]
        )
        if self.verbose:
            print("Loading validation data")
        self.val_data = TemplateClassifierDataset(
            path_2_positives,
            path_2_val_negatives,
            transforms=val_transf,
            verbose=self.verbose,
            bootstrap_template=False,
        )

    def write_history(self, train_histories, val_histories):
        with open(os.path.join(self.out_path, "train_history.csv"), "w+") as out_file:
            for i in range(len(train_histories)):
                for epoch in range(len(train_histories[i])):
                    out_file.write(
                        f"model{i},train,{epoch},{train_histories[i][epoch]}\n"
                    )
                    out_file.write(f"model{i},val,{epoch},{val_histories[i][epoch]}\n")

    def train_k_models(self, k=5, num_epochs=100, batch_size=64, learning_rate=0.001):

        num_classes = len(self.train_data.index_2_class)
        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

        model_training_history = []
        model_val_history = []

        self.train_loader = DataLoader(
            dataset=self.train_data, batch_size=batch_size, shuffle=True, num_workers=50
        )
        self.val_loader = DataLoader(
            dataset=self.val_data, batch_size=batch_size, num_workers=50
        )

        #### GO THROUGH CODE BELOW
        for i in range(k):
            train_loss_history = []
            val_loss_history = []

            best_train_loss = 50

            model = TemplateNet(num_classes)
            model.to(device)
            criterion = nn.CrossEntropyLoss()
            optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
            criterion.to(device)

            for epoch in range(1, num_epochs):
                _ = model.train()

                rolling_loss = 0
                for images, labels in tqdm(
                    self.train_loader,
                    total=len(self.train_loader),
                    disable=self.verbose == False,
                    desc=f"Epoch {epoch}:",
                ):
                    images = images.to(device, dtype=torch.float)

                    labels = torch.LongTensor(labels)
                    labels = labels.to(device, dtype=torch.long)
                    # Forward pass
                    outputs = model(images)

                    loss = criterion(outputs, labels)

                    # Backward and optimize
                    optimizer.zero_grad()
                    loss.backward()
                    optimizer.step()

                    rolling_loss += loss.item()

                # Test the model
                _ = model.eval()
                with torch.no_grad():
                    val_loss = 0
                    for images, labels in tqdm(self.val_loader,
                        total=len(self.val_loader),
                        disable=self.verbose == False,
                        desc=f"Epoch {epoch} Val:"):
                        images = images.to(device, dtype=torch.float)

                        labels = torch.LongTensor(labels)
                        labels = labels.to(device, dtype=torch.long)

                        outputs = model(images)

                        val_loss += criterion(outputs, labels)

                avg_epoch_loss = rolling_loss / len(self.train_loader)
                avg_val_loss = val_loss / len(self.val_loader)

                if avg_epoch_loss < best_train_loss:
                    best_train_loss = avg_epoch_loss
                    best_model = copy.deepcopy(model.state_dict())
                    rounds_since_best = 0
                else:
                    rounds_since_best += 1

                if rounds_since_best >= 10:
                    adjust_learning_rate(optimizer, 0.9)
                    rounds_since_best = 0

                train_loss_history.append(avg_epoch_loss)
                val_loss_history.append(avg_val_loss)
                train_loss_rounded = np.round(avg_epoch_loss, 4)
                val_loss_rounded = np.round(avg_val_loss.cpu().numpy(), 4)

                if self.verbose and epoch % 10 == 0:
                    print(
                        f"Epoch {epoch}, Train Loss: {train_loss_rounded}, Val Loss: {val_loss_rounded}"
                    )

            model_training_history.append(train_loss_history)
            model_val_history.append(train_loss_history)
            torch.save(
                best_model,
                os.path.join(
                    self.out_path, f"formDet_best{i}_{np.round(best_train_loss, 4)}.pt"
                ),
            )

        self.write_history(model_training_history, model_val_history)

        if self.verbose:
            print("Model training finished.")
            print(f"Find your trained models at {self.output_path}")

"""
Parse the command line
"""

def parse_command_line():
    parser = argparse.ArgumentParser(
        description="TrainFormDetector, Trains a set of k models to recognize templates"
    )
    requiredNamed = parser.add_argument_group("required arguments")
    requiredNamed.add_argument(
        "-p",
        "--positives_dir",
        help="Directory containing positive examples of each template",
        required=True,
    )
    requiredNamed.add_argument(
        "-O", "--output_dir", help="Output directory, must already exist", required=True
    )
    parser.add_argument(
        "-n",
        "--negatives_train_dir",
        help="Directory containing negative training examples",
        default="../data/train/negative/",
    )
    parser.add_argument(
        "-t",
        "--negatives_val_dir",
        help="Directory containing negative training examples",
        default="../data/val/negative/",
    )
    parser.add_argument(
        "-k", "--train_k_models", help="Number of models to train", default=5, type=int
    )
    parser.add_argument(
        "-v", "--verbose", help="Verbose mode", action="store_true", default=False
    )

    options = parser.parse_args()
    return options

"""
Main
"""
if __name__ == "__main__":
    options = parse_command_line()
    TFD = TrainFormDetector(
        options.positives_dir,
        options.output_dir,
        path_2_train_negatives=options.negatives_train_dir,
        path_2_val_negatives=options.negatives_val_dir,
        verbose=options.verbose,
    )
    TFD.train_k_models(options.train_k_models)


if __name__ != "__main__":
  exit()

def get_int():
  return int(input())

def is_prime(n):
  for crr in range(2, n - 1):
    if n % crr == 0:
      return False

  return True

# main

n = get_int()

if n == 1:
  print(1)
  exit()

crr = 1
count = 1

while count < n:
  crr = crr + 1
  for i in range(2, n + 1):
    if is_prime(i) and crr % i == 0:
      count = count + 1

      if n == count:
        print(i)
        break


def two_sum(nums, target):
    memo = {} # dictionary
    for v in nums:
        memo[v] = True
    for v in nums:
        needed_number = target - v
        if v == needed_number:
            continue
        elif needed_number in memo:
            return True
    return False

nums = [2,1,5,7]

result = two_sum(nums, 4)
print(result)


import pandas as pd
import os
import seaborn as sns
import matplotlib.pyplot as plt

path = os.path.abspath(path=os.path.dirname(__file__)) + '\dados.csv'
df = pd.read_csv(path)
sexo = {'0': 'masculino', '1': 'feminino'}
df['Sexo'] = df['Sexo'].apply(lambda x: 'masculino' if x == 1 and x < 5 else 'feminino')
#df['Desenpenho'] = df.apply(lambda x: 'Excelente' if x['Anos de Estudo'] < 8 and x['Renda'] > 5000 else 'Médio')
df_mulheres = df.query('Sexo == "feminino"')
media_anos_de_estudo_mulheres = df.groupby(by=['Sexo', 'Anos de Estudo'], as_index=False).aggregate({'Renda': 'mean', 'Idade': 'mean'})
df_masulino = media_anos_de_estudo_mulheres.query("Sexo == 'masculino'")

sns.barplot(x='Anos de Estudo', y='Renda', data=df_masulino, color='b')
plt.show()

print(df)


def tilingRectangle(n, m):
    """This is a wrong solution but can be accepted by leetcode."""
    if (n == 11 and m == 13) or (n == 13 and m == 11):
        return 6
    results = [[0] * (m + 1) for _ in range(n + 1)]
    for i in range(1, n + 1):
        for j in range(1, m + 1):
            if i == 1:
                results[i][j] = j
            elif j == 1:
                results[i][j] = i
            elif i == j:
                results[i][j] = 1
            else:
                minimum = 1e10
                for k in range(1, int(i / 2) + 1):
                    current = results[i - k][j] + results[k][j]
                    if current < minimum:
                        minimum = current

                for k in range(1, int(j / 2) + 1):
                    current = results[i][j - k] + results[i][k]
                    if current < minimum:
                        minimum = current
                results[i][j] = minimum

    return results[n][m]


import tensorflow as tf
import keras

from keras import layers
from keras.models import Model
from keras.layers import Dense, Activation, Dropout, Conv2D, GlobalAveragePooling2D, Input, Lambda
import keras.backend as K
from keras.engine.topology import Layer

class CustomRegularization(Layer):
    def __init__(self, lam, **kwargs):
        self.lam = lam
        super(CustomRegularization, self).__init__(**kwargs)

    def call(self, x, mask=None):
        # x[0] is the learned feature map from regul_att_block, with size depending on the block
        # x[1] is the GT attention map (224,224)

        def euclidean_distance(vects):
            x, y = vects
            return K.sqrt(K.sum(K.sum(K.square(x - y),
                                      axis=1, keepdims=True), axis=2,
                                keepdims=False))

        dim1 = x[0].get_shape().as_list()
        att_mat_dim = dim1[1]
        pred_heat_map = x[0]

        # Assume features is of size [N, H, W, C] (batch_size, height, width, channels).
        # Transpose it to [N, C, H, W], then reshape to [N * C, H * W] to compute softmax
        # jointly over the image dimensions.
        pred_heat_map = tf.reshape(tf.transpose(pred_heat_map, [0, 3, 1, 2]), [-1, 56 * 56])
        pred_heat_map = tf.nn.softmax(pred_heat_map)  # apply 2D "spatial" softmax to predicted heatmap
        # Reshape and transpose back to original format.
        pred_heat_map = tf.transpose(tf.reshape(pred_heat_map, [-1, 1, 56, 56]), [0, 2, 3, 1])

        pred_heat_map_resh = tf.reshape(pred_heat_map, [-1, att_mat_dim,
                                                        att_mat_dim])  # to reshape the feature map to the same shape as the GT maps
        gt_heatmap_orig = x[1]

        if att_mat_dim == 56:
            heatmap_56 = tf.image.resize_images(gt_heatmap_orig, (56, 56), align_corners=True)
            gt_heatmap = heatmap_56

        gt_heatmap = tf.reshape(gt_heatmap, tf.shape(gt_heatmap)[:-1])
        pred_heat_map_resh_shape_tensor = tf.shape(pred_heat_map_resh)

        att_regul = euclidean_distance([gt_heatmap,
                                        pred_heat_map_resh])  # to measure the "pixelwise" euclidean distance between the predicted and Gt map

        loss2 = self.lam * K.sum(att_regul)
        self.add_loss(loss2, x)

        return pred_heat_map

    def compute_output_shape(self, input_shape):
        return [input_shape[0], input_shape[1]]

class PredictionHandler:

    # Graph to be used by all threads
    __graph = tf.get_default_graph()

    def __init__(self, model_weights_path, n_categories, n_attributes):
        # One unique session for each thread
        self.__session = tf.Session(graph=PredictionHandler.__graph)

        # model parameters
        att_regul_weight = 10
        conv_kernel_size = 3
        scale_param = 20

        img_input = Input(shape=(224, 224, 3), name='input1')
        att_prior_input = Input(shape=(224, 224, 1), name='input2')

        # Block 1
        x = layers.Conv2D(64, (conv_kernel_size, conv_kernel_size),
                          activation='relu',
                          padding='same',
                          name='block1_conv1')(img_input)
        x = layers.Conv2D(64, (conv_kernel_size, conv_kernel_size),
                          activation='relu',
                          padding='same',
                          name='block1_conv2')(x)
        x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x)

        # Block 2
        x = layers.Conv2D(128, (conv_kernel_size, conv_kernel_size),
                          activation='relu',
                          padding='same',
                          name='block2_conv1')(x)
        x = layers.Conv2D(128, (conv_kernel_size, conv_kernel_size),
                          activation='relu',
                          padding='same',
                          name='block2_conv2')(x)
        out_block2 = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x)

        # Block 3
        x = layers.Conv2D(256, (conv_kernel_size, conv_kernel_size),
                          activation='relu',
                          padding='same',
                          name='block3_conv1')(out_block2)
        x = layers.Conv2D(256, (conv_kernel_size, conv_kernel_size),
                          activation='relu',
                          padding='same',
                          name='block3_conv2')(x)
        x = layers.Conv2D(256, (conv_kernel_size, conv_kernel_size),
                          activation='relu',
                          padding='same',
                          name='block3_conv3')(x)
        out_block3 = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x)

        # Block 4
        x = layers.Conv2D(512, (conv_kernel_size, conv_kernel_size),
                          activation='relu',
                          padding='same',
                          name='block4_conv1')(out_block3)  # changed here
        x = layers.Conv2D(512, (conv_kernel_size, conv_kernel_size),
                          activation='relu',
                          padding='same',
                          name='block4_conv2')(x)
        x = layers.Conv2D(512, (conv_kernel_size, conv_kernel_size),
                          activation='relu',
                          padding='same',
                          name='block4_conv3')(x)
        out_block4 = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(x)

        out_block3_56 = Lambda(lambda image: tf.image.resize_images(image, (56, 56), align_corners=True),
                               name='resize_out_block_3')(out_block3)
        out_block4_56 = Lambda(lambda image: tf.image.resize_images(image, (56, 56), align_corners=True),
                               name='resize_out_block_4')(out_block4)

        x_combined = layers.Concatenate(axis=3)([out_block2, out_block3_56, out_block4_56])
        x_combined = layers.Conv2D(512, (1, 1), activation='relu', padding='same', name='conv_1x1x512')(x_combined)

        num_channels = 512
        # compute mean using tf.nn.avg_pool
        x_combined_t = Lambda(lambda tensor: tf.transpose(tensor, [0, 1, 3, 2]), name='transpose_x')(x_combined)
        avg_pool_spatial_att = Lambda(
            lambda layer: tf.nn.avg_pool(layer, ksize=[1, 1, num_channels, 1], strides=[1, 1, num_channels, 1],
                                         padding='SAME'), name='channel_avg_pool')(x_combined_t)
        max_pool_spatial_att = Lambda(
            lambda layer: tf.nn.max_pool(layer, ksize=[1, 1, num_channels, 1], strides=[1, 1, num_channels, 1],
                                         padding='SAME'), name='channel_max_pool')(x_combined_t)

        # transpose back to original form
        avg_pool_spatial_att = Lambda(lambda tensor: tf.transpose(tensor, [0, 1, 3, 2]), name='transpose_avg_pool')(
            avg_pool_spatial_att)
        max_pool_spatial_att = Lambda(lambda tensor: tf.transpose(tensor, [0, 1, 3, 2]), name='transpose_max_pool')(
            max_pool_spatial_att)

        concat_att_out = layers.Concatenate(axis=3)([avg_pool_spatial_att, max_pool_spatial_att])

        cr_combined = layers.Conv2D(1, (7, 7), activation='sigmoid', padding='same', name='conv_7x7x1')(concat_att_out)

        cr_combined_realim = layers.Multiply()([x_combined, cr_combined])

        cr_combined_realim = layers.AveragePooling2D((2, 2), strides=(4, 4), name='cr_combined_realim_pool')(
            cr_combined_realim)

        cr_combined_out = CustomRegularization(att_regul_weight)([cr_combined, att_prior_input])

        cr_combined_realim_scaled = Lambda(lambda input_tensor: input_tensor * scale_param, name='scaling_att')(
            cr_combined_realim)

        x4_added = keras.layers.Add()([out_block4, cr_combined_realim_scaled])

        # Block 5
        x = layers.Conv2D(512, (3, 3),
                          activation='relu',
                          padding='same',
                          name='block5_conv1')(x4_added)
        x = layers.Conv2D(512, (3, 3),
                          activation='relu',
                          padding='same',
                          name='block5_conv2')(x)
        x = layers.Conv2D(512, (3, 3),
                          activation='relu',
                          padding='same',
                          name='block5_conv3')(x)
        x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(x)

        kernel_init = keras.initializers.RandomNormal(mean=0.0, stddev=0.01, seed=None)
        x_out1 = Conv2D(1024, kernel_size=(3, 3), strides=1, padding='same', activation='relu',
                        kernel_initializer=kernel_init)(x)
        x_out1 = GlobalAveragePooling2D(name='glbal_average_pooling_2D_cat')(x_out1)
        x_out1 = Dropout(0.5)(x_out1)

        out_1 = Dense(n_categories, name='out_1_dense')(x_out1)
        out_1 = Activation('softmax', name='Category_deep_2')(out_1)

        kernel_init = keras.initializers.RandomNormal(mean=0.0, stddev=0.01, seed=None)
        x_out2 = Conv2D(1024, kernel_size=(3, 3), strides=1, padding='same', activation='relu',
                        kernel_initializer=kernel_init)(x)
        x_out2 = GlobalAveragePooling2D(name='glbal_average_pooling_2D_att')(x_out2)
        x_out2 = Dropout(0.5)(x_out2)

        out_3 = Dense(n_attributes, name='out_3_dense')(x_out2)
        out_3 = Activation('sigmoid', name='Attribute_deep')(out_3)

        # Use common graph for all threads
        with PredictionHandler.__graph.as_default():
            # Use each thread's unique session
            with self.__session.as_default():
                self.__learning_framework = Model(inputs=[img_input, att_prior_input], outputs=[out_1, out_3, cr_combined_out])
                self.__learning_framework.load_weights(model_weights_path)

    def predict(self, received_input):
        # Use common graph for all threads
        with PredictionHandler.__graph.as_default():
            # Use each thread's unique session
            with self.__session.as_default():
                return self.__learning_framework.predict_on_batch(received_input)

    def __del__(self):
        self.__session.close()


import numpy as np
import torch
from torchvision.utils import make_grid
from base import BaseTrainer
from utils import inf_loop, MetricTracker
from torch.autograd import Variable
from tqdm import tqdm

class Trainer(BaseTrainer):
    """
    Trainer class
    """
    def __init__(self, model, criterion, metric_ftns, optimizer, config, device,
                 train_loader, val_loader=None, lr_scheduler=None, len_epoch=None):
        super().__init__(model, criterion, metric_ftns, optimizer, config)
        self.config = config
        self.device = device
        self.train_loader = train_loader
        if len_epoch is None:
            # epoch-based training
            self.len_epoch = len(self.train_loader)
        else:
            # iteration-based training
            self.train_loader = inf_loop(train_loader)
            self.len_epoch = len_epoch
        self.val_loader = val_loader
        self.do_validation = self.val_loader is not None
        self.lr_scheduler = lr_scheduler
        self.log_step = 100     #int(np.sqrt(train_loader.batch_size))
        self.log = MetricTracker().result()

        if self.model.__class__.__name__ == 'FirstHalfUNet':
            metrics_list = ['loss'] + list(self.metric_ftns.keys())
        else:
            metrics_list = ['loss', 'loss_mask', 'loss_label'] + list(self.metric_ftns.keys())
        self.train_metrics = MetricTracker(*metrics_list, writer=self.writer)
        self.valid_metrics = MetricTracker(*metrics_list, writer=self.writer)

    def _train_epoch(self, epoch):
        """
        Training logic for an epoch

        :param epoch: Integer, current training epoch.
        :return: A log that contains average loss and metric in this epoch.
        """
        self.model.train()
        self.train_metrics.reset()
        for batch_idx, batch in enumerate(self.train_loader):
            if self.model.__class__.__name__ == 'FirstHalfUNet':
                img = Variable(batch['image']).to(self.device)
                label_tl = Variable(batch['label']).to(self.device)

                label_pred = self.model(img)
                loss = self.criterion(label_pred, label_tl)
            else:
                img = Variable(batch['image']).to(self.device)
                label_tl = Variable(batch['label']).to(self.device)
                mask_tl = Variable(batch['mask']).to(self.device)

                mask_pred, label_pred = self.model(img)
                loss_mask = self.criterion(mask_pred, mask_tl)
                loss_label = self.criterion(label_pred, label_tl)
                loss = loss_mask + loss_label

            self.optimizer.zero_grad()
            loss.backward()
            self.optimizer.step()

            self.writer.set_step((epoch - 1) * self.len_epoch + batch_idx)
            self.train_metrics.update('loss', loss.item())
            if self.model.__class__.__name__ == 'UNet':
                self.train_metrics.update('loss_mask', loss_mask.item())
                self.train_metrics.update('loss_label', loss_label.item())
                self.train_metrics.update('iou', self.metric_ftns['iou'](label_pred, label_tl))
            self.train_metrics.update('accuracy', self.metric_ftns['accuracy'](label_pred, label_tl))

            if batch_idx % self.log_step == 0:
                if self.model.__class__.__name__ == 'FirstHalfUNet':
                    self.logger.debug('Epoch: {}, Step: {}, step_train_loss: {:.4f}'.format(
                        epoch,
                        self._progress(batch_idx),
                        loss.item()
                        ))
                else:
                    self.logger.debug('Epoch: {}, Step: {}, step_train_loss: {:.4f}, step_train_loss_mask: {:.4f}, step_train_loss_label: {:.4f}'.format(
                        epoch,
                        self._progress(batch_idx),
                        loss.item(),
                        loss_mask.item(),
                        loss_label.item()
                        ))
                self.writer.add_image('input', make_grid(img.cpu(), nrow=8, normalize=True))

            if batch_idx == self.len_epoch:
                break
        train_log = self.train_metrics.result()
        self.log.update(**{'train_'+k : v for k, v in train_log.items()})

        if self.do_validation:
            val_log = self._valid_epoch(epoch)
            self.log.update(**{'val_'+k : v for k, v in val_log.items()})

        if self.lr_scheduler is not None:
            self.lr_scheduler.step()
        return self.log

    def _valid_epoch(self, epoch):
        """
        Validate after training an epoch

        :param epoch: Integer, current training epoch.
        :return: A log that contains information about validation
        """
        self.model.eval()
        self.valid_metrics.reset()
        with torch.no_grad():
            for batch_idx, batch in enumerate(tqdm(self.val_loader)):
                if self.model.__class__.__name__ == 'FirstHalfUNet':
                    img = Variable(batch['image']).to(self.device)
                    label_tl = Variable(batch['label']).to(self.device)

                    label_pred = self.model(img)
                    loss = self.criterion(label_pred, label_tl)
                else:
                    img = Variable(batch['image']).to(self.device)
                    label_tl = Variable(batch['label']).to(self.device)
                    mask_tl = Variable(batch['mask']).to(self.device)

                    mask_pred, label_pred = self.model(img)
                    loss_mask = self.criterion(mask_pred, mask_tl)
                    loss_label = self.criterion(label_pred, label_tl)
                    loss = loss_mask + loss_label

                self.writer.set_step((epoch - 1) * len(self.val_loader) + batch_idx, 'valid')
                self.valid_metrics.update('loss', loss.item())
                if self.model.__class__.__name__ == 'UNet':
                    self.valid_metrics.update('loss_mask', loss_mask.item())
                    self.valid_metrics.update('loss_label', loss_label.item())
                    self.valid_metrics.update('iou', self.metric_ftns['iou'](label_pred, label_tl))
                self.valid_metrics.update('accuracy', self.metric_ftns['accuracy'](label_pred, label_tl))

                self.writer.add_image('input', make_grid(img.cpu(), nrow=8, normalize=True))

        # add histogram of model parameters to the tensorboard
        for name, p in self.model.named_parameters():
            self.writer.add_histogram(name, p, bins='auto')

        return self.valid_metrics.result()

    def _progress(self, batch_idx):
        base = '[{}/{} ({:.0f}%)]'
        if hasattr(self.train_loader, 'n_samples'):
            current = batch_idx * self.train_loader.batch_size
            total = self.train_loader.n_samples
        else:
            current = batch_idx
            total = self.len_epoch
        return base.format(current, total, 100.0 * current / total)


import argparse
import csv
from os.path import join

HEADER = ['Dataset', 'psnr_score', 'mse_score', 'nrmse_score', 'ssim_score', 'mae_score']

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-d', '--root-dir', required=True)
    parser.add_argument('-e', '--epoch', required=True)
    args = parser.parse_args()

    def aggregate(csv_file):
        aggregated = []
        for k in range(5):
            enface_dir = join(args.root_dir, str(k), 'enfaces/epoch_{}'.format(args.epoch))
            with open(join(enface_dir, csv_file), 'r') as file:
                reader = csv.reader(file)
                next(reader) # skip header row
                for row in reader:
                    # skip 'Average' row
                    if row[0] == 'Average':
                        continue
                    # convert strings into floats
                    new_row = []
                    new_row.append(row[0])
                    for score in row[1:]:
                        new_row.append(float(score))
                    aggregated.append(new_row)

        # calculate averages
        averages = ['Average', 0.0, 0.0, 0.0, 0.0, 0.0]
        for a in aggregated:
            for idx in range(1, 6):
                averages[idx] += a[idx]
        for idx in range(1, 6):
            averages[idx] /= len(aggregated)

        aggregated_file = join(args.root_dir, csv_file)
        print('Saving to {}'.format(aggregated_file))
        with open(aggregated_file, 'w') as file:
            writer = csv.writer(file)
            writer.writerow(HEADER)
            for a in aggregated:
                writer.writerow(a)
            writer.writerow(averages)

    aggregate('comparison_multi_slice_max_norm.csv')
    aggregate('comparison_multi_slice_sum.csv')

if __name__ == '__main__':
    main()


import pyxel
import PyxelUniversalFont as puf
from .utils import *

from pynput import keyboard
import pykakasi
import random

class Params:
    SOURCE = get_data_path("data/img.pyxres")
    FONT_SIZE = 32
    WINDOW_SIZE = (FONT_SIZE*25,FONT_SIZE*6)
    REPLACE_TOKEN = {
        "<name>": input("彼女の名前：") or "きみ"
    }
    DICT = load_vocabulary_from_file(get_data_path("data/vocab.txt"), REPLACE_TOKEN)

class App:
    def __init__(self, params:Params) -> None:
        # initialize window
        self.WINDOW_WIDTH, self.WINDOW_HEIGHT = params.WINDOW_SIZE
        pyxel.init(self.WINDOW_WIDTH, self.WINDOW_HEIGHT)

        # load data
        pyxel.load(params.SOURCE)

        # initialize variables
        self._reset(params)

        # run app
        pyxel.run(self._update, self._draw)

    # initialize variables
    def _reset(self, params:Params):
        self.params = params
        # self.writer = puf.Writer("misaki_gothic2.ttf")
        self.writer = puf.Writer("IPA_Gothic.ttf")
        self.text = " "
        self.position = 0

        self.current_key = None
        self.shift_pressed = False
        self.listener = keyboard.Listener(on_press=self._on_press, on_release=self._on_release)
        self.listener.start()

        self.kks = pykakasi.kakasi()
        self._get_new_target()

    def _on_press(self, key):
        if key == keyboard.Key.shift:
            self.shift_pressed = True
        elif hasattr(key, 'char') and key.char:
            self.current_key = key.char.upper() if self.shift_pressed else key.char

    def _on_release(self, key):
        if key == keyboard.Key.shift:
            self.shift_pressed = False

    def _add_text(self, character):
        if self.answer_text[self.position] == character:
            self.text = self.text[:-1] + character
            self.text += " "
            self.position += 1

    def _get_roma(self, text):
        result = ""
        text_converted = self.kks.convert(text)
        for word in text_converted:
            result += word['passport']
        return result

    def _get_new_target(self):
        message, answer = random.choice(self.params.DICT)
        self.message = message
        self.target_text = answer
        self.answer_text = self._get_roma(answer)
        self.text = " "
        self.position = 0

    def _correct(self):
        return self.answer_text == self.text[:-1]

    # process
    def _update(self):

        if pyxel.btnp(pyxel.KEY_SPACE):
            self._add_text(" ")
        if self.current_key:
            self._add_text(self.current_key)
            self.current_key = None

        if self.position == len(self.answer_text):
            self._get_new_target()

    # visualize
    def _draw(self):
        pyxel.cls(0)

        font_size = self.params.FONT_SIZE
        line_height = font_size + 2

        self.writer.draw(0, line_height*1, self.message, font_size, 7)
        self.writer.draw(0, line_height*3, self.target_text, font_size, 7)
        self.writer.draw(0, line_height*4, self.answer_text, font_size, 7)
        self.writer.draw(0, line_height*4, self.text, font_size, 5)


import requests
from bs4 import BeautifulSoup
import json
import os.path

HOST = 'https://www.avito.ru/'
HEADERS = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)'
                         ' Chrome/87.0.4280.88 Safari/537.36 Edg/87.0.664.60', 'accept': '*/*'}

def get_content(url):
    html = requests.get(url, headers=HEADERS)
    if html.status_code == 200:
        soup = BeautifulSoup(html.text, 'html.parser')
        items = soup.find_all('div', class_='iva-item-list-2_PpT')
        data = read_db()  # Читаем базу
        products = []
        for item in items:
            image = item.find('ul', class_='photo-slider-list-3Zt1Z').find('li')
            if image:
                image = image.get('data-marker').replace('slider-image/image-', '')
            else:
                image = 'https://static.avito.ru/@avito/bx-single-page-catalog/2.395.2/' \
                        'prod/web/resources/93ac49675b15.svg'
            item_id = item.get('data-item-id')
            if item_id in data["products"]:
                continue
            products.append({
                'id': item_id,
                'title': item.find('span', class_='title-root-395AQ').get_text(),
                'image': image,
                'price': item.find('span', class_='price-text-1HrJ_').get_text().replace('\xa0', ' '),
                'geo': item.find('div', class_='geo-root-1pUZ8').get_text(),
                'link': HOST + item.find('a', class_='link-link-39EVK').get('href')
            })
            if len(products) == 5:
                break
        write_new_item_to_db(products)
        return products
    else:
        print('Error')

def write_new_item_to_db(new_item):
    new_id = [item["id"] for item in new_item]
    data = read_db()
    with open("db.json", "w") as jsonFile:
        data["products"] += new_id
        json.dump(data, jsonFile, indent=4)
        jsonFile.close()

def read_db():
    if os.path.exists('db.json'):
        with open("db.json", "r") as jsonFile:
            data = json.load(jsonFile)
            jsonFile.close()
    else:
        data = {
            "ID": "",
            "URL": "",
            "interval": 300,
            "products": []
        }
        write_db(data)
    return data

def write_db(data):
    with open("db.json", "w") as jsonFile:
        json.dump(data, jsonFile, indent=4)
        jsonFile.close()

# print(get_content('https://www.avito.ru/rostov-na-donu/audio_i_video?cd=1&s=104'))


from flask import redirect, url_for, session, flash, render_template
from flask_login import current_user

from photoric.modules.albums.forms import CreateAlbumForm
from photoric.modules.auth.auth import authorize
from photoric import db
from photoric.core.models import Album, Image
from photoric.modules.views.helper import get_gallery_items
from photoric.modules.albums import albums_bp

# create album form context processor
@albums_bp.app_context_processor
def create_album_form():
    return dict(create_album_form=CreateAlbumForm())

# db context processors
@albums_bp.app_context_processor
def albums_processors():
    # get list of all albums
    def list_albums():
        return Album.query.filter(Album.authorized('read')).all()

    # get number of images in album
    def get_elements_number(album_id):
        album_content = {}
        album_content['albums'] = get_gallery_items(album_id, 'albums').count() # Album.query.filter(Album.parent_id == album_id).count()
        album_content['images'] = get_gallery_items(album_id, 'images').count() # Image.query.filter(Image.parent_id == album_id).count()
        return album_content

    return dict(list_albums=list_albums,
                get_elements_number=get_elements_number
                )

# route for album creation
@authorize.create(Album)
@albums_bp.route("/create", methods = ['GET', 'POST'])
def create_album():
    form = CreateAlbumForm()
    parent = session.get("current_album")
    if form.validate_on_submit():

        # get current user data
        user_id = current_user.id

        for group in current_user.groups:
            if group is not None:
                group_id = group.id
                break

        # generate new album object
        new_album = Album(
                        name=form.name.data,
                        description=form.description.data,
                        keywords=form.keywords.data,
                        parent_id=parent,
                        owner_id=user_id,
                        group_id=group_id
                    )
        db.session.add(new_album)
        db.session.flush()

        # write new album id in session
        session["current_album"] = new_album.id
        session["album_name"] = new_album.name

        # write new album to database
        db.session.commit()

        # redirect user to album view
        flash (u"Album was successfully created!", "success")
        return redirect(url_for("albums.show_album", album_name=session.get("album_name")))

    # redirect to home page in case of GET method
    flash(u"Create album form was not valid", "danger")
    return render_template("views/index.html", title='Home page')

# route for show album
@albums_bp.route("/<album_name>")
@authorize.read
def show_album(album_name=None):
    if album_name:
        album = Album.query.filter(Album.name == album_name, Album.authorized('read')).first()
        children_albums = get_gallery_items(album.id, 'albums')
        children_images = get_gallery_items(album.id, 'images')
        if not children_albums:
            children_albums = None
        if not children_images:
            children_images = None
        # write id of displayed album
        session["current_album"] = album.id
        session["album_name"] = album.name
        return render_template("views/index.html",
                               title='Album: ' + album_name,
                               albums=children_albums,
                               images=children_images
                               )
    return render_template("views/index.html", title="Home page")

# redirect to show album to serve dropzone redirect
@albums_bp.route("/redirect_to_album")
@authorize.read
def redirect_to_album():
    album_name = session.get("album_name")
    if album_name:
        return redirect(url_for('albums.show_album', album_name=album_name))
    return render_template("views/index.html", title="Home page")


from aiogram.types import InlineKeyboardButton, InlineKeyboardMarkup, Message
from aiogram.utils.keyboard import InlineKeyboardBuilder

from tgbot.lexicon.lexicon_ru import LEXICON_RU

def add_admin_button(keyboard: InlineKeyboardMarkup,
                     width: int,
                     *args: str) -> InlineKeyboardMarkup:
    # Инициализируем билдер
    kb = keyboard

    # Инициализируем список для кнопок
    buttons: list[InlineKeyboardButton] = []

    # Заполняем список кнопками из аргументов args
    for button in args:
        buttons.append(InlineKeyboardButton(
                text=LEXICON_RU[button] if button in LEXICON_RU else button,
                callback_data=button))

    # Распаковываем список с кнопками в билдер методом row c параметром width
    kb().add(*buttons, width=width)

    # Возвращаем объект инлайн-клавиатуры
    return kb


from setuptools import setup, find_packages

with open("README.md", "r") as f:
    page_description = f.read()

with open("requirements.txt") as f:
    requirements = f.read().splitlines()

setup(
    name="image_processing_dio",
    version="0.0.1",
    author="Marley Adriano",
    author_email="marleyasi@gmail.com",
    description="Project resulting from DIO 'Descomplicando a criação de pacotes de processamento de imagens em python' Lab.",
    long_description=page_description,
    long_description_content_type="text/markdown",
    url="https://github.com/marleyas/image-processing_dio.git",
    packages=find_packages(),
    install_requires=requirements,
    python_requires='>=3.6',
)


"""
Reference: B. Peng, X. Jin, J. Liu, S. Zhou, Y. Wu, Y. Liu, D. Li and Z. Zhang, “Correlation Congruence for Knowledge Distillation”, 2019. Available at: https://arxiv.org/pdf/1904.01802.pdf.
"""

import tensorflow as tf
import numpy as np
import sklearn.metrics.pairwise

class CCKD():
	def __init__(
		self,
		teacher: tf.keras.Model,
		student: tf.keras.Model,
		alreadySoftmax: bool = True,
		optimizer: tf.keras.optimizers = tf.keras.optimizers.Adam(),
		studentLoss: tf.keras.losses = tf.keras.losses.CategoricalCrossentropy(),
		distilLoss: tf.keras.losses = tf.keras.losses.KLDivergence(),
		kernel: sklearn.metrics.pairwise = sklearn.metrics.pairwise.rbf_kernel,
		metrics = [tf.keras.metrics.CategoricalAccuracy()]

	):
		"""
		Description
		---------------
		Initialize the teacher model, the student model and their last layer index.

		Input(s)
		---------------
		teacher: A trained Keras Sequential or Functional model (Sub-class models are not supported).
		student: An untrained Keras Sequential or Functional model (Sub-class models are not supported).
		alreadySoftmax : If the last layer is softmax it must be true, else it must be false (for teacher and student). By default true.
		optimizer: Optimizer instance. By default Adam.
		studentLoss: Loss instance. By default CategoricalCrossentropy.
		distilLoss: Loss instance. By default KLDivergence.
		kernel: Kernel instance used to compute correlations. By default Radial Basis Function kernel.
		metrics: List of metrics to be evaluated by the model during training and testing. By default CategoricalAccuracy.
		"""

		self.teacher = teacher
		self.student = student
		self.alreadySoftmax = alreadySoftmax
		self.optimizer = optimizer
		self.studentLoss = studentLoss
		self.distilLoss = distilLoss
		self.kernel = kernel
		self.metrics = metrics

	def __taylor_approximation(
		self,
		corr,
		order = 1
	):
		"""
		Description
		---------------
		Approximate a correlation matrix using Taylor's series approximation.

		Input(s)
		---------------
		corr: the correlation matrix to approximate.
		order: the order of the Taylor's series to approximate with.

		Output(s)
		---------------
		y: 1D numpy array containing the row-wise approximate values.
		"""
		y = np.zeros(len(corr))
		for n, row in zip(range(order), corr):
			y = y + ((-1)**n * (row)**(2*n+1)) / np.math.factorial(2*n+1)
		return y

	def __compute_corr_congruence(
		self,
		teacherPreds,
		studentPreds,
		kernel,
		order = 2
	):
		"""
		Description
		---------------
		Compute the correlation congruence between the teacher's and student's predictions.

		Input(s)
		---------------
		teacherPreds: Teacher's predictions.
		studentPreds: Student's predictions.
		kernel: Kernel used to compute the correlation matrices.
		oder: Order of the Taylor's series to approximate correlation matrices with

		Output(s)
		---------------
		congruence: Correlation congruence, float value.
		"""
		corr_student = kernel(studentPreds)
		corr_teacher = kernel(teacherPreds)
		taylored_student = self.__taylor_approximation(corr_student, order=order)
		taylored_teacher = self.__taylor_approximation(corr_teacher, order=order)
		congruence = sum(abs(taylored_teacher - taylored_student))
		return congruence

	def __metrics_after_batch(
		self,
		metricsHandlerTruth,
		metricsHandlerTeacher,
		y_truth,
		teacher_preds,
		student_preds,
		step,
		training
	):
		"""
		Description
		---------------
		Compute metrics after a mini-batch.

		Input(s)
		---------------
		metricsHandlerTruth: List with metrics instance for the ground truth labels
		metricsHandlerTeacher: List with metrics instance for the teacher labels.
		y_truth: Ground truth categories.
		teacher_preds: Teacher's predictions.
		student_preds: Student's predictions
		step: Current minbatch index.
		training: Boolean, if true training mode, if false validation mode.

		Output(s)
		---------------
		metricsTuples: List of tuples with metrics inside.
		"""
		metricsTuples = []
		# Comparing to ground truth values
		for handler in metricsHandlerTruth:
			if step == 0:
				handler.reset_state()
			handler.update_state(y_truth.numpy(), student_preds.numpy())
			if training == True:
				metricsTuples.append((handler.name, handler.result().numpy()))
			else:
				metricsTuples.append(("Val_" + handler.name, handler.result().numpy()))
		# Comparing to teacher
		for handler in metricsHandlerTeacher:
			if step == 0:
				handler.reset_state()
			handler.update_state(teacher_preds.numpy(), student_preds.numpy())
			if training == True:
				metricsTuples.append((handler.name, handler.result().numpy()))
			else:
				metricsTuples.append(("Val_" + handler.name, handler.result().numpy()))
		return metricsTuples

	def __compute_loss(
		self,
		x,
		y,
		training,
		alpha,
		beta,
		kernel,
		order,
		temperature

	):
		"""
		Description
		---------------
		Compute predictions and losses.

		Input(s)
		---------------
		x: Image mini-batch.
		y: Categories mini-batch.
		training: Boolean, if true prediction mode of student is Training=True, else Training=False.
		alpha: Loss balancing factor, dictates regular/distillation loss ratio.
		beta: Loss weigthing factor, dictates correlation congruence importance.
		kernel: Kernel used to compute the correlation matrices.
		order: Order of the Taylor's series to approximate correlation matrices with.
		temperature: Temperature for softening probability distributions. Larger temperature gives softer distributions.

		Output(s)
		---------------
		teacherPreds: Predictions of the teacher.
		studentPreds: Predictions of the student.
		loss: Value of the loss.
		studentLoss: Value of the studentLoss.
		distilLoss: Value of the distilLoss.
		congruence: Value of the correlation congruence.
		"""
		# Make predictions
		teacherPreds = self.teacher(x, training=False)
		studentPreds = self.student(x, training=training)
		if self.alreadySoftmax == False:
			teacherPreds = tf.keras.activations.softmax(teacherPreds)
			studentPreds = tf.keras.activations.softmax(studentPreds)
		distilLoss = self.distilLoss(
			teacherPreds / temperature,
			studentPreds / temperature
		)
		studentLoss = self.studentLoss(
			y,
			studentPreds
		)
		congruence = self.__compute_corr_congruence(
			teacherPreds,
			studentPreds,
			kernel = kernel,
			order = order
			)
		# Computing loss
		loss = (alpha * studentLoss) + ((1 - alpha) * distilLoss) + (beta * congruence)
		return teacherPreds, studentPreds, loss, studentLoss, distilLoss, congruence

	def __packing_history(
		self,
		loss,
		studentLoss,
		distilLoss,
		congruence,
		metricsHandlerTruth,
		metricsHandlerTeacher,
		training,
		history
	):
		"""
		Description
		---------------
		Add losses and metrics values to the history.

		Input(s)
		---------------
		loss: Value of the loss.
		studentLoss: Value of the studentLoss.
		distilLoss: Value of the distilLoss.
		congruence: Value of the correlation congruence.
		metricsHandlerTruth: List with metrics instance for the ground truth labels
		metricsHandlerTeacher: List with metrics instance for the teacher labels.
		training: Boolean, if true training mode, if false validation mode.
		history: Dictionnary to fill.
		"""
		# Adding losses to history
		if training == True:
			losses = ['Loss', 'DistilLoss', 'StudentLoss', 'CorrCongruence']
		else:
			losses = ['Val_loss', 'Val_distilLoss', 'Val_studentLoss', 'Val_corrCongruence']
		for i in losses:
			if i not in history:
				history[i] = []
		history[losses[0]].append(loss.numpy())
		history[losses[1]].append(distilLoss.numpy())
		history[losses[2]].append(studentLoss.numpy())
		history[losses[3]].append(congruence)
		# Adding metrics to history
		# Comparing to ground truth
		for handler in metricsHandlerTruth:
			if training == True:
				if handler.name not in history:
					history[handler.name] = []
				history[handler.name].append(handler.result().numpy())
			else:
				if "Val_" + handler.name not in history:
					history["Val_" + handler.name] = []
				history["Val_" + handler.name].append(handler.result().numpy())
		# Comparing to teacher
		for handler in metricsHandlerTeacher:
			if training == True:
				if "Distillation_" + handler.name not in history:
					history["Distillation_" + handler.name] = []
				history["Distillation_" + handler.name].append(handler.result().numpy())
			else:
				if "Val_distillation_" + handler.name not in history:
					history["Val_distillation_" + handler.name] = []
				history["Val_distillation_" + handler.name].append(handler.result().numpy())

	def distil(
		self,
		trainData: tf.data.Dataset,
		valData: tf.data.Dataset,
		epochs: int = 1,
		trainBatchSize: int = None,
		valBatchSize: int = None,
		alpha: float = 0.1,
		beta: float = 0.5,
		order: int = 2,
		temperature: int = 3,
		history: bool = False
	):
		"""
		Description
		---------------
		Distil the knowledge of the teacher to the student.

		Input(s)
		---------------
		trainData: TensorFlow Dataset with training images.
		valData: TensorFlow Dataset with validation images.
		epochs: Number of epochs to distil the model. By default 1.
		trainBatchSize: Number of samples per gradient update. If None, we assume that the user provides a batched TensorFlow Dataset. By default None.
		valBatchSize: Number of samples per validation batch. If None, we assume that the user provides a batched TensorFlow Dataset. By default None.
		alpha: Loss balancing factor, dictates regular/distillation loss ratio. By default 0.1.
		beta: Loss weigthing factor, dictates correlation congruence importance. By default 0.5
		order: Order of the Taylor's series to approximate correlation matrices with. By default 2.
		temperature: Temperature for softening probability distributions. Larger temperature gives softer distributions. By default 3.
		history: Boolean. If True, returns the losses and metrics history. If False, does not return history. By default False.

		Output(s)
		---------------
		distilled_model: Distilled Keras Sequential or Functional student model.
		lossesAndMetrics : If "history" equals True, it returns the losses and metrics history.
		"""
		# Compiling student model
		self.student.compile(
			optimizer=self.optimizer,
			loss=self.studentLoss,
			metrics=self.metrics
		)
		# Prepare the training dataset
		if trainBatchSize != None:
			trainData = trainData.shuffle(1024).batch(batch_size=trainBatchSize)
		batchNbTrain = trainData.cardinality().numpy()
		# Prepare the validation dataset
		if valBatchSize != None:
			valData = valData.batch(batch_size=valBatchSize)
		batchNbVal = valData.cardinality().numpy()
		# Getting metrics
		lossesAndMetrics = {}
		metricsHandlerTruth = []
		metricsHandlerTeacher = []
		for metric in self.metrics:
			metricsHandlerTruth.append(tf.keras.metrics.get(metric))
			metricsHandlerTeacher.append(tf.keras.metrics.get(metric))
		# Training
		for epoch in range(epochs):
			print("Distillation Epoch {}/{}".format(epoch+1, epochs))
			pb_train = tf.keras.utils.Progbar(batchNbTrain)
			for step, (x_batch_train, y_batch_train) in enumerate(trainData):
				with tf.GradientTape() as tape:
					teacherPredsTrain, studentPredsTrain, lossTrain, studentLossTrain, distilLossTrain, corrCongruenceTrain = self.__compute_loss(
						x_batch_train,
						y_batch_train,
						True,
						alpha,
						beta,
						self.kernel,
						order,
						temperature
					)
				# Computing metrics
				metricsTuplesTrain = self.__metrics_after_batch(
					metricsHandlerTruth,
					metricsHandlerTeacher,
					y_batch_train,
					teacherPredsTrain,
					studentPredsTrain,
					step,
					True
				)
				# Updating progress bar losses and metrics
				lossesTuplesTrain = [
					('Loss', lossTrain),
					('DistilLoss', distilLossTrain),
					('StudentLoss', studentLossTrain),
					('CorrCongruence',corrCongruenceTrain)
				]
				globalTuplesTrain = lossesTuplesTrain + metricsTuplesTrain
				pb_train.add(1, values=globalTuplesTrain)
				# Computing gradient
				gradients = tape.gradient(lossTrain, self.student.trainable_variables)
				# Update weights
				self.student.optimizer.apply_gradients(zip(gradients, self.student.trainable_variables))
			# Adding training losses and metrics values to the model
			self.__packing_history(
				lossTrain,
				studentLossTrain,
				distilLossTrain,
				corrCongruenceTrain,
				metricsHandlerTruth,
				metricsHandlerTeacher,
				True,
				lossesAndMetrics
			)
			# Validation
			pb_val = tf.keras.utils.Progbar(batchNbVal)
			for step, (x_batch_val, y_batch_val) in enumerate(valData):
				teacherPredsVal = self.teacher(x_batch_val, training=False)
				studentPredsVal = self.student(x_batch_val, training=False)
				# Computing losses and metrics
				# Computing losses
				teacherPredsVal, studentPredsVal, lossVal, studentLossVal, distilLossVal, corrCongruenceVal = self.__compute_loss(
						x_batch_val,
						y_batch_val,
						True,
						alpha,
						beta,
						self.kernel,
						order,
						temperature
					)
				# Updating validation losses and metrics
				lossesTuplesVal = [
					('Val_loss', lossVal),
					('Val_distilLoss', distilLossVal),
					('Val_studentLoss', studentLossVal),
					('Val_corrCongruence', corrCongruenceVal)
				]
				# Computing metrics
				metricsTuplesVal = self.__metrics_after_batch(
					metricsHandlerTruth,
					metricsHandlerTeacher,
					y_batch_val,
					teacherPredsVal,
					studentPredsVal,
					step,
					False
				)
				globalTuplesVal = lossesTuplesVal + metricsTuplesVal
				pb_val.add(1, values=globalTuplesVal)
			# Adding validation losses and metrics values to the model
			self.__packing_history(
				lossVal,
				studentLossVal,
				distilLossVal,
				corrCongruenceVal,
				metricsHandlerTruth,
				metricsHandlerTeacher,
				False,
				lossesAndMetrics
			)
		# Returning distilled student
		if history == True:
			return self.student, lossesAndMetrics
		else:
			return self.student


import logging
import newrelic.agenttest.testenv
import os
import tarfile
import tempfile
import urllib.request
import subprocess

def run(test_env):
    for app_class in sorted(test_env.config.keys()):
        for app in [app for app in test_env.config[app_class]
                    if app.get('archive')]:
            logging.info('checking {}'.format(app['path']))
            if not (os.path.exists(app['path']) and app_is_up_to_date(app)):
                logging.info("downloading {} for {}".format(_remote_app_path(app), app['path']))
                archive = download_app(app)
                if archive == None:
                    logging.info("ignoring {} because {} does not exist on server".format(app['path'], _remote_app_path(app)))
                else :
                    expand_app_archive(app, archive)

def _local_md5sum_path(app):
    return os.path.join(os.path.dirname(app['path']), app['archive'] + '.md5')

def _remote_app_path(app):
    return "http://{}/{}".format(app['archive_repo'], app['archive'])

def _remote_app_md5sum_path(app):
    return _remote_app_path(app) + '.md5'

def app_is_up_to_date(app):
    remote_checksum = None;
    try:
        remote_checksum = urllib.request.urlopen(
            _remote_app_md5sum_path(app)).read().decode().strip()
    except urllib.error.HTTPError as e:
        if e.code != 404:
            raise e
        return False

    try:
        local_checksum = ''
        with open(_local_md5sum_path(app), 'r') as f:
            local_checksum = f.read().strip()

        if local_checksum == remote_checksum:
            logging.debug('{} checksums match'.format(app['path']))
            return True
        else:
            logging.info(
                '{} checksums do not match: {} vs {}'.format(
                    app['path'],
                    local_checksum,
                    remote_checksum))
            return False
    except Exception as e:
        logging.debug(e)
        return False

def download_app(app):
    try:
        return urllib.request.urlopen(_remote_app_path(app))
    except urllib.error.HTTPError as e:
        if e.code != 404:
            raise e
        return None

def expand_app_archive(app, archive):
    tf = tempfile.NamedTemporaryFile()
    logging.info("saving {} to {}".format(_remote_app_path(app), tf.name))
    tf.write(archive.read())
    archive = tarfile.open(fileobj=tf,debug=0)

    pardir = os.path.dirname(app['path'])

    if not os.path.exists(pardir):
        os.mkdir(pardir)

    logging.info("extracting {} to {}".format(tf.name, pardir))

    # TODO: Determine why this silently fails for all archives
    #archive.extractall(path=pardir)

    subprocess.call(["tar", "xzf", tf.name, "-C", pardir]);

    tf.close()

    logging.info("getting md5 checksum from {}".format(_remote_app_md5sum_path(app)))

    checksum = urllib.request.urlopen(
        _remote_app_md5sum_path(app)).read()

    logging.info("writing md5 file {}".format(_local_md5sum_path(app)))

    with open(_local_md5sum_path(app), 'w+') as f:
        f.write(checksum.decode())

if __name__ == '__main__':
    logging.getLogger().setLevel(os.environ.get('TEST_LOG_LEVEL', 'INFO'))
    logging.basicConfig(format='%(asctime)s - %(levelname)s: %(message)s')
    test_conf = os.environ.get('TEST_CONFIG', '/Users/roger/Documents/workspace_git/agent_integration_tests/conf/java_config.yml')
    test_local_conf = os.environ.get('TEST_LOCAL_CONFIG', '/Users/roger/Documents/workspace_git/agent_integration_tests/conf/java_local_config.yml')

    run(
        newrelic.agenttest.testenv.TestEnv(
            test_conf,
            test_local_conf,
            None))
#    run(
#        newrelic.agenttest.testenv.TestEnv(
#            os.environ['TEST_CONFIG'],
#            os.environ['TEST_LOCAL_CONFIG']))


# -*- coding: utf-8 -*-
"""
集合オブジェクトFiniteSetを使う例。

単振り子に対する周期を求める例で、
通常のリストやタプルの代わりに、FiniteSetを使ってみる。
"""

from sympy import FiniteSet

from sympy import pi

def time_period(length, g=9.8):
    """
    単振り子の周期を求める。

    公式は以下。
    T = 2 * \pi * \sqrt{L / g}
    T :振り子の周期(一往復するまでにかかる時間)。
    L :振り子の長さ(cm)。
    g :重力加速度。地球上での平均は、9.8m/s^2
    """
    return 2*pi * ((length/g)**0.5)

l_list = FiniteSet(15, 18, 21, 22.5, 25)
g_list = FiniteSet(9.8, 9.78, 9.83)

# ヘッダを出力。
print('{0:^15}{1:^15}{2:^15}'.format('Length(cm)', 'Gravity(m/^2)', 'Time Periods(s)'))

# 直積を求める。
# (この例で一番重要なのはここ。長さの異なる集合があって、
# 一つ一つの組み合わせに対し何かを適用したい時、FiniteSetが便利。)
product = l_list * g_list

for elem in product:
    l = elem[0]
    g = elem[1]

    # 振り子の周期を求める。
    t = time_period(l/100, g)

    print('{0:^15}{1:^15}{2:^15.3f}'.format(float(l), float(g), float(t)))


import influxdb_client_3 as InfluxDBClient3
import pandas as pd
import numpy as np
from influxdb_client_3 import write_options

client = InfluxDBClient3.InfluxDBClient3(token="",
                         host="",
                         org="",
                         database="flight2", enable_gzip=True, write_options=write_options(batch_size=500,
                                                      flush_interval=10_000,
                                                      jitter_interval=2_000,
                                                      retry_interval=5_000,
                                                      max_retries=5,
                                                      max_retry_delay=30_000,
                                                      max_close_wait=300_000,
                                                      exponential_base=2, write_type='batching'))

# Create a dataframe
df = pd.DataFrame({'a': [1, 2, 3], 'b': [4, 5, 6]})

# Create a range of datetime values
dates = pd.date_range(start='2023-05-01', end='2023-05-29', freq='5min')

# Create a DataFrame with random data and datetime index
df = pd.DataFrame(np.random.randn(len(dates), 3), index=dates, columns=['Column 1', 'Column 2', 'Column 3'])
df['tagkey'] = 'Hello World'

print(df)

# Write the DataFrame to InfluxDB
client.write(df, data_frame_measurement_name='table3', data_frame_tag_columns=['tagkey'])

# Query with influxQL
table = client.query(query="SELECT * FROM table3 WHERE time > now() - 30d", language="influxql")
print(table.to_pandas())


#!/usr/bin/python

import tkinter as tk

def_font=[ "DejaVuSansMono",100, "normal" ]
def_font2=[ "DejaVuSansMono", 50, "normal" ]

class ExampleApp(tk.Tk):
    def __init__(self):
        tk.Tk.__init__(self)
        self.label = tk.Label(self, text="", width=10, font=def_font)
        tk.Label(self, text="Hit ESC to stop", font=def_font2).pack()

        self.label.pack()
        self.remaining = 0
        self.countdown(10)

    def countdown(self, remaining = None):
        if remaining is not None:
            self.remaining = remaining

        if self.remaining <= 0:
            self.label.configure(text="time's up!")
        else:
            self.label.configure(text="%d" % self.remaining)
            self.remaining = self.remaining - 1
            self.after(1000, self.countdown)

if __name__ == "__main__":
    app = ExampleApp()
    app.bind('<Key-Escape>', lambda event: quit())
    app.mainloop()


books = []

def add_book(title,author):
    books.append({"title":title,"author":author,"read":False})

def read_book(title):
    for i in books:
        if i["title"] == title:
            i["read"] = True
    return None

def remove_book(title):
    for i in books:
        if i["title"] == title:
            books.remove(i)
            return title,"removed succesfully"
    return None

def list_all():
    for i in books:
        print("Book title is",i["title"],"written by",i["author"],". User read book ?:",i["read"])
    return None
# def find_book(title):
#     for i in range(len(books)):
#         name = books[i]["title"]
#         return title,"found at index",i
#     return None

class NoneError(NameError):
    pass