from __future__ import print_function, division, absolute_import

import re
import os
import sys
import wave
import time
import shutil
import warnings
from numbers import Number
from multiprocessing import Pool, cpu_count, Process, Queue
from six import add_metaclass, string_types
from six.moves import zip, zip_longest, range, cPickle
from abc import ABCMeta, abstractmethod, abstractproperty

from collections import defaultdict
import numpy as np

from odin.ml import MiniBatchPCA
from odin.utils.mpi import MPI
from odin.preprocessing import speech, video, image, signal
from odin.utils import (Progbar, as_tuple, get_all_files, ctext,
                        get_tempdir, is_string, batching,
                        add_notification, keydefaultdict)
from .dataset import Dataset
from .recipes import FeederRecipe
from .utils import MmapDict, SQLiteDict

_default_module = re.compile('__.*__')

__all__ = [
    'WaveProcessor',
    'SpeechProcessor',
]

# ===========================================================================
# Helper
# ===========================================================================
# ==================== For speech ==================== #
def _append_energy_and_deltas(s, energy, delta_order):
    # s.shape = [Time, Dimension]
    if s is None:
        return None
    if energy is not None:
        s = np.hstack((s, energy[:, None]))
    # compute delta
    if delta_order > 0:
        deltas = speech.compute_delta(s.T, order=delta_order)
        # tranpose back to [Time, Dim]
        s = np.hstack([s] + [i.T for i in deltas])
    return s

# ==================== general ==================== #
@add_metaclass(ABCMeta)
class FeatureProcessor(object):

    """ FeatureProcessor
    Following attribtues must be set for overriding this class:
    jobs: list
        list of all jobs for processing
    njobs: int
        number of jobs, if njobs is 0, then njobs = len(jobs)

    By overriding this class, these three properties must be carefully
    defined:
    _features_properties: tuple, list
        list of (name-str, dtype-dtype, save_statistics-bool),
        this list determines which features will be processed and saved.
    _external_indices: tuple, list
        Return a list of feature names that have separated indices
        file.
    _excluded_pca: list
        All feature properties with name given in this list will
        be excluded during pca calculation.
    """

    def __init__(self, output_path, datatype='memmap',
                 pca=True, pca_whiten=False,
                 save_stats=True, substitute_nan=None,
                 ncache=0.12, ncpu=1):
        super(FeatureProcessor, self).__init__()
        if datatype not in ('memmap', 'hdf5'):
            raise ValueError('datatype must be "memmap", or "hdf5"')
        self.datatype = datatype
        self.output_path = output_path
        # PCA
        self.pca = bool(pca)
        self.pca_whiten = bool(pca_whiten)
        # STATS
        self.save_stats = bool(save_stats)
        self.substitute_nan = substitute_nan
        self.ncpu = ncpu
        self.ncache = ncache
        # defaults
        self.jobs = []
        self.njobs = 0
        # ====== internal control for feature processor ====== #
        # list of (name, dtype, static-able)
        self._features_properties = []
        # list of features name
        self._external_indices = []
        self._excluded_pca = []

    # ==================== Abstract properties ==================== #
    @property
    def features_properties(self):
        """ Return list of features' properties
        [(name, dtype, statistic-able), ...]
        Note
        ----
        A statistic-able feature will also be calculated and stored
        the sum1, sum2, and PCA
        """
        if len(self._features_properties) == 0:
            raise ValueError("`_features_properties` must be defined and length>0 "
                             "when overriding FeatureProcessor.")
        for name, dtype, static_able in self._features_properties:
            if not is_string(name) or \
            not (is_string(dtype) or isinstance(dtype, np.dtype)) or \
            not (isinstance(bool(static_able), type(True))):
                raise ValueError("features_properties is list of 3 elements, "
                    "includes: name (string), dtype (string or numpy.dtype), "
                    "and statistic-able (bool), but given element is: %s" %
                    str([name, dtype, static_able]))
        return self._features_properties

    @property
    def external_indices(self):
        """Return a list of feature names that have separated indices
        file """
        for feature_name in self._external_indices:
            if not is_string(feature_name):
                raise ValueError('external_indices is list of feature_name which '
                    'is string type.')
        return self._external_indices

    @property
    def excluded_pca(self):
        """ All feature properties with name given in this list will
        be excluded during pca calculation"""
        for feature_name in self._excluded_pca:
            if not is_string(feature_name):
                raise ValueError('_excluded_pca is list of feature_name which '
                    'is string type.')
        return self._excluded_pca

    def _map_multiple_works(self, jobs):
        for j in jobs:
            for result in self.map(j):
                yield result

    @abstractmethod
    def map(self, job):
        """This function return an iterator of results"""
        pass

    @abstractmethod
    def _validate(self, ds, path, nb_samples, logger):
        """
        ds: Dataset (in read_only mode, auto opened and closed)
        """
        pass

    def validate(self, path, nb_samples=8):
        def logger(title, check):
            print(ctext('   *', 'cyan'),
                  ctext(title, 'yellow'),
                  "✓" if bool(check) else "✗")
        print(ctext('[%s]Validating dataset:' % self.__class__.__name__, 'red'),
              '"%s"' % self.output_path)
        ds = Dataset(self.output_path, read_only=True)
        path = str(path)
        nb_samples = int(nb_samples)
        if not os.path.exists(path):
            os.mkdir(path)
        elif os.path.isfile(path):
            raise ValueError("`path` must be a path to folder.")
        else:
            shutil.rmtree(path)
            os.mkdir(path)
        self._validate(ds, path, nb_samples, logger)
        ds.close()

    def run(self):
        if not hasattr(self, 'jobs'):
            raise Exception('the Processor must has "jobs" attribute, which is '
                            'the list of all jobs.')
        njobs = len(self.jobs) if self.njobs == 0 else self.njobs
        dataset = Dataset(self.output_path)
        datatype = self.datatype
        if self.ncpu is None: # auto select number of CPU
            ncpu = min(njobs, int(1.2 * cpu_count()))
        else:
            ncpu = self.ncpu
        # ====== indices ====== #
        databases = keydefaultdict(lambda name: MmapDict(
            path=os.path.join(dataset.path, name),
            cache_size=10000, read_only=False))
        # ====== statistic ====== #
        # mapping: feature_name -> able-to-calculate-statistics
        statistic_able = {name: stats_able
                          for name, dtype, stats_able in self.features_properties}
        sum1 = defaultdict(int)
        sum2 = defaultdict(int)
        # init PCA
        pca = defaultdict(lambda *args, **kwargs:
            MiniBatchPCA(n_components=None, whiten=self.pca_whiten,
                         copy=True, batch_size=None) if self.pca else None)
        # load old statistics and PCA if found
        for name, is_stats in statistic_able.iteritems():
            if is_stats:
                if name + '_sum1' in dataset:
                    sum1[name] = dataset[name + '_sum1'][:]
                if name + '_sum2' in dataset:
                    sum2[name] = dataset[name + '_sum2'][:]
                if name + '_pca' in dataset:
                    pca[name] = dataset[name + '_pca']
        # all data are cached for periodically flushed
        cache = defaultdict(list)
        if self.ncache <= 1:
            cache_limit = max(2, int(0.12 * njobs))
        else:
            cache_limit = int(self.ncache)
        # ref_vars[start]
        ref_vars = {'start': defaultdict(int), 'processed_count': 0}

        # ====== helper ====== #
        def flush_feature(name, cache_data):
            if len(cache_data) > 0:
                cache_data = np.concatenate(cache_data, 0)
                # NOTE: if nb_samples < nb_features, fitting PCA
                # will course error
                if self.pca and statistic_able[name] and \
                name not in self.excluded_pca and \
                (cache_data.ndim >= 2 and cache_data.shape[-1] > 1):
                    pca[name].partial_fit(cache_data)
                # flush data
                if name in dataset:
                    dataset[name].append(cache_data)
                else:
                    dataset[(name, datatype)] = cache_data

        # ====== repeated for each result returned ====== #
        def wrapped_reduce(result):
            name, job_count, data = result
            ref_vars['processed_count'] += 1
            # check data
            if not isinstance(data, (tuple, list)):
                data = (data,)
            saved_indices = []
            # processing
            for prop, d in zip(self.features_properties, data):
                # feature-type-name, dtype, stats-able
                feat_name, feat_type, feat_stat = prop
                # specal case: dict type
                if 'dict' in str(feat_type).lower():
                    databases[feat_name][name] = \
                        (d.tolist() if isinstance(d, np.ndarray) else d)
                    del d
                    continue
                # auto-create new indices
                if feat_name in self.external_indices:
                    ids_name = 'indices_%s' % feat_name
                else:
                    ids_name = 'indices'
                # do not save and increase the count of one indices multiple time
                if ids_name not in saved_indices:
                    databases[ids_name][name] = (ref_vars['start'][ids_name],
                                                 ref_vars['start'][ids_name] + len(d))
                    ref_vars['start'][ids_name] += len(d)
                    saved_indices.append(ids_name)
                # cache data, only if we have more than 0 sample
                if len(d) > 0:
                    cache[feat_name].append(d.astype(feat_type))
                    if self.save_stats and feat_stat: # save stats
                        sum1[feat_name] += np.sum(d, axis=0, dtype='float64')
                        sum2[feat_name] += np.sum(np.power(d, 2), axis=0, dtype='float64')
                del d
            # ====== flush cache ====== #
            if ref_vars['processed_count'] % cache_limit == 0: # 12 + 8
                for i, j in cache.iteritems():
                    flush_feature(i, j)
                cache.clear()
            # ====== update progress ====== #
            return name, job_count
        # ====== processing ====== #
        mpi = MPI(jobs=self.jobs,
                  map_func=self._map_multiple_works,
                  reduce_func=wrapped_reduce,
                  ncpu=ncpu,
                  buffer_size=min(8, max(len(self.jobs) // ncpu, 1)),
                  maximum_queue_size=ncpu * 3,
                  chunk_scheduler=True)
        prog = Progbar(target=njobs, name=self.__class__.__name__,
                       interval=0.1, print_report=True, print_summary=True)
        for name, job_count in mpi:
            prog['File'] = '%-20s' % name
            prog.add(job_count)
        # ====== end, flush the last time ====== #
        for i, j in cache.iteritems():
            flush_feature(i, j)
        cache = None
        dataset.flush()
        prog.add_notification("Flushed all data to disk")
        # ====== saving indices ====== #
        for name, db in databases.iteritems():
            db.flush(save_indices=True)
            db.close()
            prog.add_notification('Flush MmapDict "%s" to disk' %
                                  ctext(name, 'yellow'))

        # ====== save mean and std ====== #
        def save_mean_std(sum1, sum2, pca, name, dataset):
            N = dataset[name].shape[0]
            mean = sum1 / N
            std = np.sqrt(sum2 / N - mean**2)
            if self.substitute_nan is not None:
                mean = np.where(np.isnan(mean), self.substitute_nan, mean)
                std = np.where(np.isnan(std), self.substitute_nan, std)
            else:
                assert not np.any(np.isnan(mean)), 'Mean contains NaN, name: %s' % name
                assert not np.any(np.isnan(std)), 'Std contains NaN, name: %s' % name
            dataset[name + '_sum1'] = sum1
            dataset[name + '_sum2'] = sum2
            dataset[name + '_mean'] = mean
            dataset[name + '_std'] = std
            if pca is not None and pca.is_fitted:
                dataset[name + '_pca'] = pca
        # save all stats
        if self.save_stats:
            for n, d, s in self.features_properties:
                if s: # save stats
                    prog.add_notification('Saving statistics of: %s' % n)
                    s1, s2 = sum1[n], sum2[n],
                    if self.pca and n not in self.excluded_pca:
                        pca_ = pca[n]
                    else:
                        pca_ = None
                    save_mean_std(s1, s2, pca_, n, dataset)
        # ====== dataset flush() ====== #
        dataset.flush()
        dataset.close()
        # ====== saving the configuration ====== #
        config_path = os.path.join(self.output_path, 'config')
        # if found exist config, increase the count
        if os.path.exists(config_path):
            config_count = 1
            while True:
                if not os.path.exists(config_path + str(config_count)):
                    config_path = config_path + str(config_count)
                    break
                config_count += 1
        # save the new configuration
        config = MmapDict(config_path)
        config['__configuration_time__'] = time.time()
        for i in dir(self):
            if _default_module.match(i) is not None:
                continue
            j = getattr(self, i)
            if isinstance(j, (Number, string_types, bool)):
                config[i] = j
        config.flush(save_indices=True)
        config.close()
        prog.add_notification("Saved Processor configuration.")
        prog.add_notification("Closed all dataset.")

# ===========================================================================
# Speech features
# ===========================================================================
def _valid_segment_name(segments):
    for _, i in segments:
        if '.' in i[0] or ':' in i[0]:
            raise ValueError("Segment name cannot contain: '.' or ':', the given"
                " name is: %s" % i[0])

def _segments_preprocessing(segments, audio_ext):
    """ Filter segments into map of jobs
    Return
    ------
    jobs: dict
        file_name -> [segments, ...]
    nb_jobs: int
        total number of segment found
    """

    audio_ext = as_tuple('' if audio_ext is None else audio_ext,
                         t=string_types)
    # ====== load jobs ====== #
    if isinstance(segments, Dataset):
        # WAVE dataset
        if ('indices' in segments and 'raw' in segments and 'sr' in segments):
            file_list = [(name, segments, 0., -1., 0)
                         for name, (start, end) in segments['indices'].iteritems()]
        else: # assume that each key in dataset is a files
            file_list = [(os.path.basename(segments[k]), segments[k], 0.0, -1.0, 0)
                         for k in segments.keys()] # segment, path, start, end
    # NOT loaded segments
    elif isinstance(segments, str):
        if not os.path.exists(segments):
            raise ValueError('Path to segments must exists, however, '
                             'exist(segments)={}'.format(os.path.exists(segments)))
        # given a directory
        if os.path.isdir(segments):
            file_list = get_all_files(segments)
            file_list = [(os.path.basename(i), i, 0.0, -1.0)
                         for i in file_list] # segment, path, start, end
        # given csv file
        else:
            file_list = np.genfromtxt(segments, dtype=str, delimiter=' ')
    # LOADED segments
    elif isinstance(segments, (tuple, list, np.ndarray)):
        # just a list of path to file
        if isinstance(segments[0], str):
            file_list = [(os.path.basename(i), os.path.abspath(i), 0.0, -1.0)
                         for i in segments]
        # list of all information
        elif isinstance(segments[0], (tuple, list)):
            if len(segments[0]) == 1: # only path is given
                segments = [(path, path, 0., -1., 0) for path in segments]
            elif len(segments[0]) == 2: # name and path are given
                segments = [(name, path, 0., -1., 0) for name, path in segments]
            elif len(segments[0]) != 4 and len(segments[0]) != 5:
                raise Exception('segments must contain information in following order:'
                                '[name] [path] [start] [end] [channel]')
            file_list = segments
    # filter using support audio extension
    file_list = [f for f in file_list
                 if ((isinstance(f[1], str) and
                    any(ext in f[1][-len(ext):] for ext in audio_ext)) or
                 isinstance(f[1], Dataset))]
    # if no channel is provided, append the channel
    file_list = [list(f) + [0] if len(f) == 4 else f for f in file_list]
    nb_jobs = len(file_list)
    # convert into: audio_path -> list_of_segments[(name, start, end, channel), ...]
    jobs = []
    file_jobs = defaultdict(list)
    for segment, path_or_ds, start, end, channel in file_list:
        # Dataset related jobs
        if isinstance(path_or_ds, Dataset):
            jobs.append((path_or_ds, [(segment, start, end, channel)]))
        # audio files jobs
        else:
            file_jobs[path_or_ds].append(
                (segment, float(start), float(end), int(channel)))
    file_jobs = sorted(file_jobs.items(), key=lambda x: x[0])
    jobs += file_jobs
    _valid_segment_name(jobs)
    # check empty jobs
    if len(jobs) == 0:
        raise Exception('NO jobs found for processing.')
    return jobs, nb_jobs

def _load_audio(path_or_ds, segments,
                sr, sr_info={}, sr_new=None, best_resample=True,
                maxlen=None, vad_split=False, vad_split_args={},
                remove_dc_offset=True):
    """ Return iterator of (name, data, sr) """
    # directory path for Dataset
    if is_string(path_or_ds) and os.path.isdir(path_or_ds):
        path_or_ds = Dataset(path_or_ds)
    # iterate over file path
    if is_string(path_or_ds) or isinstance(path_or_ds, file):
        s, sr_orig = speech.read(path_or_ds,
                                 remove_dc_offset=remove_dc_offset)
        # check original sample rate
        if sr_orig is not None and sr is not None and sr_orig != sr:
            raise RuntimeError('Given sample rate (%d Hz) is different from '
                               'audio file sample rate (%d Hz).' %
                               (sr, sr_orig))
        # get given sr
        if sr_orig is None:
            sr_orig = sr
        # get from sr_info
        if sr_orig is None and is_string(path_or_ds):
            sr_orig = sr_info.get(path_or_ds, None)
        # still None, then exception
        if sr_orig is None:
            raise RuntimeError("Cannot acquire original sample rate from "
                               "loaded utterance, or from given arguments "
                               "of this Processor (file: '%s')." % str(path_or_ds))
        # check if audio file is not long enough, ignore it
        if len(s) < 25:
            raise RuntimeError("Audio at path: '%s' is too short, length: %f(s)"
                               % (str(path_or_ds), len(s) / sr_orig))
        # downsampling
        if sr_new is not None:
            s = speech.resample(s, sr_orig, sr_new, best_algorithm=best_resample)
            sr_orig = sr_new
        N = len(s)
    # vad_split_audio kwargs
    minimum_duration = vad_split_args.get('minimum_duration', None)
    frame_length = vad_split_args.get('frame_length', 128)
    nb_mixtures = vad_split_args.get('nb_mixtures', 3)
    threshold = vad_split_args.get('threshold', 0.6)
    # ====== cut into segments ====== #
    for name, start, end, channel in segments:
        # iterate over dataset
        if isinstance(path_or_ds, Dataset):
            st, en = path_or_ds['indices'][name]
            s = path_or_ds['raw'][st:en]
            N = len(s)
            sr_orig = path_or_ds['sr'][name]
        # start processing
        if 0. <= start < 1. and 0. < end <= 1.: # percentage
            start = int(start * N)
            end = int(np.ceil(end * N))
        else: # given the duration in second
            start = int(float(start) * sr_orig)
            end = int(N if end <= 0 else float(end) * sr_orig)
        # check maxlen
        if maxlen is not None and (end - start > maxlen * sr_orig):
            # using VAD information to split the audio
            if vad_split:
                data = s[start:end, channel] if s.ndim > 1 else s[start:end]
                data = signal.vad_split_audio(data, sr=sr_orig,
                    maximum_duration=maxlen, minimum_duration=minimum_duration,
                    frame_length=frame_length, nb_mixtures=nb_mixtures,
                    threshold=threshold)
                accum_length = np.cumsum([0] + [len(i) for i in data[:-1]])
                for st, d in zip(accum_length, data):
                    st_ = ('%f' % (st / sr_orig)).rstrip('0').rstrip('.')
                    en_ = ('%f' % ((st + len(d)) / sr_orig)).rstrip('0').rstrip('.')
                    yield (name + ":%s:%s" % (st_, en_),
                           d,
                           sr_orig)
            # just cut into small segments
            else:
                maxlen = int(maxlen * sr_orig)
                _ = list(range(start, end, maxlen)) + [end]
                for st, en in zip(_, _[1:]):
                    st_ = ('%f' % (st / sr_orig)).rstrip('0').rstrip('.')
                    en_ = ('%f' % (en / sr_orig)).rstrip('0').rstrip('.')
                    yield (name + ":%s:%s" % (st_, en_),
                           s[st:en, channel] if s.ndim > 1 else s[st:en],
                           sr_orig)
        # return normally
        else:
            yield (name,
                   s[start:end, channel] if s.ndim > 1 else s[start:end],
                   sr_orig)

class WaveProcessor(FeatureProcessor):
    """ Concatenate all Waveform data into single memmap (or HDF5) file
    with its meta-data information included in the indices

    The saved Dataset contains 3 Data:
     * "indices": MmapDict contain the mapping from file name to (start, end).
     * "raw": the big memmap contains all concatenated raw waveform.
     * "sr": MmapDict contains the mapping from file name to its sample rate.

    Parameters
    ----------
    segments : path, list
        if path, directory of all audio file, or segment csv file in
        following format (channel can be omitted), `start` and `end`
        is in second (if `start`, or `end` is smaller than 1.0, then they
        are understand as percentage)
            name                |     path             |start|end |channel
        ------------------------|----------------------|-----|----|---
        sw02001-A_000098-001156 | /path/to/sw02001.sph | 0.0 | -1 | 0
        sw02001-B_001980-002131 | /path/to/sw02001.sph | 0.0 | -1 | 1
    output_path: str
        path to output folder
    sr: int
        sample rate
    sr_info: dict
        mapping audio_file_path -> sampling_rate for each segment
        if provided.
    sr_new: int or None
        new sample rate (if you want to down or up sampling)
    best_resample: bool
        if True, use the best but slow algorithm for resampling
    maxlen: int
        maximum length of an utterances in second, if any file is longer than
        given length, it is divided into small segments and the start time and
        end time are concatenated to the name (e.g. file:0:30)
    vad_split: boolean (default: False)
        using VAD information to split the audio in most likely silence part.
    vad_split_args: dict
        kwargs for `odin.preprocessing.signal.vad_split_audio`, includes:
        (minimum_duration, frame_length, nb_mixtures, threshold)
    dtype: numpy.dtype, None, 'auto'
        if None or 'auto', keep the original dtype of audio
    ignore_error: boolean (default: False)
        if True, ignore error files during processing
    """

    def __init__(self, segments, output_path,
                sr=None, sr_info={}, sr_new=None, best_resample=True,
                audio_ext=None, pcm=False, remove_dc_offset=True,
                maxlen=None, vad_split=False, vad_split_args={},
                dtype='float16', datatype='memmap',
                ignore_error=False, ncache=0.12, ncpu=1):
        super(WaveProcessor, self).__init__(output_path=output_path,
            datatype=datatype, pca=False, pca_whiten=False,
            save_stats=False, substitute_nan=False,
            ncache=ncache, ncpu=ncpu)
        if isinstance(segments, Dataset):
            raise ValueError("WaveProcessor does not support segments as a Dataset.")
        self.maxlen = None if maxlen is None else int(maxlen)
        self.vad_split = bool(vad_split)
        self.vad_split_args = vad_split_args
        self.jobs, self.njobs = _segments_preprocessing(segments, audio_ext)
        if dtype is None or (is_string(dtype) and dtype == 'auto'):
            s, _ = speech.read(self.jobs[0][0], pcm=pcm, dtype=None)
            dtype = s.dtype
            del s
        self.sr = sr
        self.sr_info = sr_info
        self.sr_new = sr_new
        self.best_resample = bool(best_resample)
        self.dtype = dtype
        self.pcm = pcm
        self.remove_dc_offset = remove_dc_offset
        self._features_properties = [('raw', self.dtype, False),
                                     ('sr', 'dict', False),
                                     ('dtype', 'dict', False)]
        self.ignore_error = bool(ignore_error)

    def map(self, job):
        audio_path, segments = job[0] if len(job) == 1 else job
        nb_jobs = len(segments)
        try:
            # processing all segments
            ret = []
            for name, data, sr in _load_audio(audio_path, segments,
                        self.sr, self.sr_info, self.sr_new, self.best_resample,
                        self.maxlen, self.vad_split, self.vad_split_args,
                        remove_dc_offset=self.remove_dc_offset):
                ret.append([name, 0, [data, int(sr), data.dtype.str]])
            # a hack to return proper amount of processed jobs
            ret[-1][1] = nb_jobs
            # return result
            return (i for i in ret)
        except Exception as e:
            import traceback; traceback.print_exc()
            msg = '\n[Error file]: %s, [Exception]: %s\n' % (audio_path, str(e))
            if self.ignore_error:
                add_notification(msg)
            else:
                raise RuntimeError(msg)

    def _validate(self, ds, path, nb_samples, logger):
        import matplotlib
        matplotlib.use('Agg')
        from matplotlib import pyplot as plt
        from odin.visual import plot_save
        from scipy.io.wavfile import write
        # ====== checking indices ====== #
        indices = sorted([(name, start, end)
                         for name, (start, end) in ds['indices'].iteritems()],
                         key=lambda x: x[1])
        prog = Progbar(target=len(indices) // 2, interval=0.1)
        for prev, now in zip(indices, indices[1:]):
            prog.add(1)
            assert prev[2] == now[1] # non-zero length
            assert prev[2] - prev[1] > 0 # non-zero length
            assert now[2] - now[1] > 0 # non-zero length
        assert now[-1] == len(ds['raw']) # length match length of raw
        print(); logger("Checked all indices", True)
        # ====== check sample rate ====== #
        for name, _, _ in Progbar(indices, interval=0.1,
                                  print_report=True,
                                  report_func=lambda x: [('Name', x[0])],
                                  count_func=lambda x: 1):
            sr = ds['sr'][name]
            dtype = ds['dtype'][name]
            assert sr > 0 and dtype
        logger("Checked all sample rate and data type", True)
        # ====== checking raw signal ====== #
        samples = np.random.choice(
            np.arange(len(indices)), size=nb_samples, replace=False)
        saved_samples = {}
        figure_path = os.path.join(path, 'raw.pdf')
        for i, (name, start, end) in enumerate(Progbar(indices, interval=0.1,
                                                count_func=lambda x: 1,
                                                report_func=lambda x: [('Name', x[0])],
                                                print_report=True)):
            raw = ds['raw'][start:end]
            assert not np.any(np.isnan(raw)) # No NaN value
            assert not np.all(np.isclose(raw, 0.)) # not all value closed to zeros
            # saving audio
            if i in samples:
                _ = os.path.join(path, 'tmp%d.wav' % i)
                raw = raw[:].astype('float32')
                raw = (raw - raw.mean()) / raw.std()
                write(_, rate=ds['sr'][name], data=raw)
                saved_samples[name] = _
                # saving figure
                with warnings.catch_warnings():
                    warnings.filterwarnings("ignore", category=RuntimeWarning)
                    plt.figure(figsize=(10, 4))
                    plt.plot(speech.resample(raw, ds['sr'][name], 8000,
                                             best_algorithm=False))
                    plt.title(name)
        plot_save(figure_path, dpi=80, log=False)
        logger("Checked all raw signal", True)
        for name, save_path in saved_samples.iteritems():
            logger('Saved "%s" at path: %s' % (name, save_path), True)
        logger("Checked all raw signal", True)
        logger("Saved figure at path: %s" % figure_path, True)
        logger("All reports at folder: %s" % path, True)

class SpeechProcessor(FeatureProcessor):

    ''' Extract speech features from all audio files in given directory or
    file list, then saves them to a `keras.ext.dataset.Dataset`

    Parameters
    ----------
    segments : path, list
        if path, directory of all audio file, or segment csv file in
        following format (channel can be omitted), `start` and `end` is in second
        (if `start`, or `end` is smaller than 1. then they are understand as
        percentage)
            name                |     path             |start|end |channel
        ------------------------|----------------------|-----|----|---
        sw02001-A_000098-001156 | /path/to/sw02001.sph | 0.0 | -1 | 0
        sw02001-B_001980-002131 | /path/to/sw02001.sph | 0.0 | -1 | 1
    output_path: str
        path to output folder
    sr: int
        sample rate
    sr_info: dict
        mapping audio_file_path -> sampling_rate for each segment
        if provided.
    sr_new: int or None
        new sample rate (if you want to down or up sampling)
    best_resample: bool
        if True, use the best but slow algorithm for resampling
    win: float
        window length in millisecond
    hop: float
        hop length between windows, in millisecond
    nb_melfilters: int, or None
        number of Mel bands to generate, if None, mel-filter banks features
        won't be returned
    nb_ceps: int, or None
        number of MFCCs to return, if None, mfcc coefficients won't be
        returned
    get_spec: bool
        if True, include the log-power spectrogram
    get_qspec: bool
        if True, return Q-transform coefficients
    get_phase: bool
        if True, return phase components of STFT
    get_pitch:
        if True, include the Pitch frequency (F0)
    get_vad: int, bool
        if True, include the indicators of voice activities detection.
        if int, `get_vad` is the number of Gaussian mixture components for VAD.
        by default, use 2 distribution.
    get_energy: bool
        if True, include the log energy of each frames
    get_delta: bool or int
        if True and > 0, for each features append the delta with given order-th
        (e.g. delta=2 will include: delta1 and delta2)
    fmin : float > 0 [scalar]
        lower frequency cutoff.
    fmax : float > 0 [scalar]
        upper frequency cutoff.
    preemphasis: float `(0, 1)`
        pre-emphasis coefficience
    pitch_threshold: float in `(0, 1)`
        Voice/unvoiced threshold. Default is 0.3.
    pitch_fmax: float
        maximum frequency of pitch
    pitch_algo: 'swipe', 'rapt', 'avg'
        SWIPE - A Saw-tooth Waveform Inspired Pitch Estimation.
        RAPT - a robust algorithm for pitch tracking.
        avg - apply swipe and rapt at the same time, then take average.
        Default is 'SWIPE'
    vad_smooth: int, bool
        window length to smooth the vad indices.
        If True default window length is 3.
    vad_minlen: float (in second)
        the minimum length of audio segments that can be considered
        speech.
    cqt_bins : int > 0
        Number of frequency bins for constant Q-transform, starting at `fmin`
    center : boolean
        - If `True`, the signal `y` is padded so that frame
          `D[:, t]` is centered at `y[t * hop_length]`.
        - If `False`, then `D[:, t]` begins at `y[t * hop_length]`
    power : float > 0 [scalar]
        Exponent for the magnitude spectrogram.
        e.g., 1 for energy, 2 for power, etc.
    log: bool
        if True, convert all power spectrogram to DB
    backend: 'odin', 'sptk'
        support backend for calculating the spectra
    pca: bool
        save trained PCA for each features
    pca_whiten : bool
        When True (False by default) the ``components_`` vectors are divided
        by ``n_samples`` times ``components_`` to ensure uncorrelated outputs
        with unit component-wise variances.
        Whitening will remove some information from the transformed signal
        (the relative variance scales of the components) but can sometimes
        improve the predictive accuracy of the downstream estimators by
        making data respect some hard-wired assumptions.
    maxlen: int
        maximum length of an utterances in second, if any file is longer than
        given length, it is divided into small segments and the start time and
        end time are concatenated to the name (e.g. file:0:30)
    vad_split: boolean (default: False)
        using VAD information to split the audio in most likely silence part.
    vad_split_args: dict
        kwargs for `odin.preprocessing.signal.vad_split_audio`, includes:
        (minimum_duration, frame_length, nb_mixtures, threshold)
    save_raw: bool
        if True, saving the raw signal together with all the acoustic features
    save_stats: bool
        same the first order and second order statistics, standard deviation
        of all features
    substitute_nan: bool
        if the statistics contain NaN, replace them with zero of given
        value
    dtype: 'float16', 'float32', 'float64'
        the dtype of saved features
    datatype: 'memmap', 'hdf5'
        store processed features in memmap or hdf5
    ignore_error: boolean (default: False)
        if True, ignore error files during processing
    ncache: float or int
        number of samples are kept until flush to the disk.
    ncpu: int
        number of CPU used for this task.

    Return
    ------
    spec, mspec, mfcc, pitch, vad_idx

    Example
    -------
    >>> feat = F.SpeechProcessor(datapath, output_path, audio_ext='wav', fs=8000,
    >>>                          win=0.025, hop=0.01, n_filters=40, n_ceps=13,
    >>>                          delta_order=2, energy=True, pitch_threshold=0.5,
    >>>                          get_spec=True, get_mspec=True, get_mfcc=True,
    >>>                          get_pitch=False, get_vad=True,
    >>>                          save_stats=True, substitute_nan=None,
    >>>                          dtype='float32', datatype='memmap', ncpu=4)
    >>> feat.run()
    '''

    def __init__(self, segments, output_path,
                sr=None, sr_info={}, sr_new=None, best_resample=True,
                win=0.02, hop=0.01, window='hann',
                nb_melfilters=None, nb_ceps=None,
                get_spec=True, get_qspec=False, get_phase=False,
                get_pitch=False, get_f0=False,
                get_vad=True, get_energy=False, get_delta=False,
                fmin=64, fmax=None,
                pitch_threshold=0.3, pitch_fmax=260, pitch_algo='swipe',
                vad_smooth=3, vad_minlen=0.1,
                cqt_bins=96, preemphasis=None,
                center=True, power=2, log=True, backend='odin',
                pca=True, pca_whiten=False,
                audio_ext=None,
                maxlen=None, vad_split=False, vad_split_args={},
                save_raw=False, save_stats=True, substitute_nan=None,
                dtype='float16', datatype='memmap',
                ignore_error=False, ncache=0.12, ncpu=1):
        super(SpeechProcessor, self).__init__(output_path=output_path,
            datatype=datatype, pca=pca, pca_whiten=pca_whiten,
            save_stats=save_stats, substitute_nan=substitute_nan,
            ncache=ncache, ncpu=ncpu)
        self.maxlen = None if maxlen is None else int(maxlen)
        self.vad_split = bool(vad_split)
        self.vad_split_args = vad_split_args
        self.jobs, self.njobs = _segments_preprocessing(segments, audio_ext)
        # ====== which features to get ====== #
        features_properties = []
        if save_raw:
            features_properties.append(('raw', dtype, False))
        if get_spec: features_properties.append(('spec', dtype, True))
        if get_energy: features_properties.append(('energy', dtype, True))
        if nb_melfilters is not None:
            features_properties.append(('mspec', dtype, True))
        if nb_ceps is not None:
            features_properties.append(('mfcc', dtype, True))
        if get_qspec:
            features_properties.append(('qspec', dtype, True))
            if nb_melfilters is not None:
                features_properties.append(('qmspec', dtype, True))
            if nb_ceps is not None:
                features_properties.append(('qmfcc', dtype, True))
            if get_phase: features_properties.append(('qphase', dtype, True))
        if get_phase: features_properties.append(('phase', dtype, True))
        if get_pitch: features_properties.append(('pitch', dtype, True))
        if get_f0: features_properties.append(('f0', dtype, True))
        if get_vad:
            features_properties.append(('vad', 'uint8', False))
            features_properties.append(('vadids', 'dict', False))
        # store the sample rate of each file also
        features_properties.append(('sr', 'dict', False))
        self._features_properties = features_properties
        # control FeatureProcessor behaviour
        self._external_indices = ['vadids']
        self._excluded_pca = ['energy', 'vad']
        # ====== local variable ====== #
        self.get_spec = get_spec
        self.get_pitch = get_pitch
        self.get_f0 = get_f0
        self.get_qspec = get_qspec
        self.get_phase = get_phase
        self.get_vad = get_vad
        self.get_energy = get_energy
        self.get_delta = 0 if get_delta is None else int(get_delta)
        self.save_raw = save_raw
        # ====== feature information ====== #
        self.sr = sr
        self.sr_new = sr_new
        self.sr_info = sr_info
        self.best_resample = bool(best_resample)
        self.win = win
        self.hop = hop
        self.window = window
        self.nb_melfilters = nb_melfilters
        self.nb_ceps = nb_ceps
        # constraint pitch threshold in 0-1
        self.pitch_threshold = min(max(pitch_threshold, 0.), 1.)
        self.pitch_fmax = pitch_fmax
        self.pitch_algo = pitch_algo
        self.vad_smooth = vad_smooth
        self.vad_minlen = vad_minlen
        self.cqt_bins = cqt_bins
        self.fmin = fmin
        self.fmax = fmax
        self.preemphasis = preemphasis
        self.center = center
        self.power = power
        self.log = log
        self.backend = backend
        self.ignore_error = bool(ignore_error)

    # ==================== Abstract properties ==================== #
    def map(self, job):
        '''
        Return
        ------
        [(name, spec, mspec, mfcc, pitch, vad), ...]
        '''
        audio_path, segments = job[0] if len(job) == 1 else job
        nb_jobs = len(segments)
        try:
            ret = []
            for name, data, sr_orig in _load_audio(audio_path, segments,
                                self.sr, self.sr_info, self.sr_new, self.best_resample,
                                self.maxlen, self.vad_split, self.vad_split_args):
                with warnings.catch_warnings():
                    warnings.filterwarnings("ignore", category=UserWarning)
                    features = speech.speech_features(data.ravel(), sr=sr_orig,
                        win=self.win, hop=self.hop, window=self.window,
                        nb_melfilters=self.nb_melfilters, nb_ceps=self.nb_ceps,
                        get_spec=self.get_spec, get_qspec=self.get_qspec,
                        get_phase=self.get_phase,
                        get_pitch=self.get_pitch, get_f0=self.get_f0,
                        get_vad=self.get_vad, get_energy=self.get_energy,
                        get_delta=self.get_delta,
                        pitch_threshold=self.pitch_threshold,
                        pitch_fmax=self.pitch_fmax,
                        pitch_algo=self.pitch_algo,
                        vad_smooth=self.vad_smooth, vad_minlen=self.vad_minlen,
                        cqt_bins=self.cqt_bins, fmin=self.fmin, fmax=self.fmax,
                        sr_new=None, preemphasis=self.preemphasis,
                        center=self.center, power=self.power, log=self.log,
                        return_raw=self.save_raw, backend=self.backend)
                if features is not None:
                    saved_features = []
                    found_NaN = False
                    for i in self.features_properties[:-1]:
                        feat = features[i[0]]
                        if isinstance(feat, np.ndarray) and \
                        sum(feat.shape) > 0 and np.isnan(np.min(feat)):
                            found_NaN = True
                        else:
                            saved_features.append(feat)
                    # append the sample rate
                    if found_NaN:
                        warnings.warn('Ignore segments: %s, error: NaN values' % name)
                    else:
                        saved_features.append(sr_orig if self.sr_new is None
                                              else self.sr_new)
                        ret.append([name, 0, saved_features])
                else:
                    warnings.warn('Ignore segments: %s, no features found' % name)
            # a hack to return proper amount of processed jobs
            ret[-1][1] = nb_jobs
            # return the results as a generator
            return (i for i in ret)
        except Exception as e:
            import traceback; traceback.print_exc()
            msg = '\n[Error file]: %s, [Exception]: %s\n' % (audio_path, str(e))
            if self.ignore_error:
                add_notification(msg)
            else:
                raise RuntimeError(msg)

    def _validate(self, ds, path, nb_samples, logger):
        print(ds)
        import matplotlib
        matplotlib.use('Agg')
        from matplotlib import pyplot as plt
        from odin.visual import plot_save
        from scipy.io.wavfile import write
        # ====== get all saved features ====== #
        features_check = []
        stats_check = []
        for name, dtype, stats in self.features_properties:
            pass
        # ====== checking indices ====== #
        indices = sorted([(name, start, end)
                         for name, (start, end) in ds['indices'].iteritems()],
                         key=lambda x: x[1])
        for prev, now in zip(indices, indices[1:]):
            assert prev[2] == now[1] # non-zero length
            assert prev[2] - prev[1] > 0 # non-zero length
            assert now[2] - now[1] > 0 # non-zero length
        assert now[-1] == len(ds['raw']) # length match length of raw
        logger("Checked all indices", True)
        # ====== check sample rate ====== #
        for name, _, _ in indices:
            sr = ds['sr'][name]
            assert sr > 0
        logger("Checked all sample rate and data type", True)
        logger("All reports at folder: %s" % path, True)

# ===========================================================================
# Images
# ===========================================================================
class ImageFeatures(FeederRecipe):
    """ ImageFeauters extractor
    This function take output from Feeder with SpeechFeatures recipe
    and update output dataset

    Parameters
    ----------
    image_ext: str, or list of str
        extensions of images
    grayscale: bool
        force to convert Image to grayscale or not
    crop: 4-tuple of int
         (left, upper, right, lower)
    target_size: 2-tuple of int
        desire size for image (image will be padded if the size
        mis-match)
    transpose: int, or list of int
        if a list of int is provided, will return a list of images
        <0: Do nothing
        0: PIL.Image.FLIP_LEFT_RIGHT
        1: PIL.Image.FLIP_TOP_BOTTOM
        2: PIL.Image.ROTATE_90
        3: PIL.Image.ROTATE_180
        4: PIL.Image.ROTATE_270
        5: PIL.Image.TRANSPOSE
    resample_mode: int
        0 = PIL.Image.NEAREST: use nearest neighbour
        1 = PIL.Image.LANCZOS: a high-quality downsampling filter
        2 = PIL.Image.BILINEAR: linear interpolation
        3 = PIL.Image.BICUBIC: cubic spline interpolation

    """

    def __init__(self, image_ext=None, grayscale=False,
                 crop=None, target_size=None,
                 transpose=None, resample_mode=2):
        super(ImageFeatures, self).__init__()
        self.image_ext = ('',) if image_ext is None else as_tuple(image_ext,
                                                                  t=string_types)
        self.crop = crop if crop is None else as_tuple(crop, 4, int)
        self.grayscale = bool(grayscale)
        self.target_size = target_size
        self.transpose = (-1,) if transpose is None else as_tuple(transpose, t=int)
        self.resample_mode = resample_mode

    def shape_transform(self, shape):
        """ Return the new shape that transformed by this Recipe """
        return (shape[0] * len(self.transpose),) + shape[1:]

    def preprocess_indices(self, indices):
        # filter using support audio extension
        file_list = np.array([f for f in indices
                              if any(ext in f for ext in self.image_ext)])
        return file_list

    def map(self, path):
        '''
        Return
        ------
        [(name, spec(x, sum1, sum2), # if available, otherwise None
                mspec(x, sum1, sum2), # if available, otherwise None
                mfcc(x, sum1, sum2), # if available, otherwise None
                pitch(x, sum1, sum2), # if available, otherwise None
                vad), ...]
        '''
        X = image.read(path, grayscale=self.grayscale,
                       crop=self.crop, scale=None,
                       target_size=self.target_size,
                       transpose=self.transpose,
                       resample_mode=self.resample_mode)
        if not isinstance(X, (tuple, list)):
            X = (X,)
        name = os.path.basename(path)
        ret = []
        for i, j in zip(self.transpose, X):
            ret.append(('%s,%d' % (name, i),
                        np.expand_dims(j, axis=0)))
        return ret

    def reduce(self, images):
        for img in images:
            for name, x in img:
                # contains different transpose of images
                yield (name, x)

# ===========================================================================
# Video features
# ===========================================================================
def video_features_extraction(X, boundingbox, desire_size):
    finalX = [X]
    dtype = X.dtype
    if boundingbox is not None:
        finalX = [list() for i in range(len(boundingbox[0]) // 4)]
        # travel through each frames
        for x, bound in zip(X, boundingbox):
            # get each bounding box
            for i, box in enumerate(np.reshape(bound, (-1, 4))):
                x_, y_, w_, h_ = box
                # zero area, ignore it
                if w_ == 0 or h_ == 0:
                    if desire_size is None: continue
                    tmp = np.zeros(desire_size, dtype=dtype)
                # ====== get the bounding ====== #
                else:
                    if desire_size is not None:
                        # crop in the center
                        x_ = x_ + w_ // 2 - desire_size[-2] // 2
                        w_ = desire_size[-2] # width
                        y_ = y_ + h_ // 2 - desire_size[-1] // 2
                        h_ = desire_size[-1] # height
                    tmp = x[:, x_:x_ + w_, y_:y_ + h_]
                    # if actual size smaller than desire_size
                    # perform padding with 0.
                    if tmp.shape[-2] != w_ or tmp.shape[-1] != h_:
                        _ = np.zeros(desire_size, dtype=dtype)
                        startX = int(w_ // 2 - tmp.shape[-2] / 2)
                        startY = int(h_ // 2 - tmp.shape[-1] / 2)
                        _[:, startX: startX + tmp.shape[-2],
                          startY: startY + tmp.shape[-1]] = tmp
                        tmp = _
                # add to final results
                finalX[i].append(tmp)
        # create 1 big array hold all images
        finalX = [np.asarray(x) for x in finalX]
        finalX = (finalX[0] if len(finalX) == 1
                  else np.concatenate(finalX, axis=1))
    return (finalX,
            np.sum(finalX, axis=0, dtype='float64'),
            np.sum(finalX**2, axis=0, dtype='float64'))

class VideoFeature(FeederRecipe):

    ''' Extract speech features from all audio files in given directory or
    file list, then saves them to a `keras.ext.dataset.Dataset`

    Parameters
    ----------
    segments : path, list
        if path, directory of all audio file, or segment csv file in
        following format (channel can be omitted), start and end is in second
            name                |     path             |start|end |
        ------------------------|----------------------|-----|----|
        sw02001-A_000098-001156 | /path/to/sw02001.mp4 | 0.0 | -1 |
        sw02001-B_001980-002131 | /path/to/sw02001.mp4 | 0.0 | -1 |
    size : tuple(width, height)
        desire size of the return features images
    boundingbox : None, dict
        mapping from filename to sequence of bounding box
        (region of interest), name -> [x(from left),y(from top),width,height]
        For example: if is multiple of 4, then extract multiple regions
        sw02001-A_000098-001156 ->  [[30, 40, 15, 20, .... ], ...]
        sw02001-B_001980-002131 ->  [[30, 40, 15, 20, .... ], ...]
    robust : bool
        run in robust mode, auto ignore error files

    datatype : memmap, hdf5

    Example
    -------
    '''

    def __init__(self, segments, output, size=None,
                 boundingbox=None, video_ext=None,
                 datatype='memmap', robust=True):
        super(VideoFeature, self).__init__('VideoFeature')

    def initialize(self):
        # reversed to height width for easy processing
        if self.size is not None:
            self.size = as_tuple(self.size, N=2, t=int)
        segments = self.segments
        video_ext = as_tuple('' if self.video_ext is None
                             else self.video_ext, 1, str)
        # ====== load jobs ====== #
        if isinstance(segments, str):
            if not os.path.exists(segments):
                raise ValueError('Path to segments must exists, however, '
                                 'exist(segments)={}'.format(os.path.exists(segments)))
            if os.path.isdir(segments):
                file_list = get_all_files(segments)
                file_list = [(os.path.basename(i), i, 0.0, -1.0)
                             for i in file_list] # segment, path, start, end
            else: # csv file
                file_list = np.genfromtxt(segments, dtype=str, delimiter=' ')
        elif isinstance(segments, (tuple, list)):
            if isinstance(segments[0], str): # just a list of path to file
                file_list = [(os.path.basename(i), os.path.abspath(i), 0.0, -1.0)
                             for i in segments]
            elif isinstance(segments[0], (tuple, list)):
                if len(segments[0]) != 4:
                    raise Exception('segments must contain information in following for:'
                                    '[name] [path] [start] [end]')
                file_list = segments
        # filter using support audio extension
        file_list = [f for f in file_list if any(ext in f[1] for ext in video_ext)]
        # convert into: audio_path -> segment(name, start, end, channel)
        self.jobs = defaultdict(list)
        names = []
        for segment, file, start, end in file_list:
            self.jobs[file].append((segment, float(start), float(end)))
            names.append(segment)
        self.jobs = sorted(self.jobs.items(), key=lambda x: x[0])
        # ====== load bounding box ====== #
        if self.boundingbox is not None:
            if not isinstance(self.boundingbox, dict):
                raise ValueError('Bounding box must be a dictionary')
            if set(names) != set(self.boundingbox.keys()):
                raise Exception('Segments names and boundingbox keys mismatch.')
        # ====== check output ====== #
        self.dataset = Dataset(self.output)
        self._temp_path = get_tempdir()
        print('Temporary dir created at:', self._temp_path)
        # remove old cache files
        for p in os.listdir(self._temp_path):
            os.remove(os.path.join(self._temp_path, p))

    def map_func(self, f):
        '''
        Return
        ------
        [(name, spec(x, sum1, sum2), # if available, otherwise None
                mspec(x, sum1, sum2), # if available, otherwise None
                mfcc(x, sum1, sum2), # if available, otherwise None
                pitch(x, sum1, sum2), # if available, otherwise None
                vad), ...]
        '''
        video_path, segments = f
        # read the whole video
        frames, fps = video.read(video_path)
        size = self.size
        if size is not None:
            size = (frames.shape[1],) + size
        # generating features
        features = []
        for name, start, end in segments:
            start = int(float(start) * fps)
            end = int(frames.shape[0] if end < 0 else end * fps)
            data = frames[start:end]
            # ====== check bounding box ====== #
            box = (None if self.boundingbox is None
                   else self.boundingbox[name])
            tmp = video_features_extraction(data, box, size)
            if tmp is not None:
                features.append((name,) + tmp)
            else:
                msg = 'Ignore segments: %s, error: NaN values' % name
                warnings.warn(msg)
        # return an iterator of features
        del frames
        for name, x, sum1, sum2 in features:
            path = os.path.join(self._temp_path, name)
            # save big array, because the video can be very
            # big so we don't transfer it to Queue
            f = open(path, 'w'); np.save(f, x); f.close()
            yield name, path, sum1, sum2

    def reduce_func(self, results):
        # contains (name, spec, mspec, mfcc, vad)
        dataset = self.dataset
        datatype = self.datatype

        index = []
        sum1, sum2 = 0., 0.

        n = 0
        for name, path, s1, s2 in results:
            # load big array
            f = open(path, 'r'); X = np.load(f); f.close()
            if 'frames' in dataset: dataset['frames'].append(X)
            else: dataset[('frames', datatype)] = X
            # update running statistics
            sum1 += s1; sum2 += s2; n = X.shape[0]
            # index
            index.append([name, n])
            os.remove(path)
        dataset.flush()
        return (sum1, sum2, index)

    def finalize_func(self, results):
        # contains (sum1, sum2, n)
        dataset = self.dataset
        path = dataset.path
        sum1, sum2 = 0., 0.
        n = 0
        indices = []
        for s1, s2, index in results:
            # spec
            sum1 += s1
            sum2 += s2
            # indices
            for name, size in index:
                # name, start, end
                indices.append([name, int(n), int(n + size)])
                n += size
        # ====== saving indices ====== #
        with open(os.path.join(path, 'indices.csv'), 'w') as f:
            for name, start, end in indices:
                f.write('%s %d %d\n' % (name, start, end))
        # ====== helper ====== #
        mean = sum1 / n
        std = np.sqrt(sum2 / n - mean**2)
        assert not np.any(np.isnan(mean)), 'Mean contains NaN, name:' % os.path.basename(path)
        assert not np.any(np.isnan(std)), 'Std contains NaN, name:' % os.path.basename(path)
        dataset[name + '_mean'] = mean
        dataset[name + '_std'] = std
        # ====== clean up and release cv2 ====== #
        dataset.flush()
        dataset.close()
        # remove all temp file
        if os.path.exists(self._temp_path):
            os.remove(self._temp_path)
            self._temp_path = get_tempdir()
        return path

    def __setstate__(self, states):
        self.name = states[0]
        for name, value in states[1].iteritems():
            setattr(self, name, value)

    def __getstate__(self):
        return self.name, self._arguments


import os
import json
import pandas as pd
import win32com.client
import logging
from calendar_operations import add_to_calendar_with_earliest_start_end, remove_from_calendar

# Configure the logging system
logging.basicConfig(
    level=logging.DEBUG,
    format="%(asctime)s %(levelname)s: %(message)s",
    datefmt="%Y-%m-%d %H:%M:%S",
)

def load_config():
    script_dir = os.path.dirname(os.path.abspath(__file__))
    script_dir = os.path.dirname(__file__)
    config_path = os.path.join(script_dir, "config.json")
    with open(config_path, "r") as f:
        return json.load(f)

if __name__ == "__main__":
    logging.info("Starting main application.")

    try:
        config = load_config()
    except FileNotFoundError as e:
        logging.error("Config file not found!")
        exit(1)
    except json.JSONDecodeError as e:
        logging.error("Error decoding JSON config!")
        exit(1)

    xlsx_file = config.get("excel_file", "./16DayOct.xlsx")
    try:
        df = pd.read_excel(xlsx_file)
    except FileNotFoundError as e:
        logging.error(f"Excel file {xlsx_file} not found!")
        exit(1)

    # Log DataFrame details
    logging.debug(f"Shape of the DataFrame before filters: {df.shape}")
    logging.debug(f"Data types in the DataFrame before filters: {df.dtypes}")

    print("Debug: Head of DataFrame after reading Excel:")
    print(df.head())
    logging.debug(f"DataFrame Head: {df.head()}")

    logging.info("Applying filters.")
    df = df[~df["Function Room"].isin(config["exclude_function_room"])]
    df = df[~df["Event Type"].isin(config["exclude_event_type"])]
    df = df[df["Event Type"].isin(config["include_event_type"])]

    # Log DataFrame details after filters
    logging.debug(f"Shape of the DataFrame after filters: {df.shape}")
    logging.debug(f"Data types in the DataFrame after filters: {df.dtypes}")

    # Initialize the Outlook object
    outlook = win32com.client.Dispatch("Outlook.Application").GetNamespace("MAPI")
    root_folder = outlook.GetDefaultFolder(9)

    # Initialize the specific calendars
    calendars = {}
    for folder in root_folder.Folders:
        if folder.Name in ["Definite", "Tentative", "Prospect"]:
            calendars[folder.Name] = folder

    if not calendars:
        logging.error("Could not find required calendars.")
        exit(1)

    action = (
        input("Would you like to 'Add' or 'Remove' events? (Enter 'Add' or 'Remove'): ")
        .strip()
        .lower()
    )

    if action == "add":
        logging.info("Adding events to calendars.")
        add_to_calendar_with_earliest_start_end(df, calendars)
    elif action == "remove":
        logging.info("Removing events from calendars.")
        remove_from_calendar(df, calendars, outlook)
    else:
        logging.error("Invalid action choice.")
        print("Invalid choice. Please enter 'Add' or 'Remove'.")


# Assignment 12 Analysis -- Emily
'''
	-pull out spike and y-layers in image
	-(spike) look at distributions of synapse density in the spike across x,y,z
	-make old plots prettier/writeup this weekend
	-(spike) in old hist graphs, make bin size a function of sample size
	-extract image(s) in dataset of y-layers
		-talk about it, analyze spatial distribution
	-heat maps/hex plots of y-layers, how is density distributed across them?
'''

# Setup
from mpl_toolkits.mplot3d import axes3d
import matplotlib.pyplot as plt
#%matplotlib inline
import numpy as np
import urllib2
import scipy.stats as stats

np.set_printoptions(precision=3, suppress=True)
url = ('https://raw.githubusercontent.com/Upward-Spiral-Science'
       '/data/master/syn-density/output.csv')
data = urllib2.urlopen(url)
csv = np.genfromtxt(data, delimiter=",")[1:] # don't want first row (labels)

# chopping data based on thresholds on x and y coordinates
x_bounds = (409, 3529)
y_bounds = (1564, 3124)

def check_in_bounds(row, x_bounds, y_bounds):
    if row[0] < x_bounds[0] or row[0] > x_bounds[1]:
        return False
    if row[1] < y_bounds[0] or row[1] > y_bounds[1]:
        return False
    if row[3] == 0:
        return False

    return True

indices_in_bound, = np.where(np.apply_along_axis(check_in_bounds, 1, csv,
                                                 x_bounds, y_bounds))
data_thresholded = csv[indices_in_bound]
n = data_thresholded.shape[0]

def synapses_over_unmasked(row):
    s = (row[4]/row[3])*(64**3)
    return [row[0], row[1], row[2], s]

syn_unmasked = np.apply_along_axis(synapses_over_unmasked, 1, data_thresholded)
syn_normalized = syn_unmasked

# Extract images in dataset of y-layers
from image_scraping_jay import get_image_url
from PIL import Image
from cStringIO import StringIO
import requests
import urllib
import io

'''
y_bounds = [(1564,1837), (1837,2071), (2071,2305), (2305,2539), (2539,3124)]
for _, bounds in enumerate(y_bounds):
	new_im = Image.new('RGB',(1000,1000))
	temp = data_thresholded[:,1]
	temp = temp[np.logical_and(temp>=bounds[0], temp<bounds[1])]
	ys = np.unique(temp)
	i = 0
'''

def get_image(xrange,yrange,xs,ys):
	spacing = 100
	new_im = Image.new('RGB',(spacing*(10+len(xs[xrange[0]:xrange[1]])),spacing*(10+len(ys[yrange[0]:yrange[1]]))))
	i = 0
	for y in ys[yrange[0]:yrange[1]]:
		j = 0
		for x in xs[xrange[0]:xrange[1]]:
			z = 1054
			im_url = get_image_url(x,y,z,1)
			opener = urllib2.build_opener()
			opener.addheaders = [('User-agent', 'Mozilla/5.0')]
			response = opener.open(im_url)
			img_file = StringIO(response.read())
			im = Image.open(img_file)
			#size =  (int(np.floor(1000/len(xs))),int(np.floor(500/len(ys))))
			#size =  (int(4000),int(np.floor(4000/len(ys))))
			size = (spacing,spacing)
			im.thumbnail(size, Image.ANTIALIAS)
			new_im.paste(im,(j,i))
			j += spacing + 10
		i += spacing + 10
	new_im.save('new_im'+str(i)+'.bmp')
	new_im.show()

xs = np.unique(data_thresholded[:,0])
ys = np.unique(data_thresholded[:,1])
get_image((0,1),(0,len(ys)-1),xs,ys)


#	File Created: 10/15/2020
#	Author: Jack Connor <jconnor@baaqmd.gov>

from postman_problems.tests.utils import create_mock_csv_from_dataframe
from graph import create_rpp_edgelist
import initialize_lib as il

def InnerAndOuterToEdgeListFile(directory = 'C:\\Users\jconnor\OneDrive - Bay Area Air Quality Management District\Documents\Projects\Maps\Route Testing\\',
        InnerFileName = 'Haight Inner Polygon.csv',
        OuterFileName = 'Haight Outer Polygon.csv', turn_weight_coefficient=1):
    """
    Obtains street network data and generates edgelist, graphs, and other parameters to be used in rpp solver and visualization
    Args:
        directory (str): directory to find polygon files
        InnerFileName (str): inner polygon filename
        OuterFileName (str): outer polygon filename
    Returns:
        START_NODE (int): node at which to start the rpp circuit. Here is is set to the western most node
        req_comp_g (NetworkX DiGraph): graph of required streets
        complete_g (NetworkX DiGraph): graph of all streets within outer polygon
        elfn (csv object): csv dataframe containing rpp edgelist
        GranularConnector_EdgeList (list): list of all granular connector edges

    """
    inner_g, outer_g = il.create_inner_and_outer_graph(directory, InnerFileName, OuterFileName)

    inner_g = il.create_turn_weight_edge_attr(inner_g, normalization_coefficient=turn_weight_coefficient)
    outer_g = il.create_turn_weight_edge_attr(outer_g, normalization_coefficient=turn_weight_coefficient)

    inner_g = il.determine_and_remove_extraneous_reverse_edges(inner_g)

    il.simple_network_plotter(inner_g)

    inner_g_strongly_connected = il.strong_connector(inner_g, outer_g)

    il.strongly_connected_graph_plotter(inner_g_strongly_connected)

    START_NODE = il.pick_western_most_start_node(inner_g_strongly_connected)

    inner_g_strongly_connected, GranularConnector_EdgeList = il.remove_granular_connector_edges(inner_g_strongly_connected)

    outer_g = il.add_distance_keys_to_graph(outer_g)

    dfrpp = create_rpp_edgelist(g_strongly_connected = inner_g_strongly_connected, graph_full = outer_g, edge_weight = il.turn_weight_function_distance, max_distance = 3200)

    elfn = create_mock_csv_from_dataframe(dfrpp)

    req_comp_g = inner_g_strongly_connected.copy()

    complete_g = outer_g.copy()

    return START_NODE, req_comp_g, complete_g, elfn, GranularConnector_EdgeList


"""Test huggingface embeddings."""

from langchain_community.embeddings.huggingface import (
    HuggingFaceEmbeddings,
    HuggingFaceInstructEmbeddings,
)

def test_huggingface_embedding_documents() -> None:
    """Test huggingface embeddings."""
    documents = ["foo bar"]
    embedding = HuggingFaceEmbeddings()
    output = embedding.embed_documents(documents)
    assert len(output) == 1
    assert len(output[0]) == 768

def test_huggingface_embedding_query() -> None:
    """Test huggingface embeddings."""
    document = "foo bar"
    embedding = HuggingFaceEmbeddings(encode_kwargs={"batch_size": 16})
    output = embedding.embed_query(document)
    assert len(output) == 768

def test_huggingface_instructor_embedding_documents() -> None:
    """Test huggingface embeddings."""
    documents = ["foo bar"]
    model_name = "hkunlp/instructor-base"
    embedding = HuggingFaceInstructEmbeddings(model_name=model_name)
    output = embedding.embed_documents(documents)
    assert len(output) == 1
    assert len(output[0]) == 768

def test_huggingface_instructor_embedding_query() -> None:
    """Test huggingface embeddings."""
    query = "foo bar"
    model_name = "hkunlp/instructor-base"
    embedding = HuggingFaceInstructEmbeddings(model_name=model_name)
    output = embedding.embed_query(query)
    assert len(output) == 768

def test_huggingface_instructor_embedding_normalize() -> None:
    """Test huggingface embeddings."""
    query = "foo bar"
    model_name = "hkunlp/instructor-base"
    encode_kwargs = {"normalize_embeddings": True}
    embedding = HuggingFaceInstructEmbeddings(
        model_name=model_name, encode_kwargs=encode_kwargs
    )
    output = embedding.embed_query(query)
    assert len(output) == 768
    eps = 1e-5
    norm = sum([o**2 for o in output])
    assert abs(1 - norm) <= eps


import numpy as np
from Environment import Environment, play
from Util import _printoptions
from itertools import product

class FrozenLake(Environment):
    def __init__(self, lake, slip, max_steps, seed=None):
        """
        lake: A matrix that represents the lake. For example:
         lake =  [['&', '.', '.', '.'],
                  ['.', '#', '.', '#'],
                  ['.', '.', '.', '#'],
                  ['#', '.', '.', '$']]
        slip: The probability that the agent will slip
        max_steps: The maximum number of time steps in an episode
        seed: A seed to control the random number generator (optional)
        """

        # start (&), frozen (.), hole (#), goal ($)
        self.lake = np.array(lake)
        self.lake_flat = self.lake.reshape(-1)

        self.slip = slip

        n_states = self.lake.size + 1
        n_actions = 4

        pi = np.zeros(n_states, dtype=float)
        pi[np.where(self.lake_flat == '&')[0]] = 1.0

        self.absorbing_state = n_states - 1

        # TODO:
        # Call parent constructor
        Environment.__init__(self, n_states, n_actions, max_steps, pi, seed)

        # Up, left, down, right (corresponding to w, a, s, d)
        self.actions = [(-1, 0), (0, -1), (1, 0), (0, 1)]

        # Matrix containing rewards for TAKING AN ACTION at a state
        self.reward_map = np.zeros(self.lake.shape, dtype=np.float)
        # Set goal state to 1
        self.reward_map[np.where(self.lake == '$')] = 1

        # Matrix indicating where the absorbing states are (holes & goal states are 1, others are 0)
        self.abs_states = np.zeros(self.lake.shape, dtype=np.float)
        # Set goal state to 1
        self.abs_states[np.where(self.lake == '$')] = 1
        self.abs_states[np.where(self.lake == '#')] = 1

        # Helpers for conversions from indices to states (coordinates) and states (coordinates) to indices
        self.state_idx_to_coords = list(product(range(self.reward_map.shape[0]), range(self.reward_map.shape[1])))
        self.coords_to_state_idx = {s: i for (i, s) in enumerate(self.state_idx_to_coords)}

        # Precompute probabilities for transitions
        # self.probabilities = np.zeros((self.n_states, self.n_states, self.n_actions))
        self.probabilities = {state: {action: [] for action in range(n_actions)} for state in range(n_states)}

        def increment(row, col, action):
            """
            Helper function for generating transition probability matrix.
            Makes sure that our agents don't leave the playing field.
            :param row: Current row
            :param col: Current column
            :param action: Incoming action
            :return: New row and column coordinates
            """
            # Boundary checks to make sure that agents don't leave the field
            if action == 0:  # up
                row = max(row - 1, 0)
            elif action == 1:  # left
                col = max(col - 1, 0)
            elif action == 2:  # down
                row = min(row + 1, self.lake.shape[0] - 1)
            elif action == 3:
                col = min(col + 1, self.lake.shape[1] - 1)
            return row, col

        def update_prob_matrix(row, col, action):
            """
            Helper function for generating transition probability matrix.
            Takes a current position (row, col) and an action and returns
            the new state, the reward for the state and whether the agent is done (won or fell in hole)
            :param row:
            :param col:
            :param action:
            :return:
            """
            # Get next field coordinates with boundary checks
            new_row, new_col = increment(row, col, action)
            # Convert coordinates to state ID
            new_state = self.coords_to_state_idx[(new_row, new_col)]
            # Get new coordinate type
            f_type = self.lake[row][column]
            # Check whether we reached the goal or fell in hole
            done = f_type == '$' or f_type == '#'
            # Reward = 1 if agent is at goal, 0 otherwise
            reward = float(f_type == '$')
            return new_state, reward, done

        # Generate transition probabilities
        # Adapted from the openai gym implementation
        # https://gym.openai.com/envs/FrozenLake-v0/
        # Go through all fields and
        for row in range(self.lake.shape[0]):
            for column in range(self.lake.shape[1]):
                # Get state index from coordinates
                state_idx = self.coords_to_state_idx[(row, column)]
                # Go through all actions (up, left, down, right)
                for action in range(n_actions):
                    # Get transition probabilities for current state and action
                    current_list = self.probabilities[state_idx][action]
                    # Check if field is goal or hole
                    field_type = self.lake[row][column]
                    # If goal or hole, make inescapable (probability 1.0)
                    if field_type == '$':  # goal => reward 1, done = True
                        current_list.append((1.0, state_idx, 1.0, True))
                    elif field_type == '#':  # hole => reward 0, done = True
                        current_list.append((1.0, state_idx, 0.0, True))
                    # Otherwise check where we can go from here:
                    else:
                        # Add probabilities for successful action and slips
                        for b in range(n_actions):
                            if b == action:
                                # Successful action
                                # The asterisk ('*') unpacks the return value of the update_prob_matrix() function
                                # The probability for a successful action is 1 minus the slip probability
                                # for each remaining move
                                # Note: For this to work the slip probability has to be in the range [0...1]
                                current_list.append(
                                    (1.0 - (n_actions - 1.0) * self.slip, *update_prob_matrix(row, column, b))
                                )
                            else:
                                # Slip :(
                                current_list.append((self.slip, *update_prob_matrix(row, column, b)))

    def step(self, action):
        # This call updates self.step as well
        state, reward, done = Environment.step(self, action)

        # Check whether agent is done
        done = (state == self.absorbing_state) or done

        return state, reward, done

    def p(self, next_state, state, action):
        # Initialise transition probability
        transition_p = 0.0
        # Get possible moves for the state and action (successful move + slips)
        possible_moves = self.probabilities[state][action]
        for proba, next_s, reward, done in possible_moves:
            if next_state == next_s:
                transition_p += proba

        return transition_p

    def r(self, next_state_idx, state_idx, action):
        possible_actions = self.probabilities[state_idx][action]
        for proba, next_state, reward, done in possible_actions:
            if next_state == next_state_idx:
                return reward

    def render(self, policy=None, value=None):
        if policy is None:
            lake = np.array(self.lake_flat)

            if self.state < self.absorbing_state:
                lake[self.state] = '@'

            print(lake.reshape(self.lake.shape))
        else:
            # UTF-8 arrows look nicer, but cannot be used in LaTeX
            # https://www.w3schools.com/charsets/ref_utf_arrows.asp
            actions = ['^', '<', '_', '>']

            print('Lake:')
            print(self.lake)

            print('Policy:')
            policy = np.array([actions[a] for a in policy[:-1]])
            print(policy.reshape(self.lake.shape))

            print('Value:')
            with _printoptions(precision=3, suppress=True):
                print(value[:-1].reshape(self.lake.shape))

if __name__ == "__main__":
    seed = 0

    # Small lake
    lake = [['&', '.', '.', '.'],
            ['.', '#', '.', '#'],
            ['.', '.', '.', '#'],
            ['#', '.', '.', '$']]

    env = FrozenLake(lake, slip=0.1, max_steps=16, seed=seed)
    play(env)


from queue import PriorityQueue
class Solution:
    def findKthLargest(self, nums, k):
        pq = PriorityQueue()
        pq_size = 0
        for i in range(len(nums)):
            if pq_size < k:
                pq_size += 1
                pq.put((nums[i], i))
            else:
                pq.put((nums[i], i))
                pq.get()
        return pq.get()[0]

# s = Solution()
# print(s.findKthLargest([3,2,1,5,6,4], 2))

215.数组中的第K个最大元素.py


# filter() FUNKSIYA
# juft sonlarni qaytaruvchi funksiya
import random as r
sonlar = r.sample(range(100),10)
def juftmi(x):
    """x juft bo'lsa True, aks holda, False qaytramiz"""
    return x%2==0

juft_sonlar = list(filter(juftmi,sonlar))
print(sonlar)
print(juft_sonlar)

# lambda YORDAMIDA YOZIB KO'RAMIZ
import random as r
sonlar = r.sample(range(100),10)
juft_sonlar = list(filter(lambda son: son%2==0,sonlar))
print(sonlar)
print(juft_sonlar)

# filter() yordamida matnlarni saralaymiz
mevalar = ['olma','anor','anjir',"o'rik",'qovun','banan']
mevB = list(filter(lambda meva:meva.startswith('a'),mevalar))
print(mevB)

# Quyidagi dastur mevalar ro'yxatidan nomi 4
# va undan kam harfdan iborat mevalarni saralab oladi
mevalar = ['olma','anor','anjir',"o'rik",'qovun','banan']
mevalar2 = list(filter(lambda meva:len(meva)<=4, mevalar))
print(mevalar2)

# boshlanish va oxirigi harfi bo'yicha so'zlarni topadigan funksiya
mevalar = ['olma','anor','anjir',"o'rik",'qovun','banan']
mevalar2 = list(filter(lambda meva:(meva.startswith('a') and \
                         meva.endswith('r')), mevalar))
print(mevalar2)


# -*- coding: utf-8 -*-
"""
Created on Fri Apr 15 08:42:27 2022

@author: Admin
"""

from Analytics import eurostats_loader
import pandas as pd
import numpy as np
from statsmodels.tsa.deterministic import CalendarFourier, DeterministicProcess
from scipy.signal import periodogram
from sklearn.linear_model  import LinearRegression
from sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor
import matplotlib as mpl
import matplotlib.pyplot as plt

mpl.rcParams['axes.prop_cycle'] = mpl.cycler(color=["#0645ad", "#800020"])

#%% Create Data
Weekly_Deaths = eurostats_loader()
Weekly_Deaths = Weekly_Deaths[Weekly_Deaths.index.year >= 2008]

Weekly_Deaths_train = Weekly_Deaths[Weekly_Deaths.index.year < 2019]
Weekly_Deaths_val = Weekly_Deaths[Weekly_Deaths.index.year == 2019]
Weekly_Deaths_test = Weekly_Deaths[Weekly_Deaths.index.year > 2019]

#%% Periodogram of Series
fs = pd.Timedelta("1Y") / pd.Timedelta("1W")
f, Pxx = periodogram(Weekly_Deaths_train.Value, fs=fs, detrend="linear", scaling="spectrum")
# The Power Spectrum Pxx indicates the share of each frequency of the variance of the Time Series.
print(np.sum(Pxx) == np.var(Weekly_Deaths_train.Value))
plt.plot(f, Pxx)
plt.xlabel('Frequency [Year]')
plt.ylabel('Power Spectrum [V**2]')
plt.show()

#%% Modeling of Time Series

fourierA = CalendarFourier(freq="A", order=3)

y_train = Weekly_Deaths_train.Value

dp = DeterministicProcess(index=y_train.index.to_period("W"),
                          constant=True,               # dummy feature for bias (y-intercept)
                          order=1,                     # trend (order 1 means linear)
                          additional_terms=[fourierA],) # annual seasonality (fourier)
X_train = dp.in_sample()

val_index = Weekly_Deaths_val.index
X_val = dp.out_of_sample(len(val_index), val_index)

#%% Fit Models and Plot Results.

Models = [LinearRegression(fit_intercept=False),
          GradientBoostingRegressor(max_depth=1)]

num_models = len(Models)
fig, axes = plt.subplots(num_models,2,layout="constrained",sharex="col",figsize = (15,8))

for i,model in enumerate(Models):

    model.fit(X_train,y_train)
    y_pred = pd.Series(model.predict(X_train).reshape(-1), index=y_train.index)
    y_val = pd.Series(model.predict(X_val).reshape(-1), index=val_index)

    mse = np.mean(np.sqrt((y_val-Weekly_Deaths_val.Value)**2))

    ax_train = axes[i,0]
    #ax_train.set_title(type(model).__name__)
    y_pred.plot(ax=ax_train)
    Weekly_Deaths_train.Value.plot(ax=ax_train)

    ax_val = axes[i,1]
    ax_val.axes.annotate(f"MSE: {mse:.2f}", xy=(0.8,0.9), xycoords='axes fraction', fontsize=10)
    y_val.plot(ax=ax_val)
    Weekly_Deaths_val.Value.plot(ax=ax_val)

#%% Calculate Excess Deaths with best model.

y_comb_train = Weekly_Deaths[Weekly_Deaths.index.year < 2020]

dp = DeterministicProcess(index=y_comb_train.index.to_period("W"),
                          constant=True,               # dummy feature for bias (y-intercept)
                          order=1,                     # trend (order 1 means linear)
                          additional_terms=[fourierA],  # annual seasonality (fourier)
                          drop=True,)                  # drop terms to avoid collinearity

X_comb_train = dp.in_sample()

model = GradientBoostingRegressor(max_depth=1)
model.fit(X_comb_train, y_comb_train)
y_comb_pred = pd.Series(model.predict(X_comb_train).reshape(-1), index=y_comb_train.index)

test_index = Weekly_Deaths_test.index
X_test = dp.out_of_sample(len(test_index), test_index)
y_test = pd.Series(model.predict(X_test).reshape(-1), index=test_index)
excess_deaths = np.sum(Weekly_Deaths_test.Value-y_test)

fig_test, ax_test = plt.subplots()
ax_test.axes.annotate(f"Estimated Excess Deaths: {excess_deaths:.0f}", xy=(0.55,0.9), xycoords='axes fraction', fontsize=8)
y_test.plot(ax=ax_test)
Weekly_Deaths_test.Value.plot(ax=ax_test)


n, k = map(int, input().split(' '))
coin = []
for i in range(n):
    coin.append(int(input()))

# 파이썬 reverse
# list[::-1] - 리스트, 튜플, 문자열, 딕셔너리X
# reverse() - 리스트, 튜플X, 문자열X, 딕셔너리X, 반환 None
# reversed() - 리스트, 튜플, 문자열, 딕셔너리X, 반환 O (다만 인자로 넘겨준 자료구조가 넘어오는 것은 아님)
coin.reverse()
# print(f'coin.reverse() : {coin}')
result = 0

for num in coin:
    div, mod = divmod(k, num)
    # print(f'div : {div}, mod : {mod}')
    result += div
    k = mod

print(result)


# coding:utf-8
import torch

from transformers import BartTokenizer, BartForConditionalGeneration, BartConfig
hf_config = BartConfig.from_pretrained('facebook/bart-large')
hf_config.force_bos_token_to_be_generated = True
tok = BartTokenizer.from_pretrained("facebook/bart-large")
hf_model = BartForConditionalGeneration.from_pretrained('facebook/bart-large', config=hf_config)

# fsq bart base
# from fairseq.models.bart import BARTModel
# bart = BARTModel.from_pretrained('/nfs/Desktop/data/pretrained-model/bart.large', checkpoint_file='model.pt')

# mask_token_id = bart.task.source_dictionary.indices["<mask>"]
# mask_token_weight_fairseq = bart.model.encoder.embed_tokens.weight[mask_token_id].detach()

# mask_token_id_hf = tok.mask_token_id
# mask_token_weight_hf = hf_model.model.encoder.embed_tokens.weight[mask_token_id_hf].detach()

# print((mask_token_weight_hf - mask_token_weight_fairseq).abs().max())

# input_string = " My dog is <mask>"
# labels_string = " My dog is cute"
input_string = "Rudolph Agnew <mask> 55 years old and former chairman of <mask> ated <mask> <mask> PLC , was named a nonexecutive director of this British industrial conglomerate ."
labels_string = "Rudolph Agnew , 55 years old and former chairman of Consolidated Gold Fields PLC , was named a nonexecutive director of this British industrial conglomerate ."

input_ids = tok(input_string, return_tensors="pt").input_ids
labels = tok(labels_string, return_tensors="pt").input_ids
# print("labels", labels)
# print(tok.batch_decode(labels.tolist()))
decoder_input_ids = hf_model.prepare_decoder_input_ids_from_labels(labels)
print("input_ids", input_ids)
print(tok.batch_decode(input_ids.tolist()))
print("dec_inp_ids", decoder_input_ids)
print(tok.batch_decode(decoder_input_ids.tolist()))
print("labels", labels)
print(tok.batch_decode(labels.tolist()))
loss = hf_model(input_ids=input_ids, decoder_input_ids=decoder_input_ids, labels=labels)[0]
print("Loss1", loss)

# from transformers import BartTokenizer, BartForConditionalGeneration
# tokenizer = BartTokenizer.from_pretrained('facebook/bart-large')
# model = BartForConditionalGeneration.from_pretrained('facebook/bart-large')

# TXT = "My friends are <mask> but they eat too many carbs."
# input_ids = tokenizer([TXT], return_tensors='pt')['input_ids']
# print(input_ids)
# logits = model(input_ids).logits

# masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero().item()
# probs = logits[0, masked_index].softmax(dim=0)
# values, predictions = probs.topk(5)
# print(tokenizer.decode(predictions).split())


from gensim.models import Word2Vec
import numpy as np

model = Word2Vec.load('Word2Vec_1.model') #load model

#print(model.wv.vocab.keys())
#print(model.wv.similar_by_word("เอเลี่ยน"))

max_value = 0
min_value = 0
for i in model.wv.vocab.keys():
    max_value = max(np.max(model.wv.get_vector(i)),max_value)
    min_value = min(np.min(model.wv.get_vector(i)),min_value)
    print(min_value,"-",max_value)


from gobang_board import *
from random import randint
from math import sqrt
from model import GoBangNet
from gobang_board import get_symmetries
import numpy as np
import time

c_puct = 1
DEFAULT_SEARCH_COUNT = 800

class TreeSearch:
    def __init__(self, net):
        self.net = net

        self.Qsa = dict()  # Q[(s, a)]
        self.Nsa = dict()  # N[(s, a)]
        self.Ns = dict()
        self.Ps = dict()  # P[s] is a vector, P[s][a]

        self.Es = dict()
        self.Rs = dict()
        self.Vs = dict()

        self.root = GoBangBoard()

    def search(self, s):
        board = s.get_str_representation()
        if board not in self.Es:
            self.Es[board] = s.is_game_ended()
        if self.Es[board]:
            if board not in self.Rs:
                self.Rs[board] = s.get_reward()
            return self.Rs[board]

        if board in self.Ns:  # not a leaf node
            valid_moves = self.Vs[board]
            best_confidence = -float('inf')
            best_a = None

            for a in range(225):
                if valid_moves[a]:
                    if (board, a) in self.Qsa:
                        u = self.Qsa[(board, a)] + c_puct * self.Ps[board][a] * sqrt(self.Ns[board]) / (
                                1 + self.Nsa[(board, a)])
                    else:  # (s, a) not in Qsa and Nsa
                        u = c_puct * self.Ps[board][a] * sqrt(self.Ns[board] + 1e-7)
                        # Qsa[(s, a)] = 0, to prevent all zeros

                    # print('u:', u)
                    if u > best_confidence:
                        best_confidence = u
                        best_a = a

            next_s = s.move(best_a)
            v = self.search(next_s)

            v *= -1  # changes into win rate of current player

            self.Ns[board] += 1
            if (board, best_a) in self.Qsa:
                self.Qsa[(board, best_a)] = (self.Nsa[(board, best_a)] * self.Qsa[(board, best_a)] + v) / (
                        self.Nsa[(board, best_a)] + 1)
                self.Nsa[(board, best_a)] += 1
            else:
                self.Qsa[(board, best_a)] = v
                self.Nsa[(board, best_a)] = 1

            return v
        else:  # is a leaf node
            v = self.expand(s)
            return v

    def expand(self, s):
        board = s.get_str_representation()
        p, v = self.net.predict(s)
        p = p.numpy()
        valid_moves = s.get_valid_actions()
        self.Vs[board] = valid_moves
        p = p * valid_moves
        if p.sum() == 0:
            print('warning! p is 0! changed to uniform distribution')
            p = valid_moves
        p /= p.sum()
        self.Ps[board] = p
        self.Ns[board] = 0
        return v

    def get_pi(self, s, tau):
        board = s.get_str_representation()
        counts = np.array([self.Nsa[(board, a)] if (board, a) in self.Nsa else 0 for a in range(225)])
        # print('counts:', counts)
        # counts = counts * self.Vs[s]
        if np.max(counts) == 0:
            print('warning! max N is 0!')
        if tau == 0:
            bestAs = np.array(np.argwhere(counts == np.max(counts))).flatten()
            bestA = np.random.choice(bestAs)
            result = [0] * len(counts)
            result[bestA] = 1
            return result
        else:
            counts = counts ** (1 / tau)
            result = counts / counts.sum()
            return result

    def search_from_root(self, search_count=DEFAULT_SEARCH_COUNT):
        for i in range(search_count):
            self.search(self.root)

    def get_pi_and_get_move(self, tau, target=None):
        if target is None:
            target = self.root
        pi_distribution = self.get_pi(target, tau)
        move = np.random.choice(225, p=pi_distribution)
        return pi_distribution, move

    def progress(self, move):
        self.root = self.root.move(move)
        # board = self.root.get_str_representation()
        self.add_noise(self.root, 0.3)
        # self.Ps[board] = 0.1 * np.random.dirichlet(np.ones(225)) + self.Ps[board]

    def add_noise(self, s, noise_level=0.3):
        board = s.get_str_representation()
        if board not in self.Ps:
            print('note! add noise to un-expanded node')
            self.expand(s)
        self.Ps[board] = noise_level * np.random.dirichlet(np.ones(225)) + (1-noise_level) * self.Ps[board]

def generate_single_game(net, print_every_step=False, sim_per_step=200):
    t1 = time.time()
    data = []
    tree = TreeSearch(net)
    move_count = 0
    while not tree.root.is_game_ended():
        t2 = time.time()
        tree.search_from_root(sim_per_step)
        pi_distribution, move = tree.get_pi_and_get_move(tau=0 if move_count > 5 else 1)
        new_data = get_symmetries(tree.root.get_network_input()[0], pi_distribution)
        data.append(new_data)
        if tree.Nsa[(tree.root.get_str_representation(), move)] == 0:
            print('note: selecting a move with 0 visit')
            board = tree.root.get_str_representation()
            print(np.array([tree.Nsa[(board, a)] if (board, a) in tree.Nsa else 0 for a in range(225)]))
            print(move)
        tree.progress(move)
        move_count += 1
        if print_every_step:
            tree.root.print_board()
        print('time per step:', time.time() - t2)

    print(tree.root.get_reward())
    tree.root.print_board()
    print('winner:', another_player(tree.root.next_player))

    r = tree.root.get_reward()
    r *= -1
    all_data = []
    print('last move reward:', r)
    for i in range(len(data) - 1, -1, -1):
        for sym in range(8):
            data[i][sym].append(r)
        r *= -1
        all_data.extend(data[i])

    print('generate single game time (s):', time.time() - t1)
    return all_data


# объявление функции
def is_one_away(word1, word2):
    if len([i for i in range(len(word1)) if word1[i] != word2[i] ]) ==1 and len(word1)==len(word2):
        print([c for c in word1 if c not in word2])
        return True
    else:
        print([c for c in word1 if c not in word2])
        return False

# считываем данные
txt1 = 'aab'
txt2 = 'abb'

# вызываем функцию
#print(is_one_away(txt1, txt2))

# объявление функции

def is_palindrome(text):
    begin, end = 0, len(text)-1
    while(begin < end):
        while (not text[begin].lower().isalpha()):
            begin +=1
        while(not text[end].lower().isalpha()):
            end -= 1
        print(text[begin].lower() + '-' + str(begin) + '-' + text[end].lower()+ '-' + str(end))
        if text[begin].lower() != text[end].lower():
            print('here')
            return False
        begin +=1
        end -= 1
    return True

def is_palindrome1(text):
    temp = [c.lower() for c in text if c.isalpha()]
    print(*temp, sep ='')
    print(*temp[::-1], sep ='')
    if temp==temp[::-1]:
        return True
    else:
        return False

def is_correct_bracket(text):
    l = []
    r = []
    for i in range(len(text)):
        if text[i] == '(':
            l.append(i)
        elif text[i] == ')':
            r.append(i)
    if len(l) != len(r):
        return False
    else:
        for i in range(len(l)):
            if l[i] > r[i]:
                return False
        return True
def is_correct_bracket2(text):
    while '()' in text:
        text = text.replace('()', '')
        print(text)

    if not text:
        return True
    else:
        return False

# объявление функции
import re
def convert_to_python_case(text):
    data = [c.lower() for c in re.findall('[A-Z][^A-Z]*', text)]
    res = '_'.join(data)
    print(data)
    print(res)

def convert_to_python_case2(text):
    data = []
    data.extend(text)
    print(data)
    for i in range (len(data)):
        if data[i] in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ':
            data[i]=data[i].lower()
            data.insert(i,'_')
    del data[0]
    return ( ''.join(data))

def convert_to_python_case3(text):
    res=''
    for char in text:
        if char.isupper():
            res += '_'
        res += char.lower()
    return(res[1:])

def convert_to_python_case4(text):
    return(''.join(['_' + char.lower() if char.isupper() else char for char in text]).lstrip('_'))

# считываем данные

txt = 'ThisIsCamelCased'

# вызываем функцию
print(convert_to_python_case4(txt))


def add():
    while True:
        a = input('Введите имя, или стоп для завершения')
        b=int(input('Введите число предметов'))
        mas=[]
        count=0
        for i in range(b):
            c=int(input('Введите сумму по предмету'))
            mas.append(c)
            count+=1
        sum_1=sum(mas)
        if sum_1>80:
            print('Наградить дипломом.')
        elif sum_1>50 and sum_1<=80:
            print('Наградить похвальной грамотой.')
        else:
            print('Выдать грамоту об участии.')
        if a=='stop':
            print('Прием закончен')
            break
add()


# Union-Find
n, m = map(int, input().split()) # 사람 수, 파티 수
true_lst = list(map(int, input().split()))

parent = [i for i in range(n+1)]
def find(x): # 부모 선정
    if parent[x] == x:
        return x
    p = find(parent[x])
    parent[x] = p
    return p

def union(x, y):
    x, y = find(x), find(y)
    if x != y:
        parent[y] = parent[x]
    pass

for i in range(true_lst[0]):
    union(true_lst[1], true_lst[i + 1]) # 첫 번째 사람을 부모로 선정

party = []
for _ in range(m):
    party.append(list(map(int, input().split())))
    for current in range(party[-1][0]-1):
        union(party[-1][current+1], party[-1][current+2])

answer = 0
for i in party:
    for current in range(i[0]):
        # 파티원들이 진실을 아는 사람의 부모와 동일 -> 해당 파티는 과장 x
        if (len(true_lst) > 1 and find(i[current+1]) == find(true_lst[1])):
            answer += 1
            break
print(m - answer)

# Set 활용
n, m = map(int, input().split())
true_list = set(map(int, input().split()[1:]))
party = []
for _ in range(m):
    party.append(set(map(int, input().split()[1:])))

answer = [1 for _ in range(m)]
for _ in range(m): # true_list가 중간중간에 변경되므로 파티 수만큼 돌려 완탐
    for i,v  in enumerate(party):
        if true_list & v: # 공통 원소가 있으면
            answer[i] = 0
            true_list |= v # 진실을 아는 사람과 마추친 사람은 거짓말을 할 수 없음

print(sum(answer))


from django.http import HttpResponse
from django.shortcuts import redirect, render
from base.models import Message, Room, Topic
from django.db.models import Q

# Custom Service object imports
# from base.services.RoomService import room_service_obj

# Custom Service imports
from base.services.RoomService import RoomService

# Create your views here.

def home(request):
    q = request.GET.get("q") if request.GET.get("q") is not None else ""
    rooms = Room.objects.filter(
        Q(topic__name__icontains=q) | Q(name__icontains=q) | Q(description__icontains=q)
    )
    page = request.GET.get("page", 1)
    rooms_obj = RoomService.pagination(request, page, rooms)
    rooms_count = Room.objects.count()
    topics = Topic.objects.all()[:5]
    all_messages = Message.objects.all()
    room_data = []

    for room in rooms_obj:
        participants_count = room.participants.count()
        room_data.append(
            {"room": room, "participants_count": participants_count}
        )

    context = {
        "topics": topics,
        "rooms_data": room_data,
        "rooms": rooms_obj,
        "rooms_count": rooms_count,
        "all_messages": all_messages,
    }

    return render(request, "base/home.html", context)


# Devuelve un booleano si se puede acceder al archivo seleccionado por el usuario
def archivoAccesible(ruta):
    accesible=True
    try:
        file = open(ruta, "r", encoding="utf-8")
        content = file.read()
        file.close()
    except FileNotFoundError:
        print("El archivo no se pudo encontrar")
        accesible=False
    except PermissionError:
        print("No se tienen los permisos necesarios para acceder al archivo")
        accesible=False
    except IOError:
        print("Ocurrió un error de E/S al acceder al archivo")
        accesible=False
    except Exception as e:
        print("Ocurrió un error inesperado:", e)
        accesible=False
    else:
        print("El archivo se leyó correctamente")

    return accesible

# Sirve para mostrar por consola el diccionario de las etiquetas más fácilmente
def mostrarDiccEtq(dicc):
    if dicc.keys():
        for etiqueta in dicc.keys():
            subtabla=dicc[etiqueta]
            print(f"La subtabla de la etiqueta {etiqueta} es:\n{subtabla}\n")
    else:
        print({})