import logging
import os
from pathlib import Path
import re
import sys
import time
from typing import Type, Any, Iterable, Optional, \
                   Tuple, List, Dict, Callable
import warnings

from bs4 import BeautifulSoup
from gensim.models import Word2Vec, KeyedVectors
from keras.callbacks import EarlyStopping
from keras.layers import Dense, Embedding, Flatten
from keras.layers.convolutional import Conv1D
from keras.layers.convolutional import MaxPooling1D
from keras.models import Sequential
from keras.optimizers import Adam, SGD
import nltk.corpus
import nltk.tokenize
from nltk.corpus import wordnet
from nltk.stem import WordNetLemmatizer
import numpy as np
import pandas as pd
# from scipy.sparse import csc_matrix
import sklearn
from sklearn.cluster import MiniBatchKMeans
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import MultinomialNB, BernoulliNB
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.metrics import roc_auc_score

# import cProfile

def _get_sklearn_version() -> Tuple[int, int, int]:
    """
    Returns the version of scikit-learn as a tuple.
    """
    (release, major, minor) = sklearn.__version__.split('.')
    return int(release), int(major), int(minor)

if _get_sklearn_version() >= (0, 19, 0):
    from sklearn.model_selection import StratifiedKFold
    from sklearn.model_selection import StratifiedShuffleSplit
else:  # We have an old version of sklearn...
    from sklearn.cross_validation import StratifiedKFold
    from sklearn.cross_validation import StratifiedShuffleSplit

def _get_current_file_dir() -> Path:
    """Returns the directory of the script."""
    try:
        return Path(os.path.realpath(__file__)).parent
    except(NameError):
        return Path(os.getcwd())

# Project root directory, i.e. the github repo directory.
_PROJECT_ROOT = _get_current_file_dir() / '..'

# Default configuration options.
# WARNING: Please, avoid changing it. Use a local `conf.py` in the project's
# root directory.
_DEFAULT_CONFIG = {
    # Options that specify where to read the data from.
    'in': {
        'labeled':   str(_PROJECT_ROOT / 'data' / 'labeledTrainData.tsv'),
        'unlabeled': str(_PROJECT_ROOT / 'data' / 'unlabeledTrainData.tsv'),
        'test':      str(_PROJECT_ROOT / 'data' / 'testData.tsv'),
        'clean':     str(_PROJECT_ROOT / 'data' / 'cleanReviews.tsv'),
    },
    # Options that specify where to write the results to.
    'out': {
        'result':       str(_PROJECT_ROOT / 'results' / 'Prediction.csv'),
        'wrong_result': str(_PROJECT_ROOT / 'results' / 'FalsePrediction.csv'),
    },
    # High-level algorithm specific options.
    'run': {
        # Type of the run, one of {'optimization', 'submission'}
        'type':             'submission',
        # How many splits to use in the StratifiedKFold
        'number_splits':    3,
        # When preprocessing the reviews, should we remove the stopwords?
        'remove_stopwords': True,
        # Should we cache the preprocessed reviews?
        'cache_clean':      True,
        # After the running the StratifiedKFold on the 90%, should we test the
        # result on the remaining 10?
        'test_10':          False,
        # Random seed used for the 90-10 split.
        'random':           54,
        # How many percent of the data should be left out for testing? I.e.
        # what is "10" in the 90-10 split.
        'alpha':            0.1,
    },
    # Type of the vectorizer, one of {'word2vec', 'bagofwords'}
    'vectorizer': 'word2vec',
    # Type of the classifier to use, one of
    # {'random-forest', 'logistic-regression', 'feed-forward', 'convolutional'}
    'classifier': 'convolutional',
    # Options specific to the bagofwords vectorizer.
    'bagofwords': {},
    # Options specific to the word2vec vectorizer.
    'word2vec': {
        # File name where to save/read the model to/from.
        'model_file': str(_PROJECT_ROOT / 'data'
                          / 'GoogleNews-vectors-negative300.bin'),
        # File name where to save/read the "compact" version to/from.
        'cache_file': str(_PROJECT_ROOT / 'data'
                          / 'GoogleNews-vectors-negative300.compact.bin'),
        # Retrain the model every time?
        'retrain':    False,
        # Averaging strategy to use, one of {'average', 'k-means', 'dummy'}
        # NOTE: Random Forest, Logistic Regression, and Feed Forward NN work
        # with 'average' and 'k-means'; Convolutional NN works __only__ with
        # 'dummy'.
        'strategy':   'dummy',
    },
    # Options specific to the random forest classifier.
    'random-forest': {
        'n_estimators': 1000,
        'n_jobs':       (-1),
        'max_depth':    10,
        'max_features': 'log2',
    },
    # Options specific to the logistic regression classifier.
    'logistic-regression': {
        # TODO: I might've messed this part up. Someone with a good
        # understanding of the required arguments, please check!
        #                                                Tom
        'penalty':           'l2',
        'dual':              True,
        'tol':               0.0001,
        'C':                 1,
        'fit_intercept':     True,
        'intercept_scaling': 1.0,
        'class_weight':      None,
        'random_state':      None,
    },
    # Options specific to Andre's convolutional NN classifier.
    'convolutional': {
        'n_units':      250,
        'n_filters':    90,
        'n_epochs':     4,
        'verbose':      2,
        'kernel_size':  6,
        'batch_size':   128,
        'use_bias':     True,
        'loss':         'binary_crossentropy',
        'optimizer':    Adam(lr=0.0021),
        'metrics':      ['accuracy'],
    },
    # Options specific to feed forward neural network classifier.
    'feed-forward': {
        'n_hidden_units1': 16,
        'n_hidden_units2': 10,
        'batch_size':      32,
        'n_epochs':        200,
        'loss':            'binary_crossentropy',
        'optimizer':       SGD(lr=0.01),
        'metrics':         ['accuracy'],
    },
    # Dummy entry for dummy averager :)
    'dummy': {},
    # Options specific to the "average" averaging strategy.
    'average': {},
    # Options specific to the "k-means" averaging strategy.
    'k-means': {
        'number_clusters_frac': 0.2,  # NOTE: This argument is required!
        'warn_on_missing': False,
        # 'max_iter':             100,
        # 'n_jobs':               4,
    },
}

# If you have a local conf.py which defines the configuration options, it will
# be used. Otherwise, this falls back to defaults.
# NOTE: See _DEFAULT_CONFIG for the format of the configuratio options.
try:
    from conf import conf
    print("Imported local config file.", file=sys.stderr)
except ImportError:
    conf = _DEFAULT_CONFIG
    print("Used the default config.", file=sys.stderr)

# TODO: Fix this.
# Turn off warnings about Beautiful Soup (Johanna has checked all of them
# manually)
warnings.filterwarnings("ignore", category=UserWarning, module='bs4')

def _read_data_from(path: str) -> pd.DataFrame:
    logging.info('Reading data from {!r}...'.format(path))
    return pd.read_csv(path, header=0, delimiter='\t', quoting=3)

class LemmatizationWithPOSTagger(object):
    def __init__(self):
        self.lemmatizer = nltk.WordNetLemmatizer()

    def get_wordnet_pos(self, treebank_tag):
        """
        Convert Part of Speech Tags to comply with the expectations of wordnet
        """
        if treebank_tag.startswith('J'):
            return wordnet.ADJ
        elif treebank_tag.startswith('V'):
            return wordnet.VERB
        elif treebank_tag.startswith('N'):
            return wordnet.NOUN
        elif treebank_tag.startswith('R'):
            return wordnet.ADV
        else:
            # As default pos in lemmatization is Noun
            return wordnet.NOUN

    def pos_tag(self, tokens):
        """
        Method to obtain POS-Tags

        """
        # find the pos tagginf for each tokens [('What', 'WP'), ('can', 'MD'), ('I', 'PRP') ....
        pos_tokens = [nltk.pos_tag(token) for token in tokens]

        # TODO: The following looks ugly...
        # lemmatization using pos tagg
        # convert into feature set of [('What', 'What', ['WP']), ('can', 'can', ['MD']), ... ie [original WORD, Lemmatized word, POS tag]
        pos_tokens = [ [(word, self.lemmatizer.lemmatize(word,self.get_wordnet_pos(pos_tag)), [pos_tag]) for (word,pos_tag) in pos] for pos in pos_tokens]
        return pos_tokens

class ReviewPreprocessor(object):
    """
    :py:class:`ReviewPreprocessor` is an utility class for processing raw HTML
    text into segments for further learning.
    """

    _stopwords = set(nltk.corpus.stopwords.words('english'))
    _tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')

    @staticmethod
    def _striphtml(text: str) -> str:
        text = re.sub(r'(www.|http[s]?:\/)(?:[a-zA-Z]|[0-9]|[$-_@.&+]'
                      r'|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+',
                      r'', text)
        text = BeautifulSoup(text, 'html.parser').get_text()
        return text

    @staticmethod
    def _2wordlist(text: str, remove_stopwords: bool = False) -> List[str]:
        text = re.sub('[^a-zA-Z]', ' ', text)
        words = text.lower().split()
        if remove_stopwords:
            words = [w for w in words if w not in
                     ReviewPreprocessor._stopwords]
        # TODO: This is ugly... Is it even correct?
        words = [words]
        lemma_pos_token = LemmatizationWithPOSTagger().pos_tag(words)
        words = [[x[1] for x in el] for el in lemma_pos_token]
        words = words[0]
        return words

    @staticmethod
    def review2wordlist(review: str, remove_stopwords: bool = False) \
            -> List[str]:
        """
        Given a review, parses it as HTML, removes all URLs and
        non-alphabetical characters, and optionally removes the stopwords. The
        review is then split into lowercase words and the resulting list is
        returned.

        :param str review:            The review as raw HTML.
        :param bool remove_stopwords: Whether to remove the stopwords.
        :return:                      Review split into words.
        :rtype:                       List[str]
        """
        return ReviewPreprocessor._2wordlist(
            ReviewPreprocessor._striphtml(review),
            remove_stopwords=remove_stopwords)

    @staticmethod
    def review2sentences(review: str, remove_stopwords: bool = False) \
            -> Iterable[List[str]]:
        """
        Given a review, splits it into sentences where each sentence
        is a list of words.

        :param str review:            The review as raw HTML.
        :param bool remove_stopwords: Whether to remove the stopwords.
        :return:                      Review split into sentences.
        :rtype:                       Iterable[List[str]].
        """
        raw_sentences = ReviewPreprocessor._tokenizer.tokenize(
            ReviewPreprocessor._striphtml(review))
        return map(
            lambda x: ReviewPreprocessor._2wordlist(x, remove_stopwords),
            filter(lambda x: x, raw_sentences))

# TODO: This function should probably also do the stemming, see
# https://github.com/mlp2018/BagofWords/issues/7.
def clean_up_reviews(reviews: Iterable[str],
                     remove_stopwords: bool = True,
                     clean_file: Optional[str] = None) -> np.ndarray:
    """
    Given an list of reviews, either loads pre-computed clean reviews from
    ``clean_file`` or applies :py:func:`ReviewPreprocessor.review2wordlist` to
    each review and converts it into an :py:class:`numpy.ndarray`. Also, if
    ``clean_file`` is not ``None`` clean reviews are saved to it.

    :param reviews:               Reviews to clean up.
    :type reviews:                Iterable[str]
    :param bool remove_stopwords: Whether to remove the stopwords.
    :param clean_file:            File where clean reviews are stored.
    :type clean_file:             Optional[str]
    :return:                      Iterable of clean reviews.
    :rtype:                       Iterable[str]
    """
    if clean_file is not None and Path(clean_file).exists():
        return _read_data_from(clean_file)['review'].values
    logging.info('Cleaning and parsing the reviews...')
    review = np.array(list(map(
        lambda x: ' '.join(
            ReviewPreprocessor.review2wordlist(x, remove_stopwords)),
        reviews)), object)
    # This assert is important to ensure we're doing things in a
    # memory-efficient way!
    assert review.dtype == np.object
    if clean_file is not None:
        logging.info('Saving clean data to file {!r}...'
                     .format(repr(clean_file)))
        pd.DataFrame(data={"review": review}) \
            .to_csv(clean_file, index=False, quoting=3)
    return review

def reviews2sentences(reviews: Iterable[str],
                      remove_stopwords: bool = True) -> Iterable[List[str]]:
    """
    Given a list of reviews, cleans them up, splits into sentences, and returns
    a list of *all* the sentences.

    :param reviews:               Reviews to clean up.
    :type reviews:                Iterable[str]
    :param bool remove_stopwords: Whether to remove the stopwords.
    :return:                      Iterable of sentences, where each sentence is
                                  a list of words.
    :rtype:                       Iterable[List[str]]
    """
    logging.info('Splittings reviews into sentences...')

    class R2SIter:
        def __init__(self, xs, remove):
            self.reviews = xs
            self.remove_stopwords = remove

        def __iter__(self):
            for review in self.reviews:
                for sentence in ReviewPreprocessor.review2sentences(
                        review, self.remove_stopwords):
                    yield sentence
    return R2SIter(reviews, remove_stopwords)

def submission_run(reviews: np.ndarray,
                   sentiments: np.ndarray,
                   test_reviews: np.ndarray,
                   ids: np.ndarray,
                   mk_vectorizer: Callable[[], Any],
                   mk_classifier: Callable[[], Any],
                   prediction_file: str) -> np.ndarray:
    """
    :param reviews:         Array of raw reviews texts.
    :type reviews:          'numpy.ndarray' of shape ``(N,)`` of type ``str``
    :param sentiments:      Array of review sentiments.
    :type sentiments:       'numpy.ndarray' of shape ``(N,)`` of type ``bool``
    :param test_reviews:    Array of test review texts.
    :type test_reviews:     'numpy.ndarray' of shape ``(N,)`` of type ``str``
    :param ids:             Array of review identifiers.
    :type ids:              'numpy.ndarray' of shape ``(N,)`` of type ``str``
    :param mk_vectorizer:   Factory function to create a new vectorizer.
    :type mk_vectorizer:    Callable[[], Vectorizer]
    :param mk_classifier:   Factory function to create a new classifier.
    :type mk_classifier:    Callable[[], Classifier]
    :param prediction_file: File where the predicted sentiments are saved to.
    :type prediction_file:  str
    """
    _, prediction = run_one_fold(
        (reviews, sentiments), test_reviews, mk_vectorizer(),
        mk_classifier())
    logging.info('Saving all predicted sentiments to {!r}...'
                 .format(prediction_file))
    pd.DataFrame(data={'id': ids, 'sentiment': prediction}) \
        .to_csv(prediction_file, index=False, quoting=3)

def bookkeeping(reviews: Type[np.ndarray],
                wrong_index: Type[np.ndarray],
                wrong_prediction_file: str):
    logging.info('Saving wrong predictions to {!r}...'
                 .format(wrong_prediction_file))
    pd.DataFrame(data={'review': reviews[wrong_index]}) \
        .to_csv(wrong_prediction_file, index=False, quoting=3, escapechar='\\')

def run_one_fold(train_data: Tuple[np.ndarray, np.ndarray],
                 test_data: Tuple[np.ndarray, np.ndarray],
                 vectorizer: Any, classifier: Any) \
        -> Tuple[float, np.ndarray]:
    """
    Given some data to train on and some data to test on, runs the whole
    feature extraction + classification procedure, computes the ROC AUC score
    of the predictions and returns it along with raw predictions.

    :param train_data: ``(array of reviews, array of sentiments)`` on which the
                       model will be trained.
    :param test_data:  ``(array of reviews, array of sentiments)`` on which the
                       accuracy of the model will be computed.
    :param vectorizer: Vectorizer to use.
    :param classifier: Classifier to use.
    :return:           ``(score, predictions)`` tuple.
    """
    score = None
    (train_reviews, train_labels) = train_data
    if isinstance(test_data, tuple):
        (test_reviews, test_labels) = test_data
    else:
        test_reviews = test_data
    logging.info('Transforming training data...')
    train_features = vectorizer.fit_transform(train_reviews)
    logging.info('Transforming test data...')
    test_features = vectorizer.transform(test_reviews)
    logging.info('Fitting...')
    classifier = classifier.fit(train_features, train_labels)
    logging.info('Predicting test labels...')
    prediction = classifier.predict(test_features)
    if isinstance(test_data, tuple):
        score = roc_auc_score(test_labels, prediction)
        logging.info('ROC AUC for this fold is {}.'.format(score))
    return score, prediction

def split_90_10(data: Tuple[np.ndarray, np.ndarray],
                alpha: float, seed: int = 42) \
        -> Tuple[Tuple[np.ndarray, np.ndarray],
                 Tuple[np.ndarray, np.ndarray]]:
    """
    Despite the very descriptive name this function does the ``1-alpha`` -
    ``alpha`` split rather than the ``90%`` - ``10%`` one.

    :param data:  The data to split.
    :param alpha: Percentage of data to use for testing.
    :param seed:  Random seed to use for splitting.
    :return: ``((reviews to train on, sentiments to train on),
                (reviews to test on, sentiments to test on))``.
    """
    assert alpha > 0 and alpha < 1
    reviews, labels = data
    train_index = None
    test_index = None
    if _get_sklearn_version() >= (0, 19, 0):
        train_index, test_index = StratifiedShuffleSplit(
            n_splits=1, test_size=alpha,
            random_state=seed).split(reviews, labels).__iter__().__next__()
    else:
        train_index, test_index = StratifiedShuffleSplit(
            labels, n_iter=1, test_size=alpha,
            random_state=seed).__iter__().__next__()
    x_train = reviews[train_index]
    x_test = reviews[test_index]
    y_train = labels[train_index]
    y_test = labels[test_index]
    return (x_train, y_train), (x_test, y_test)

def run_a_couple_of_folds(data: Tuple[np.ndarray, np.ndarray],
                          mk_vectorizer: Callable[[], Any],
                          mk_classifier: Callable[[], Any],
                          number_splits: int) \
        -> Tuple[float, float, np.ndarray]:
    # TODO: Write docs for this function.
    reviews, labels = data
    predictions = np.zeros(labels.shape, dtype=np.bool_)
    scores = np.zeros((number_splits,), dtype=np.float32)

    def go(idx, train_index, test_index):
        logging.info('Processing fold number {}...'.format(idx + 1))
        scores[idx], predictions[test_index] = run_one_fold(
            (reviews[train_index], labels[train_index]),
            (reviews[test_index], labels[test_index]),
            mk_vectorizer(), mk_classifier())

    if _get_sklearn_version() >= (0, 19, 0):
        skf = StratifiedKFold(n_splits=number_splits, shuffle=False)
        for idx, (train_index, test_index) \
                in enumerate(skf.split(reviews, labels)):
            go(idx, train_index, test_index)
    else:
        skf = StratifiedKFold(labels, n_folds=number_splits, shuffle=False)
        for idx, (train_index, test_index) in enumerate(skf):
            go(idx, train_index, test_index)

    score, score_std = np.mean(scores), np.std(scores)
    logging.info(('Overall accuracy on left-out data from k-fold '
                  'cross-validation was {}').format(score))
    wrong_index = np.where(predictions != labels)
    return score, score_std, wrong_index

def optimization_run(reviews: np.ndarray,
                     sentiments: np.ndarray,
                     mk_vectorizer: Callable[[], Any],
                     mk_classifier: Callable[[], Any],
                     conf: Dict[str, Any],
                     wrong_prediction_file: str) -> np.ndarray:
    """
    This is basically the main function.

    :param ids: Array of review identifiers.
    :type ids: 'numpy.ndarray' of shape ``(N,)``
    :param reviews: Array of raw reviews texts.
    :type reviews: 'numpy.ndarray' of shape ``(N,)``
    :param sentiments: Array of review sentiments.
    :type sentiments: 'numpy.ndarray' of shape ``(N,)``
    :param mk_vectorizer: Factory function to create a new vectorizer.
    :type mk_vectorizer: Callable[[], Vectorizer]
    :type mk_classifier: Callable[[], Classifier]
    :param n_splits
    :param test_on_10
    """
    data_90, data_10 = split_90_10((reviews, sentiments),
                                   conf['alpha'], conf['random'])
    reviews_90, _ = data_90
    reviews_10, labels_10 = data_10
    _, _, wrong_idx = run_a_couple_of_folds(
        data_90, mk_vectorizer, mk_classifier, conf['number_splits'])
    wrong_reviews = reviews_90[wrong_idx]
    if conf['test_10']:
        logging.info('Training on 90% and testing on 10%...')
        _, prediction_10 = run_one_fold(
            data_90, data_10, mk_vectorizer(), mk_classifier())
        wrong_idx_10 = np.where(prediction_10 != labels_10)
        wrong_reviews = np.concatenate(
            (wrong_reviews, reviews_10[wrong_idx_10]), axis=0)
    logging.info('Saving wrong predictions to {!r}...'
                 .format(wrong_prediction_file))
    pd.DataFrame(data={'review': wrong_reviews}) \
        .to_csv(wrong_prediction_file, index=False, quoting=3, escapechar='\\')

class SimpleAverager(object):
    """
    This class implements the simplest strategy for reducing a list of
    word2vec vectors into a single one -- averaging.
    """

    def __init__(self) -> None:
        pass

    @staticmethod
    def _make_avg_feature_vector(words: str,
                                 known_words: KeyedVectors,
                                 average_vector: Optional[np.ndarray] = None) \
            -> np.ndarray:
        """
        Given a list of words, returns the their average.

        :param str words:      Words to average.
        :param known_words:    Dictionary mapping known words to vectors.
        :type known_words:     Dict[str, np.ndarray]
        :param average_vector: Pre-allocated zero-initialised vector. It will
                               contain the average of ``words``.
        :type average_vector:  np.ndarray
        :return:               The average of ``words``.
        :rtype:                np.ndarray
        """
        word_count = sum(
            1 for _ in map(lambda x: np.add(average_vector, known_words[x],
                                            out=average_vector),
                           filter(lambda x: x in known_words, words.split()))
        )
        return np.divide(average_vector, float(word_count), out=average_vector)

    def transform(self, reviews: np.ndarray, model: KeyedVectors) \
            -> np.ndarray:
        # TODO: Docs
        (_, number_features) = model.syn0.shape
        (number_reviews,) = reviews.shape
        feature_vectors = np.zeros(
            (number_reviews, number_features), dtype='float32')
        for (i, (review, vector)) in enumerate(zip(reviews, feature_vectors)):
            if i % 1000 == 0:
                logging.info('PROGRESS: At review #{} of {}...'
                             .format(i, number_reviews))
            SimpleAverager._make_avg_feature_vector(
                review, model, vector)
        return feature_vectors

    def fit_transform(self, reviews: np.ndarray, model: KeyedVectors)\
            -> np.ndarray:
        """
        :py:class:`SimpleAverager` has no state, and :py:func:`fit_transform`
        thus simply calls :py:func:`transform`.
        """
        return self.transform(reviews, model)

class KMeansAverager(object):
    """
    This class implements the 'k-means' averaging strategy for reducing a list
    of word2vec vectors to a single one.
    """

    def __init__(self, number_clusters_frac: float,
                 warn_on_missing: bool, **kwargs: dict) -> None:
        self.number_clusters_frac = number_clusters_frac
        self.warn_on_missing = warn_on_missing
        self.kmeans_args = kwargs
        self.kmeans = None
        self.word2centroid = None

    @staticmethod
    def _make_bag_of_centroids(words: Iterable[str],
                               word2centroid: Dict[str, int],
                               bag_of_centroids: np.ndarray,
                               warn_on_missing: bool = False) \
            -> np.ndarray:
        """
        Given a stream of words and a mapping of words to centroid indices,
        converts these words to "bag of centroids" representation.

        :param words:            A list of words to convert.
        :type words:             Iterable[str]
        :param word2centroid:    A ``word → cluster index`` map.
        :type word2centroid:     Dict[str, int]
        :param bag_of_centroids: Output array.
        :type bag_of_centroids:  np.ndarray
        :param warn_on_missing:  Whether to issue warnings is some words are
                                 not in the `word2centroid` dictionary.
        :type warn_on_missing:   bool
        :return:                 Bag of centroids representation of ``words``.
        :rtype:                  np.ndarray
        """
        for word in words:
            i = word2centroid.get(word)
            if i is not None:
                bag_of_centroids[i] += 1
            elif warn_on_missing:
                warnings.warn(('While creating a bag of centroids: {!r} is '
                               'not in the word→index map.').format(word))
        return bag_of_centroids

    # NOTE: This uses the third option from here
    # https://github.com/mlp2018/BagofWords/issues/16. Is this the right
    # way to go?
    def transform(self, reviews: np.ndarray,
                  model: KeyedVectors) -> np.ndarray:
        """
        Given a list of reviews and a word2vec mapping, transforms all reviews
        to the bag of centroids representation.

        :param reviews: Reviews to transform.
        :type reviews:  np.ndarray
        :param model:   ``word→vector`` mapping.
        :type model:    Dict[str, np.ndarray]
        :return:        Array of reviews in the "bag of centroids"
                        representation.
        """
        (num_reviews,) = reviews.shape
        logging.info('Creating bags of centroids...')
        bags = np.zeros((num_reviews, self.kmeans.n_clusters), dtype='float32')
        for (review, bag) in zip(reviews, bags):
            KMeansAverager._make_bag_of_centroids(
                review.split(), self.word2centroid, bag, self.warn_on_missing)
        return bags

    def fit_transform(self, reviews: np.ndarray,
                      model: KeyedVectors) -> np.ndarray:
        """
        Given a list of reviews and a word2vec mapping, runs k-means clustering
        on the vector representation of reviews and converts them to the bag of
        centroids representation.

        :param reviews: Reviews to transform.
        :type reviews:  np.ndarray
        :param model:   ``word→vector`` mapping.
        :type model:    Dict[str, np.ndarray]
        :return:        Array of reviews in the "bag of centroids"
                        representation.
        """
        (num_reviews,) = reviews.shape
        num_clusters = int(self.number_clusters_frac * num_reviews)
        self.kmeans = MiniBatchKMeans(
            n_clusters=num_clusters, **self.kmeans_args)
        vectors = model.syn0
        logging.info('Running k-means + labeling...')
        start = time.time()
        cluster_indices = self.kmeans.fit_predict(vectors)
        end = time.time()
        logging.info('Done with k-means clustering in {:.0f} seconds!'
                     .format(end - start))
        logging.info('Creating word→index map...')
        self.word2centroid = dict(zip(model.index2word, cluster_indices))
        return self.transform(reviews, model)

class DummyAverager(object):
    """
    Averager that does not average :)
    """

    def __init__(self):
        pass

    def transform(self, reviews, model):
        return reviews, model

    def fit_transform(self, reviews, model):
        return reviews, model

class Word2VecVectorizer(object):
    """
    This class implements conversion of reviews to feature vectors using
    Word2Vec model.
    """

    _make_averager_fn = {
        'average': SimpleAverager,
        'k-means': KMeansAverager,
        'dummy':   DummyAverager,
    }

    def __init__(self,
                 strategy: str,
                 model_file: str,
                 model_args: Optional[Dict[str, Any]] = None,
                 averager_args: Optional[Dict[str, Any]] = None,
                 cache_file: Optional[str] = None,
                 retrain: bool = False,
                 train_data: Optional[Iterable[str]] = None) -> None:
        self.averager = \
            Word2VecVectorizer._make_averager_fn[strategy](**averager_args)
        self.model = None
        self.cache_file = cache_file
        self.is_compact = False
        # First of all, retrain the model if we're passed some training data.
        if train_data is not None:
            self.model = Word2VecVectorizer._retrain(
                train_data, model_file, **model_args)
        else:
            # If we already have our model cached, let's just load it.
            if cache_file is not None and Path(cache_file).exists():
                self.model = KeyedVectors.load_word2vec_format(
                    cache_file, binary=True)
                self.is_compact = True
            else:
                # Well, let's load the model and construct remove all
                # unnecessary stuff afterwards...
                # TODO: This always loads in binary format, because _retrain
                # uses binary format for saving. Not sure if that's OK.
                self.model = KeyedVectors.load_word2vec_format(
                    model_file, binary=True)

    @staticmethod
    def _retrain(train_data: Iterable[str], model_file: str, **model_args) \
            -> Word2Vec:
        # Touch the model_file _before_ we start training so that if the path
        # is wrong we don't wait hours for nothing...
        Path(model_file).touch()

        logging.info('Training Word2Vec model...')
        start = time.time()
        model = Word2Vec(reviews2sentences(train_data), **model_args).wv
        stop = time.time()
        logging.info('Done training in {:.0f} seconds!'
                     .format(stop - start))
        logging.info('Saving KeyedVectors model to {!r}...'
                     .format(model_file))
        # TODO: This always saves in binary format. I'm not sure that's what we
        # want.
        #                                                          Tom
        model.save_word2vec_format(model_file, binary=True)
        return model

    @staticmethod
    def _compress_keyed_vectors(model: KeyedVectors, reviews: np.ndarray,
                                cache_file: Optional[str] = None) -> None:
        required_words = set(word for review in reviews
                             for word in review.split())
        for (i, word) in enumerate(model.index2word):
            assert model.vocab[word].index == i

        # This is a really shady piece :)
        # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        # First of all, extract all the words we really need
        index2word = [w for w in model.index2word if w in required_words]

        def update_index(item, index):
            item.index = index
            return item
        # In the first approximation, we want
        # vocab = dict((w, model.vocab[w]) for w in index2word)
        # i.e. to keep the data for words which are in index2word. The problem
        # is that Vocab type has a field index which stores word's index in the
        # syn0 vector and index2word list. We, obbiously, break this indexing
        # by rebuilding index2word from scratch. So here we manually
        # reconstruct correct indexing.
        vocab = dict((w, update_index(model.vocab[w], i))
                     for (i, w) in enumerate(index2word))
        # Now, extract correct indices as an np.ndarray
        indices = np.fromiter(
            map(lambda x: x.index, vocab.values()), dtype=int)
        # Only keep feature vectors for the words we really need.
        vectors = model.syn0[indices]
        # Do the actual update
        model.index2word = index2word
        model.vocab = vocab
        model.syn0 = vectors
        # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        # end shady code...

        if cache_file is not None:
            model.save_word2vec_format(cache_file, binary=True)
        return model

    def fit_transform(self, reviews):
        if not self.is_compact:
            self.model = Word2VecVectorizer._compress_keyed_vectors(
                self.model, reviews, self.cache_file)
            self.is_compact = True
        return self.averager.fit_transform(reviews, self.model)

    def transform(self, reviews):
        if not self.is_compact:
            self.model = Word2VecVectorizer._compress_keyed_vectors(
                self.model, reviews, self.cache_file)
            self.is_compact = True
        return self.averager.transform(reviews, self.model)

def _make_vectorizer(conf):
    vectorizer_str = conf['vectorizer']
    if vectorizer_str == 'bagofwords':
        return CountVectorizer(**conf[vectorizer_str])
    elif vectorizer_str == 'word2vec':
        train_data = \
            _read_data_from(conf['in']['unlabeled'])['review'] \
            if conf['word2vec']['retrain'] else None
        return Word2VecVectorizer(
            **conf['word2vec'],
            averager_args=conf[conf['word2vec']['strategy']],
            train_data=train_data)
    else:
        raise Exception("Unknown vectorizer type.")

def _make_classifier(conf):
    _fn = {
        'logistic-regression':    LogisticRegression,
        'naive-bayes-bagofwords': MultinomialNB,
        'naive-bayes-word2vec':   BernoulliNB,
        'random-forest':          RandomForestClassifier,
        'convolutional':          ConvNNClassifier,
        'feed-forward':           FeedForwardNNClassifier,
    }
    return _fn[conf['classifier']](**conf[conf['classifier']])

class ConvNNClassifier(object):
    def __init__(self, n_units, n_filters, kernel_size,
                 use_bias, n_epochs, batch_size, loss,
                 optimizer, metrics, verbose):
        self.n_units = n_units
        self.n_filters = n_filters
        self.kernel_size = kernel_size
        self.use_bias = use_bias
        self.loss = loss
        self.optimizer = optimizer
        self.metrics = metrics
        self.batch_size = batch_size
        self.n_epochs = n_epochs
        self.verbose = verbose
        self.callbacks = [EarlyStopping(patience=1)]
        self.max_words = 500
        self.model = None
        self.history = None

    def _construct(self, word2vec):
        (n_words, n_features) = word2vec.vectors.shape
        model = Sequential()
        layer = Embedding(input_dim=n_words,
                          output_dim=n_features,
                          input_length=self.max_words,
                          weights=[word2vec.vectors],
                          trainable=False)
        model.add(layer)
        model.add(Conv1D(filters=self.n_filters,
                         kernel_size=self.kernel_size,
                         padding='same',
                         activation='relu',
                         use_bias=self.use_bias))
        model.add(MaxPooling1D(pool_size=self.max_words, strides=2))
        model.add(Flatten())
        model.add(Dense(self.n_units,
                        activation='relu',
                        use_bias=self.use_bias))
        model.add(Dense(1, activation='sigmoid', use_bias=self.use_bias))
        model.compile(
            loss=self.loss, optimizer=self.optimizer, metrics=self.metrics)
        logging.info(model.summary())
        return model

    @staticmethod
    def _review_to_tensor(review, word2vec, max_words):
        tensor = np.full((max_words,), fill_value=max_words)  # ?????
        for (i, word) in enumerate(filter(lambda x: x in word2vec,
                                          review.split())):
            # TODO: The following `if` is ungly, fix it please!
            if i >= max_words:
                break
            if word in word2vec:
                tensor[i] = word2vec.vocab[word].index
        return tensor

    def fit(self, train_data, labels):
        reviews, word2vec = train_data
        logging.info('Converting reviews to tensors...')
        reviews = np.array(list(map(
            lambda x: ConvNNClassifier._review_to_tensor(x, word2vec,
                                                         self.max_words),
            reviews)))
        logging.info('Done!')
        self.model = self._construct(word2vec)
        self.history = self.model.fit(x=reviews, y=labels,
                                      validation_data=None,
                                      epochs=self.n_epochs,
                                      batch_size=self.batch_size,
                                      verbose=self.verbose,
                                      callbacks=self.callbacks)
        return self

    def predict(self, train_data):
        reviews, word2vec = train_data
        (n_reviews,) = reviews.shape
        logging.info('Converting reviews to tensors...')
        reviews = np.array(list(map(
            lambda x: ConvNNClassifier._review_to_tensor(x, word2vec,
                                                         self.max_words),
            reviews)))
        logging.info('Done!')
        return self.model.predict(reviews).reshape((n_reviews,))

class FeedForwardNNClassifier(object):

    def __init__(self, n_hidden_units1, n_hidden_units2, batch_size,
                 n_epochs, optimizer, loss, metrics):
        self.n_hidden_units1 = n_hidden_units1
        self.n_hidden_units2 = n_hidden_units2
        self.batch_size = batch_size
        self.n_epochs = n_epochs
        self.optimizer = optimizer
        self.loss = loss
        self.metrics = metrics
        self.model = None
        self.history = None

    def _construct(self, n_features):
        model = Sequential()
        model.add(Dense(self.n_hidden_units1,
                        input_dim=n_features,
                        activation='relu',
                        kernel_initializer='uniform'))
        model.add(Dense(self.n_hidden_units2,
                        activation='relu',
                        kernel_initializer='uniform'))
        model.add(Dense(1, activation='sigmoid'))
        model.compile(loss=self.loss, optimizer=self.optimizer,
                      metrics=self.metrics)
        logging.info(model.summary())
        return model

    def fit(self, reviews, sentiments):
        (_, n_features) = reviews.shape
        self.model = self._construct(n_features)
        self.history = self.model.fit(x=reviews, y=sentiments,
                                      epochs=self.n_epochs,
                                      batch_size=self.batch_size)
        return self

    def predict(self, reviews):
        (n_reviews, _) = reviews.shape
        return self.model.predict(reviews).reshape((n_reviews,))

def main():
    logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
                        level=logging.INFO)
    if conf['run']['type'] == 'optimization':
        train_data = _read_data_from(conf['in']['labeled'])
        reviews = clean_up_reviews(train_data['review'],
                                   conf['run']['remove_stopwords'],
                                   conf['in']['clean'])
        sentiments = np.array(train_data['sentiment'], dtype=np.bool_)

        def mk_vectorizer():
            return _make_vectorizer(conf)

        def mk_classifier():
            return _make_classifier(conf)

        optimization_run(reviews, sentiments,
                         mk_vectorizer, mk_classifier,
                         conf['run'],
                         conf['out']['wrong_result'])
    elif conf['run']['type'] == 'submission':
        train_data = _read_data_from(conf['in']['labeled'])
        test_data = _read_data_from(conf['in']['test'])
        ids = np.array(test_data['id'], dtype=np.unicode_)
        reviews = clean_up_reviews(train_data['review'],
                                   conf['run']['remove_stopwords'],
                                   conf['in']['clean'])
        sentiments = np.array(train_data['sentiment'], dtype=np.bool_)
        test_reviews = clean_up_reviews(test_data['review'],
                                        conf['run']['remove_stopwords'])

        def mk_vectorizer():
            return _make_vectorizer(conf)

        def mk_classifier():
            return _make_classifier(conf)

        submission_run(reviews, sentiments,
                       test_reviews,
                       ids,
                       mk_vectorizer, mk_classifier,
                       conf['out']['result'])
    else:
        raise NotImplementedError()

if __name__ == '__main__':
    # cProfile.run('main()')
    main()


'''
We want make a package of goal kilos of chocolate. We have small bars (1 kilo each) and big bars (5 kilos each).
Return the number of small bars to use, assuming we always use big bars before small bars.
Return -1 if it can't be done.
INPUT:make_chocolate(4, 1, 9) OUTPUT:4
INPUT:make_chocolate(4, 1, 10) OUTPUT:-1
INPUT:make_chocolate(4, 1, 7) OUTPUT:2
'''

def make_chocolate(small, big, goal):
    big_kilo_range = range(1, big + 1)
    for i in big_kilo_range:
        big_kilos = big_kilo_range[i-1] * 5
        if (big_kilos + small) >= goal:
            return goal - big_kilos
    return -1

count = make_chocolate(4, 4, 15)
print(count)


import string
import json
import random
from string import Template
import os
from collections import Counter, defaultdict
import settings
from tqdm import tqdm

class GeneratorBasic:
    def __init__(self, args, taskconfig, data_readers):
        self.idx=0
        self.args = args
        self.taskconfig = taskconfig
        if 'max_data' in self.taskconfig:
            self.max_data = self.taskconfig['max_data']
        else:
            self.max_data = args.max_data
        self.output_folder = args.tasks_output_folder
        self.out_file = self.taskconfig['task_name']
        self.data_readers = data_readers
        self.examples = []

    def generate_data(self):
        print('generating basic')

    def sample_and_write(self, instance_and_instructions):
        if type(instance_and_instructions) is None:
            print('Please refactor code so that the sample_and_write function performs sampling and writing')
        sequences, instruction_dict = instance_and_instructions
        print('len datareader_sequences Instances', len(sequences))
        if self.args.data_sample_type=='individual':
            data_dict_all = defaultdict(list)
            for dp in sequences:
                # import pdb;pdb.set_trace()
                data_dict_all[dp['dataset']].append(dp)
            # for d, dataset_reader in enumerate(self.data_readers):
            for dataname in data_dict_all.keys():
                datareader_sequences = data_dict_all[dataname]
                random.shuffle(datareader_sequences)
                # print('len(sequences)', len(datareader_sequences))
                datareader_sequences_sampled = random.sample(datareader_sequences, min(len(datareader_sequences), self.max_data))
                if 'Instances' not in instruction_dict: instruction_dict['Instances'] = []
                instruction_dict['Instances'] += datareader_sequences_sampled[:]
                # sequences = []

        elif self.args.data_sample_type=='common':
            random.shuffle(sequences)
            sequences = random.sample(sequences, min(len(sequences), self.max_data))
            instruction_dict['Instances'] = sequences

        else:# self.args.data_sample_type=='max':
            instruction_dict['Instances'] = sequences

        for s in sequences[-5:]:
            # print(s)
            pass

        print('writing to', self.out_file, 'size of data', len(sequences))

        with open(self.output_folder+'/'+self.out_file+'.json', 'w') as f:
            f.write(json.dumps(instruction_dict, indent=2))

    def __len__(self):
        return len(self.examples)

    def __getitem__(self, idx):
        return self.examples[idx]


from fiis_scraping.bot.collect_data.collector import collect_all_b3
from fiis_scraping.util.formata.formata_data import fun_data
from logging import basicConfig, FileHandler, StreamHandler, INFO, DEBUG

def main():
    perido_consulta = fun_data().replace("/", "-")

    # iniciando as configurações de log do programa
    # criando path para o arquivo log do programa
    path_file_log = f'logs/{perido_consulta.replace("/", "-")}_relatorio.log'

    # crindo especificações de log
    file_handler = FileHandler(path_file_log, "a")
    file_handler.setLevel(INFO)
    stream_handler = StreamHandler()

    basicConfig(format='%(asctime)s  %(name)s  %(levelname)s: %(message)s',
                level=INFO,
                handlers=[file_handler, stream_handler])

    # inicia processo de coleta de dados referente aos FIIs
    collect_all_b3(perido_consulta)

if __name__ == '__main__':
    main()


def reverse(s, start, stop):
    """
    序列倒置 递归方式实现
    :param s:
    :param start:
    :param stop:
    :return:
    """

    if start < stop:
        s[start], s[stop] = s[stop], s[start]
        reverse(s, start + 1, stop - 1)

def reverse_iterative(s):
    """
    序列倒置 迭代方式实现
    :param s:
    :return:
    """

    start, stop = 0, len(s) - 1
    while start < stop:
        s[start], s[stop] = s[stop], s[start]
        start, stop = start + 1, stop - 1

if __name__ == '__main__':
    a = [1, 2, 3]
    print(a)

    reverse(a, 0, 2)
    print(a)

    reverse_iterative(a)
    print(a)


import pytest

from ai_ghostfunctions import keywords

@pytest.mark.parametrize(
    "kw,v",
    [
        (keywords.ASSISTANT, "assistant"),
        (keywords.USER, "user"),
        (keywords.SYSTEM, "system"),
        (keywords.CHAT_COMPLETION, "chat completion"),
        (keywords.COMPLETION, "completion"),
    ],
)
def test_keyword_value(kw: str, v: str) -> None:
    assert kw == v


from collections import deque

def bfs(graph, startNode, visited):
    queue = deque([startNode])
    visited = list()

    while queue:
        node = queue.popleft()

        if node not in visited:
            visited.append(node)
            queue.extend(graph[node])
    return visited

def dfs(graph, startNode, visited):
    stack = deque([startNode])
    visited = list()

    while stack:
        node = stack.pop()

        if node not in visited:
            visited.append(node)
            stack.extend(reversed(graph[node]))
    return visited

n, m, v = map(int,input().split())

graph = [[] for i in range(n + 1)]

for _ in range(m):
    i, j = map(int, input().split())
    graph[i].append(j)
    graph[j].append(i)

print(graph)
print(dfs(graph, v, [False] * (n + 1)))
print(bfs(graph, v, [False] * (n + 1)))


# LaplaceLine.py: Solve Laplace’s eqn, 3D matplot, close shell to quit

from numpy import *
import matplotlib.pylab as p;
from mpl_toolkits.mplot3d import Axes3D

#p.xkcd()                  # for fun, uncomment for xkcd atyle plots
print("Initializing")
Nmax = 100;               # grid points Nx, Ny
Niter = 70;               # sweeps/iterations to perform
V = zeros((Nmax, Nmax), float)

print( "Working hard, wait for the figure while I count to 60")
for k in range(0, Nmax-1): V[k,0] = 100.0  # line at 100V

for iter in range(Niter):  # iterations over algorithm
    if iter%10 == 0: print(iter)
    for i in range(1, Nmax-2):
        for j in range(1,Nmax-2): V[i,j] = 0.25*(V[i+1,j]+V[i-1,j]+V[i,j+1]+V[i,j-1])
x = range(0, Nmax-1, 2); y = range(0, 50, 2)    # plot every other point
X, Y = p.meshgrid(x,y)

def functz(V):    # Function returns V(x, y)
    z=V[X,Y]
    return z

Z = functz(V)
fig = p.figure()  # Create figure
ax = Axes3D(fig)  # plot axes
ax.plot_wireframe(X, Y, Z, color = 'r')  # red wireframe
ax.set_xlabel('X')                       # label axes
ax.set_ylabel('Y')
ax.set_zlabel('Potential')
print("Control-C to exit")
p.show()                                 # display fig


import datetime
import discord
from discord.ext import commands
import io
import numpy as np
from mongoengine import *
from PIL import Image
import pytesseract
import requests
import uuid
import yaml

class Reports(Document):
    reportId = IntField(required=True)
    serverName = StringField(max_length=100,required=True)
    userId = IntField(required=True)
    imageText = StringField(max_length=1337,required=True)

TOKEN = 'token_here'
bot = commands.Bot(command_prefix='cs!')
config = yaml.load(open("config.yaml"), Loader=yaml.FullLoader)
connect('csampolicebot', host='127.0.0.1', port=27017)

def is_staff():
    def predicate(ctx):
        return ctx.message.guild.get_role(config["moderation_role_ID"]) in ctx.message.author.roles
    return commands.check(predicate)

def is_owner():
    def predicate(ctx):
        return ctx.author.id == 1337 # to be replaced with bot/server owner ID
    return commands.check(predicate)

@bot.event
async def on_ready():
    print('Successfully connected! Hello!')

@bot.event
async def on_message(message):
    staff_role = message.guild.get_role(config["moderation_role_ID"])
    if not staff_role in message.author.roles:
        urls = set()
        for part in message.content.lower().split():
            if part.startswith("http") and (part.endswith(".jpg") or part.endswith(".png") or part.endswith(".jpeg")):
                urls.add(part)
        if message.attachments:
            for attachment in message.attachments:
                # just like embeds, attachments that are not images do not interest us so we won't add them to the URL list to scan
                if attachment.url.lower().endswith(".jpg") or attachment.url.lower().endswith(".png") or attachment.url.lower().endswith(".jpeg"):
                    urls.add(attachment.url)
        if len(urls) > 0:
            for url in urls:
                result, text = scancontent(url)
                if result:
                    user = message.author
                    role = message.guild.get_role(config["punished_role_ID"])
                    report_id = uuid.uuid1().int>>65
                    channel = bot.get_channel(config["alert_channel"])
                    embed = discord.Embed(title="User muted", description="", color=0x00ff00)
                    embed.add_field(name="User", value="{} <\@{}>".format(user, user.id), inline=False)
                    embed.add_field(name="Reason", value="Blacklisted word detected in image", inline=False)
                    embed.add_field(name="Mute ID", value="{}".format(report_id), inline=False)
                    embed.timestamp = datetime.datetime.utcnow()
                    report = Reports(reportId=report_id, serverName=message.guild.name, userId=user.id, imageText=text)
                    report.save()
                    await channel.send(embed=embed)
                    await user.add_roles(role)
                    await message.delete()
    await bot.process_commands(message)

@bot.command()
@is_owner()
async def delete(ctx, rID):
    try:
        report = Reports.objects.get(reportId=rID)
        report.delete()
        msg = discord.Embed(title="Report deletion", description="", color=0x00ff00)
        msg.add_field(name=":white_check_mark: Success", value="The report indicated by you has been deleted successfully.", inline=False)
        msg.timestamp = datetime.datetime.utcnow()
    except Reports.DoesNotExist:
        msg = discord.Embed(title="Report deletion", description="", color=0x00ff00)
        msg.add_field(name=":x: Something's wrong!", value="Sorry! I couldn't find any reports to delete for the ID you provided.", inline=False)
        msg.timestamp = datetime.datetime.utcnow()
    await ctx.send(embed=msg)

@bot.command()
@is_staff()
async def lookup(ctx, rID):
    try:
        report = Reports.objects.get(reportId=rID)
        user =  await bot.fetch_user(report.userId)
        msg = discord.Embed(title="Report information", description="", color=0x00ff00)
        msg.add_field(name="User", value="{} <\@{}>".format(user, user.id), inline=False)
        msg.add_field(name="Server", value="{}".format(report.serverName), inline=False)
        msg.add_field(name="Reported image content", value=report.imageText, inline=False)
        msg.timestamp = datetime.datetime.utcnow()
    except Reports.DoesNotExist:
        report = None
        msg = discord.Embed(title="Report information", description="", color=0x00ff00)
        msg.add_field(name="No report found :(", value="Sorry! I couldn't find any reports for the ID you provided.", inline=False)
        msg.timestamp = datetime.datetime.utcnow()
    await ctx.send(embed=msg)

def scancontent(url):
    try:
        r = requests.get(url, stream=True)
        r.raise_for_status()
        im = Image.open(io.BytesIO(r.content))
        image =  np.array(im)
        text = pytesseract.image_to_string(image)
        # Freeing memory because we don't need the image anymore, text is enough
        im.close()
        image = None
        for word in config["banned_words"]:
            if word in text.lower():
                return True, text
    except requests.exceptions.HTTPError as err:
        print(err)
    return False, text

def main():
    bot.run(TOKEN)

if __name__ == '__main__':
    main()


"""
Modules for extracting time synchronization statistics according to Tri-Message protocol. All functions assume
payload for ONE data packet/decoder ONLY. These modules will be called separately by API800 and API900 loaders, they
are in themselves helper functions. They do not depend on API formats, they take the time sync payloads as parameters
and use Tri-Message protocol to compute latencies, check criteria, and correct the "machine" start time B0 based on the
minimum latencies.
"""

from typing import Dict, List, Optional, Tuple, Union

# noinspection Mypy
import numpy as np

class TriMessageStats:
    """
    Stores statistics about the tri-message exchanges
    ALL timestamps in microseconds
    Properties:
        packet_id: an identifier for the packet that contains the data.  Used for reporting purposes
        latency1: latencies measured by timestamps 1 and 2
        latency3: latencies measured by timestamps 2 and 3
        offset1: offsets measured by timestamps 1 and 2
        offset3: offsets measured by timestamps 2 and 3
        best_latency: minimum latency that meets all criteria
        best_offset: best offset that meets all criteria
        best_latency_array_index: number of latency array with best latency
        best_latency_index: index in latency array of best latency
        num_messages: number of tri-message exchanges
    """

    def __init__(self,
                 packet_id: Union[str, int],
                 a1: np.ndarray,
                 a2: np.ndarray,
                 a3: np.ndarray,
                 b1: np.ndarray,
                 b2: np.ndarray,
                 b3: np.ndarray):
        """
        Calculate latency, offset, and their qualities.
        :param packet_id: an identifier for reporting purposes
        :param a1: array of server timestamp 1
        :param a2: array of server timestamp 2
        :param a3: array of server timestamp 3
        :param b1: array of device timestamp 1
        :param b2: array of device timestamp 2
        :param b3: array of device timestamp 3
        """
        self.packet_id: Union[str, int] = packet_id
        self.num_messages: int = len(a1)
        # compute latencies and offsets
        latencies_tuple: Tuple[np.ndarray, np.ndarray] = latencies(a1, a2, a3, b1, b2, b3)
        self.latency1: np.ndarray = latencies_tuple[0]
        self.latency3: np.ndarray = latencies_tuple[1]
        offsets_tuple: Tuple[np.ndarray, np.ndarray] = offsets(a1, a2, a3, b1, b2, b3)
        self.offset1: np.ndarray = offsets_tuple[0]
        self.offset3: np.ndarray = offsets_tuple[1]

        self.best_latency: Optional[float] = None
        self.best_latency_array_index: Optional[int] = None
        self.best_latency_index: Optional[int] = None
        self.best_offset: Optional[float] = 0.0

        self.find_best_latency()
        self.find_best_offset()

    def find_best_latency(self) -> None:
        """
        Finds the best latency among the latencies
        """
        try:
            # find value and index of minimum latency of nonzero latencies
            d1_min: float = np.min(self.latency1[self.latency1 != 0])
            d3_min: float = np.min(self.latency3[self.latency3 != 0])

            if d3_min > d1_min:
                self.best_latency = d1_min  # server round trip is shorter
                self.best_latency_array_index = 1
                self.best_latency_index = np.where(self.latency1 == d1_min)[0][0]
            else:
                self.best_latency = d3_min
                self.best_latency_array_index = 3
                self.best_latency_index = np.where(self.latency3 == d3_min)[0][0]
        except ValueError:
            # all latencies for one of the arrays is zero; the data is untrustworthy.  set the defaults
            self.best_latency = None
            self.best_latency_array_index = None
            self.best_latency_index = None

    def find_best_offset(self) -> None:
        """
        Finds the best offset among the offsets
        """
        # if no best latency, find it
        if self.best_latency is None:
            self.find_best_latency()
        # best latency = best offset, if best latency is still None, best offset is 0.0
        if self.best_latency_array_index == 1:
            self.best_offset = self.offset1[self.best_latency_index]
        elif self.best_latency_array_index == 3:
            self.best_offset = self.offset3[self.best_latency_index]
        else:
            self.best_offset = 0.0

    def set_latency(self,
                    a1_coeffs: np.ndarray,
                    a2_coeffs: np.ndarray,
                    a3_coeffs: np.ndarray,
                    b1_coeffs: np.ndarray,
                    b2_coeffs: np.ndarray,
                    b3_coeffs: np.ndarray) -> None:
        """
        set the latency and find the best
        :param a1_coeffs: server timestamp 1
        :param a2_coeffs: server timestamp 2
        :param a3_coeffs: server timestamp 3
        :param b1_coeffs: device timestamp 1
        :param b2_coeffs: device timestamp 2
        :param b3_coeffs: device timestamp 3
        """
        # compute latencies
        self.latency1, self.latency3 = latencies(a1_coeffs, a2_coeffs, a3_coeffs, b1_coeffs, b2_coeffs, b3_coeffs)
        self.find_best_latency()

    def set_offset(self,
                   a1_coeffs: np.ndarray,
                   a2_coeffs: np.ndarray,
                   a3_coeffs: np.ndarray,
                   b1_coeffs: np.ndarray,
                   b2_coeffs: np.ndarray,
                   b3_coeffs: np.ndarray) -> None:
        """
        set the offset and find the best
        :param a1_coeffs: server timestamp 1
        :param a2_coeffs: server timestamp 2
        :param a3_coeffs: server timestamp 3
        :param b1_coeffs: device timestamp 1
        :param b2_coeffs: device timestamp 2
        :param b3_coeffs: device timestamp 3
        """
        # latency calculations are required:
        if self.best_latency is None:
            self.set_latency(a1_coeffs, a2_coeffs, a3_coeffs, b1_coeffs, b2_coeffs, b3_coeffs)
        # compute offsets
        self.offset1, self.offset3 = offsets(a1_coeffs, a2_coeffs, a3_coeffs, b1_coeffs, b2_coeffs, b3_coeffs)
        self.find_best_offset()

def latencies(a1_coeffs: np.ndarray,
              a2_coeffs: np.ndarray,
              a3_coeffs: np.ndarray,
              b1_coeffs: np.ndarray,
              b2_coeffs: np.ndarray,
              b3_coeffs: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
    """
    Compute latencies in microseconds based on message exchange timestamps.

    Parameters
    ----------
    a1_coeffs, ... b3_coeffs: tri-message timestamps as loaded in transmit_receive_timestamps_microsec

    Returns
    -------
    d1_coeffs: array of server round trip latencies in microseconds
    d3_coeffs: array of device round trip latencies in microseconds
    """
    # Compute latencies in microseconds
    d1_coeffs: np.ndarray = 0.5 * ((a2_coeffs - a1_coeffs) - (b2_coeffs - b1_coeffs))
    d3_coeffs: np.ndarray = 0.5 * ((b3_coeffs - b2_coeffs) - (a3_coeffs - a2_coeffs))

    # convert negative latencies to 0.  negative latencies should not exist naturally
    d1_coeffs[d1_coeffs < 0] = 0
    d3_coeffs[d3_coeffs < 0] = 0

    return d1_coeffs, d3_coeffs

def offsets(a1_coeffs: np.ndarray,
            a2_coeffs: np.ndarray,
            a3_coeffs: np.ndarray,
            b1_coeffs: np.ndarray,
            b2_coeffs: np.ndarray,
            b3_coeffs: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
    """
    Compute offsets in microseconds based on message exchange timestamps.

    Parameters
    ----------
    a1_coeffs, ... b3_coeffs: tri-message timestamps as loaded in transmit_receive_timestamps_microsec

    Returns
    -------
    o1_coeffs: array of server round trip latencies in microseconds
    o3_coeffs: array of device round trip latencies in microseconds
    """
    # assume the generic equation f = a - b + d
    # where d is latency, f is offset, b is machine time and a is time sync server time
    # refer to latency equations above for definitions of d1_coeffs and d3_coeffs
    # with latency d1_coeffs, the equation is f1 = a1_coeffs - b1_coeffs + d1_coeffs
    # with latency d3_coeffs, the equation is f3 = a3_coeffs - b3_coeffs + d3_coeffs
    # In the absence of latency, offset can be calculated this way:
    # o1_coeffs = (a1_coeffs - b1_coeffs + a2_coeffs - b2_coeffs) / 2.
    # o3_coeffs = (a3_coeffs - b3_coeffs + a2_coeffs - b2_coeffs) / 2.
    # get latencies
    latencies_tuple: Tuple[np.ndarray, np.ndarray] = latencies(a1_coeffs,
                                                               a2_coeffs,
                                                               a3_coeffs,
                                                               b1_coeffs,
                                                               b2_coeffs,
                                                               b3_coeffs)
    d1_coeffs: np.ndarray = latencies_tuple[0]
    d3_coeffs: np.ndarray = latencies_tuple[1]
    # use latency to compute offset in microseconds
    o1_coeffs: np.ndarray = a1_coeffs - b1_coeffs + d1_coeffs
    o3_coeffs: np.ndarray = a3_coeffs - b3_coeffs + d3_coeffs

    return o1_coeffs, o3_coeffs

def validate_timestamps(a1_coeffs: np.ndarray, a2_coeffs: np.ndarray, a3_coeffs: np.ndarray,
                        b1_coeffs: np.ndarray, b2_coeffs: np.ndarray, b3_coeffs: np.ndarray) -> \
        Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
    """
    it's possible some of the tri-message values are duplicated; the duplicates and other invalid times
    must be removed.

       Parameters
       -------
       a1_coeffs, a2_coeffs, a3_coeffs, b1_coeffs, b2_coeffs, b3_coeffs: arrays of message exchange timestamps

       Returns
       -------
       a1_coeffs[valid_indices], a2_coeffs[valid_indices], a3_coeffs[valid_indices],
       b1_coeffs[valid_indices], b2_coeffs[valid_indices], b3_coeffs[valid_indices]:
       arrays of valid message exchange timestamps
    """
    num_timestamps = len(a1_coeffs)
    # if length is 1 or less, no need to validate, just return all the values
    if num_timestamps <= 1:
        return a1_coeffs, a2_coeffs, a3_coeffs, b1_coeffs, b2_coeffs, b3_coeffs
    # if here, there's more than 1 exchange to check
    valid_times: List[Dict] = [{}, {}, {}, {}, {}, {}]
    valid_indices = []
    all_timestamps = [a1_coeffs, a2_coeffs, a3_coeffs, b1_coeffs, b2_coeffs, b3_coeffs]
    # for each set of timestamps a1x, a2x, etc. in all exchanges
    for data_index in range(6):
        for time_index in range(num_timestamps):
            # compare the time to existing information
            time = all_timestamps[data_index][time_index]
            if time not in valid_times:
                # it's not in valid times, it's a new time
                valid_times[data_index][time] = time_index
    for index in valid_times[0].values():
        # if it's not in the first one, it's not valid.  if it doesn't show up in all others, it's not valid
        if index in valid_times[1].values() and index in valid_times[2].values() and \
                index in valid_times[3].values() and index in valid_times[4].values() and \
                index in valid_times[5].values():
            valid_indices.append(index)
    return a1_coeffs[valid_indices], a2_coeffs[valid_indices], a3_coeffs[valid_indices], \
        b1_coeffs[valid_indices], b2_coeffs[valid_indices], b3_coeffs[valid_indices]

def transmit_receive_timestamps_microsec(coeffs: np.ndarray) -> Tuple[
        np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
    """
       Recover Tri-Message timestamp coefficients. Uses concept of
       Tri-Message to synchronize a device with a reference server:

           First, server A transmits a message to device B with timestamp a1_coeffs. B receives the message at timestamp
            b1_coeffs. B then transmits a message back to A with timestamp b2_coeffs. A receives this message at
            timestamp a2_coeffs. A transmits a second message to B at timestamp a3_coeffs. B receives the message at
            timestamp b3_coeffs.

       Parameters
       ----------
       coeffs: array of tri-message coefficients (a1_coeffs, a2_coeffs, a3_coeffs, b1_coeffs, b2_coeffs, b3_coeffs)

       Returns
       -------
       a1_coeffs, a2_coeffs, a3_coeffs, b1_coeffs, b2_coeffs, b3_coeffs: arrays of message exchange timestamps
    """

    if len(coeffs) % 6 != 0:
        raise Exception("Tri-Message contains partial exchange, unsafe to use it for computations.")

    # Timing coefficients
    step: int = 6  # each tri-message exchange contains 6 timestamps, 3 from server and 3 from device
    stop: int = int(len(coeffs) / 6) * step

    a1_coeffs: np.ndarray = coeffs[0: stop: step]  # server first transmit timestamps in epoch microseconds
    a2_coeffs: np.ndarray = coeffs[1: stop: step]  # server first receive timestamps in epoch microseconds
    a3_coeffs: np.ndarray = coeffs[2: stop: step]  # server second transmit timestamps in epoch microseconds
    b1_coeffs: np.ndarray = coeffs[3: stop: step]  # device first receive timestamps in mach microseconds
    b2_coeffs: np.ndarray = coeffs[4: stop: step]  # device first transmit timestamps in mach microseconds
    b3_coeffs: np.ndarray = coeffs[5: stop: step]  # device second receive timestamps in mach microseconds

    # make sure each tri-message exchange contains 6 timestamps (done with modulo check above)
    # assert len(a1_coeffs) == len(a2_coeffs) == len(a3_coeffs) == len(b1_coeffs) == len(b2_coeffs) == len(b3_coeffs)

    return a1_coeffs, a2_coeffs, a3_coeffs, b1_coeffs, b2_coeffs, b3_coeffs


import csv

def get_month_data(data, month, year):
    for line in data:
        print(line)
        if line["Month"] == month:
            return line[year]

def get_file_content(file_name):
    with open(file_name) as csv_file:
        file_reader = csv.DictReader(csv_file, delimiter=',')
        result = []
        for line in file_reader:
            result.append(line)
    return result

file_content = get_file_content("airtravel.csv")
month_data = get_month_data(file_content, "DEC", "1959")
print(month_data)

# row = get_month_row("JUL", file_data)
# print(row[3])  # what does the 622 means?


import CoreFoundation
import objc
from PyObjCTools.TestSupport import (
    TestCase,
    min_os_level,
    os_level_key,
    os_release,
    skipUnless,
)

class TestRunLoop(TestCase):
    def testTypes(self):
        self.assertIsCFType(CoreFoundation.CFRunLoopRef)
        self.assertIsCFType(CoreFoundation.CFRunLoopSourceRef)
        self.assertIsCFType(CoreFoundation.CFRunLoopObserverRef)

        try:
            if objc.lookUpClass("__NSCFTimer") is CoreFoundation.CFRunLoopTimerRef:
                return
        except objc.error:
            pass
        try:
            if objc.lookUpClass("NSCFTimer") is CoreFoundation.CFRunLoopTimerRef:
                return
        except objc.error:
            pass
        self.assertIsCFType(CoreFoundation.CFRunLoopTimerRef)

    def testConstants(self):
        self.assertEqual(CoreFoundation.kCFRunLoopRunFinished, 1)
        self.assertEqual(CoreFoundation.kCFRunLoopRunStopped, 2)
        self.assertEqual(CoreFoundation.kCFRunLoopRunTimedOut, 3)
        self.assertEqual(CoreFoundation.kCFRunLoopRunHandledSource, 4)
        self.assertEqual(CoreFoundation.kCFRunLoopEntry, (1 << 0))
        self.assertEqual(CoreFoundation.kCFRunLoopBeforeTimers, (1 << 1))
        self.assertEqual(CoreFoundation.kCFRunLoopBeforeSources, (1 << 2))
        self.assertEqual(CoreFoundation.kCFRunLoopBeforeWaiting, (1 << 5))
        self.assertEqual(CoreFoundation.kCFRunLoopAfterWaiting, (1 << 6))
        self.assertEqual(CoreFoundation.kCFRunLoopExit, (1 << 7))
        self.assertEqual(CoreFoundation.kCFRunLoopAllActivities, 0x0FFFFFFF)
        self.assertIsInstance(CoreFoundation.kCFRunLoopDefaultMode, str)
        self.assertIsInstance(CoreFoundation.kCFRunLoopCommonModes, str)

    def testGetTypeID(self):
        self.assertIsInstance(CoreFoundation.CFRunLoopGetTypeID(), int)
        self.assertIsInstance(CoreFoundation.CFRunLoopSourceGetTypeID(), int)
        self.assertIsInstance(CoreFoundation.CFRunLoopObserverGetTypeID(), int)
        self.assertIsInstance(CoreFoundation.CFRunLoopTimerGetTypeID(), int)

    def testRunloop(self):
        runloop_mode = CoreFoundation.kCFRunLoopDefaultMode
        runloop_mode = "pyobjctest.cfrunloop"

        loop = CoreFoundation.CFRunLoopGetCurrent()
        self.assertIsInstance(loop, CoreFoundation.CFRunLoopRef)
        loop = CoreFoundation.CFRunLoopGetMain()
        self.assertIsInstance(loop, CoreFoundation.CFRunLoopRef)
        mode = CoreFoundation.CFRunLoopCopyCurrentMode(loop)
        if mode is not None:
            self.assertIsInstance(mode, str)
        self.assertResultIsCFRetained(CoreFoundation.CFRunLoopCopyAllModes)
        allmodes = CoreFoundation.CFRunLoopCopyAllModes(loop)
        self.assertIsInstance(allmodes, CoreFoundation.CFArrayRef)
        self.assertNotEqual(len(allmodes), 0)
        for mode in allmodes:
            self.assertIsInstance(mode, str)
        CoreFoundation.CFRunLoopAddCommonMode(loop, "pyobjctest")
        allmodes = CoreFoundation.CFRunLoopCopyAllModes(loop)

        tm = CoreFoundation.CFRunLoopGetNextTimerFireDate(loop, runloop_mode)
        self.assertIsInstance(tm, float)
        b = CoreFoundation.CFRunLoopIsWaiting(loop)
        self.assertIsInstance(b, bool)
        CoreFoundation.CFRunLoopWakeUp(loop)
        CoreFoundation.CFRunLoopStop(loop)

        res = CoreFoundation.CFRunLoopRunInMode("mode", 2.0, True)
        self.assertIsInstance(res, int)
        self.assertEqual(res, CoreFoundation.kCFRunLoopRunFinished)

        # CoreFoundation.CFRunLoopRun is hard to test reliably
        self.assertHasAttr(CoreFoundation, "CFRunLoopRun")

    def testObserver(self):
        runloop_mode = CoreFoundation.kCFRunLoopDefaultMode
        runloop_mode = "pyobjctest.cfrunloop"

        rl = CoreFoundation.CFRunLoopGetCurrent()

        data = {}
        state = []

        def callback(observer, activity, info):
            state.append((observer, activity, info))

        observer = CoreFoundation.CFRunLoopObserverCreate(
            None,
            CoreFoundation.kCFRunLoopEntry | CoreFoundation.kCFRunLoopExit,
            True,
            4,
            callback,
            data,
        )
        self.assertIsInstance(observer, CoreFoundation.CFRunLoopObserverRef)
        ctx = CoreFoundation.CFRunLoopObserverGetContext(observer, None)
        self.assertIs(ctx, data)
        self.assertIs(CoreFoundation.CFRunLoopObserverDoesRepeat(observer), True)
        self.assertEqual(CoreFoundation.CFRunLoopObserverGetOrder(observer), 4)
        self.assertIs(CoreFoundation.CFRunLoopObserverIsValid(observer), True)
        self.assertEqual(
            CoreFoundation.CFRunLoopObserverGetActivities(observer),
            CoreFoundation.kCFRunLoopEntry | CoreFoundation.kCFRunLoopExit,
        )
        CoreFoundation.CFRunLoopObserverInvalidate(observer)
        self.assertIs(CoreFoundation.CFRunLoopObserverIsValid(observer), False)
        ctx = CoreFoundation.CFRunLoopObserverGetContext(observer, None)
        self.assertIs(ctx, objc.NULL)
        observer = CoreFoundation.CFRunLoopObserverCreate(
            None,
            CoreFoundation.kCFRunLoopEntry | CoreFoundation.kCFRunLoopExit,
            True,
            4,
            callback,
            data,
        )
        self.assertIsInstance(observer, CoreFoundation.CFRunLoopObserverRef)
        self.assertIs(
            CoreFoundation.CFRunLoopContainsObserver(rl, observer, runloop_mode), False
        )
        CoreFoundation.CFRunLoopAddObserver(rl, observer, runloop_mode)
        self.assertIs(
            CoreFoundation.CFRunLoopContainsObserver(rl, observer, runloop_mode), True
        )

        # Use dummy stream to ensure that the runloop actually performs work
        strval = b"hello world"
        stream = CoreFoundation.CFReadStreamCreateWithBytesNoCopy(
            None, strval, len(strval), CoreFoundation.kCFAllocatorNull
        )
        self.assertIsInstance(stream, CoreFoundation.CFReadStreamRef)
        CoreFoundation.CFReadStreamScheduleWithRunLoop(stream, rl, runloop_mode)
        res = CoreFoundation.CFRunLoopRunInMode(runloop_mode, 1.0, True)
        self.assertIsInstance(res, int)
        CoreFoundation.CFReadStreamUnscheduleFromRunLoop(stream, rl, runloop_mode)

        self.assertNotEqual(len(state), 0)
        for item in state:
            self.assertIs(item[0], observer)
            self.assertIn(
                item[1], (CoreFoundation.kCFRunLoopEntry, CoreFoundation.kCFRunLoopExit)
            )
            self.assertIs(item[2], data)
        CoreFoundation.CFRunLoopRemoveObserver(rl, observer, runloop_mode)
        self.assertIs(
            CoreFoundation.CFRunLoopContainsObserver(rl, observer, runloop_mode), False
        )

    def testTimer(self):
        runloop_mode = CoreFoundation.kCFRunLoopDefaultMode
        runloop_mode = "pyobjctest.cfrunloop"
        rl = CoreFoundation.CFRunLoopGetCurrent()

        state = []
        data = {}

        def callback(timer, info):
            state.append((timer, info))

        timer = CoreFoundation.CFRunLoopTimerCreate(None, 0, 0.5, 0, 0, callback, data)

        r = CoreFoundation.CFRunLoopTimerGetNextFireDate(timer)
        self.assertIsInstance(r, float)
        CoreFoundation.CFRunLoopTimerSetNextFireDate(timer, r + 2)
        r2 = CoreFoundation.CFRunLoopTimerGetNextFireDate(timer)
        self.assertEqual(int(r2), int(r + 2))

        r = CoreFoundation.CFRunLoopTimerGetInterval(timer)
        self.assertEqual(r, 0.5)

        self.assertIs(CoreFoundation.CFRunLoopTimerGetContext(timer, None), data)
        self.assertIs(CoreFoundation.CFRunLoopTimerDoesRepeat(timer), True)
        self.assertEqual(CoreFoundation.CFRunLoopTimerGetOrder(timer), 0)
        self.assertIs(CoreFoundation.CFRunLoopTimerIsValid(timer), True)
        CoreFoundation.CFRunLoopTimerInvalidate(timer)
        self.assertIs(CoreFoundation.CFRunLoopTimerIsValid(timer), False)
        self.assertIs(CoreFoundation.CFRunLoopTimerGetContext(timer, None), objc.NULL)
        timer = CoreFoundation.CFRunLoopTimerCreate(None, 0, 0.5, 0, 0, callback, data)
        self.assertIs(
            CoreFoundation.CFRunLoopContainsTimer(rl, timer, runloop_mode), False
        )
        CoreFoundation.CFRunLoopAddTimer(rl, timer, runloop_mode)
        self.assertIs(
            CoreFoundation.CFRunLoopContainsTimer(rl, timer, runloop_mode), True
        )
        res = CoreFoundation.CFRunLoopRunInMode(runloop_mode, 1.3, True)
        self.assertIsInstance(res, int)

        CoreFoundation.CFRunLoopTimerInvalidate(timer)
        CoreFoundation.CFRunLoopRemoveTimer(rl, timer, runloop_mode)
        self.assertIs(
            CoreFoundation.CFRunLoopContainsTimer(rl, timer, runloop_mode), False
        )
        self.assertFalse(len(state) < 3)
        for item in state:
            self.assertIs(item[0], timer)
            self.assertIs(item[1], data)

    def testSource(self):
        runloop_mode = CoreFoundation.kCFRunLoopDefaultMode
        runloop_mode = "pyobjctest.cfrunloop"

        rl = CoreFoundation.CFRunLoopGetCurrent()

        state = []
        data = {}

        def schedule(info, rl, mode):
            state.append(["schedule", info, rl, mode])

        def cancel(info, rl, mode):
            state.append(["cancel", info, rl, mode])

        def perform(info):
            state.append(["perform", info])

        source = CoreFoundation.CFRunLoopSourceCreate(
            None, 55, (0, schedule, cancel, perform, data)
        )
        self.assertIsInstance(source, CoreFoundation.CFRunLoopSourceRef)
        ctx = CoreFoundation.CFRunLoopSourceGetContext(source, None)
        self.assertIsInstance(ctx, tuple)
        self.assertEqual(ctx[0], 0)
        self.assertEqual(ctx[1], schedule)
        self.assertEqual(ctx[2], cancel)
        self.assertEqual(ctx[3], perform)
        self.assertEqual(ctx[4], data)

        self.assertEqual(CoreFoundation.CFRunLoopSourceGetOrder(source), 55)
        self.assertIs(CoreFoundation.CFRunLoopSourceIsValid(source), True)
        CoreFoundation.CFRunLoopSourceInvalidate(source)
        self.assertIs(CoreFoundation.CFRunLoopSourceIsValid(source), False)
        source = CoreFoundation.CFRunLoopSourceCreate(
            None, 55, (0, schedule, cancel, perform, data)
        )
        self.assertIsInstance(source, CoreFoundation.CFRunLoopSourceRef)
        self.assertIs(
            CoreFoundation.CFRunLoopContainsSource(rl, source, runloop_mode), False
        )
        CoreFoundation.CFRunLoopAddSource(rl, source, runloop_mode)
        self.assertIs(
            CoreFoundation.CFRunLoopContainsSource(rl, source, runloop_mode), True
        )
        self.assertEqual(len(state), 1)
        self.assertEqual(state[0][0], "schedule")
        self.assertIs(state[0][1], data)
        self.assertIs(state[0][2], rl)
        self.assertEqual(state[0][3], runloop_mode)
        del state[:]

        res = CoreFoundation.CFRunLoopRunInMode(runloop_mode, 0.5, True)
        self.assertIsInstance(res, int)
        # self.assertEqual(res, CoreFoundation.kCFRunLoopRunTimedOut)

        self.assertEqual(len(state), 0)

        CoreFoundation.CFRunLoopSourceSignal(source)

        res = CoreFoundation.CFRunLoopRunInMode(runloop_mode, 0.5, True)
        self.assertIsInstance(res, int)
        self.assertEqual(res, CoreFoundation.kCFRunLoopRunHandledSource)

        self.assertEqual(len(state), 1)
        self.assertEqual(state[0][0], "perform")
        self.assertIs(state[0][1], data)
        del state[:]

        CoreFoundation.CFRunLoopRemoveSource(rl, source, runloop_mode)
        self.assertIs(
            CoreFoundation.CFRunLoopContainsSource(rl, source, runloop_mode), False
        )
        self.assertEqual(len(state), 1)
        self.assertEqual(state[0][0], "cancel")
        self.assertIs(state[0][1], data)
        self.assertIs(state[0][2], rl)
        self.assertEqual(state[0][3], runloop_mode)

    @skipUnless(
        not (
            os_level_key("10.13") <= os_level_key(os_release()) < os_level_key("10.15")
        ),
        "Crash on 10.13, 10.14??",
    )
    @min_os_level("10.6")
    def testFunctions10_6(self):
        self.assertArgIsBlock(CoreFoundation.CFRunLoopPerformBlock, 2, b"v")

        runloop_mode = CoreFoundation.kCFRunLoopDefaultMode
        rl = CoreFoundation.CFRunLoopGetCurrent()

        lst = []

        def doit():
            lst.append(True)

        CoreFoundation.CFRunLoopPerformBlock(rl, runloop_mode, doit)
        res = CoreFoundation.CFRunLoopRunInMode(runloop_mode, 0.5, True)
        self.assertIsInstance(res, int)

        self.assertEqual(lst, [True])

    @skipUnless(
        not (
            os_level_key("10.13") <= os_level_key(os_release()) < os_level_key("10.15")
        ),
        "Crash on 10.13, 10.14??",
    )
    @min_os_level("10.7")
    def testFunctions10_7(self):
        self.assertArgIsBOOL(CoreFoundation.CFRunLoopObserverCreateWithHandler, 2)
        self.assertArgIsBlock(
            CoreFoundation.CFRunLoopObserverCreateWithHandler,
            4,
            b"v^{__CFRunLoopObserver=}" + objc._C_NSUInteger,
        )

        lst = []

        def record(observer, activity):
            lst.append((observer, activity))

        ref = CoreFoundation.CFRunLoopObserverCreateWithHandler(
            None, CoreFoundation.kCFRunLoopAllActivities, False, 0, record
        )
        self.assertIsInstance(ref, CoreFoundation.CFRunLoopObserverRef)

        runloop_mode = CoreFoundation.kCFRunLoopDefaultMode
        rl = CoreFoundation.CFRunLoopGetCurrent()

        CoreFoundation.CFRunLoopAddObserver(rl, ref, runloop_mode)
        res = CoreFoundation.CFRunLoopRunInMode(runloop_mode, 0.5, True)
        CoreFoundation.CFRunLoopRemoveObserver(rl, ref, runloop_mode)

        self.assertNotEqual(lst, [])
        for a, b in lst:
            self.assertEqual(a, ref)
            self.assertIsInstance(b, int)

        self.assertArgIsBlock(
            CoreFoundation.CFRunLoopTimerCreateWithHandler, 5, b"v^{__CFRunLoopTimer=}"
        )
        lst = []
        ref = CoreFoundation.CFRunLoopTimerCreateWithHandler(
            None,
            CoreFoundation.CFAbsoluteTimeGetCurrent() + 2.9,
            0.0,
            0,
            0,
            lambda x: lst.append(x),
        )
        self.assertIsInstance(ref, CoreFoundation.CFRunLoopTimerRef)

        CoreFoundation.CFRunLoopAddTimer(rl, ref, runloop_mode)
        res = CoreFoundation.CFRunLoopRunInMode(runloop_mode, 6.0, True)
        self.assertIsInstance(res, int)
        CoreFoundation.CFRunLoopRemoveTimer(rl, ref, runloop_mode)

        import __main__

        # XXX: See issue #11 in the pyobjc tracker
        if "setup" in __main__.__file__:
            return

        self.assertNotEqual(lst, [])
        for a in lst:
            self.assertEqual(a, ref)

    @min_os_level("10.9")
    def testFunctions10_9(self):
        lst = []
        ref = CoreFoundation.CFRunLoopTimerCreateWithHandler(
            None,
            CoreFoundation.CFAbsoluteTimeGetCurrent() + 2.9,
            0.0,
            0,
            0,
            lambda x: lst.append(x),
        )
        self.assertIsInstance(ref, CoreFoundation.CFRunLoopTimerRef)
        CoreFoundation.CFRunLoopTimerSetTolerance(ref, 5.0)
        v = CoreFoundation.CFRunLoopTimerGetTolerance(ref)
        self.assertIsInstance(v, float)


#Code to find minimum and maximum in an array with the help of for-loop

def minmax(arr):
    n=len(arr)

    if n==1:
        min=arr[0]
        max=arr[0]
    elif n==2:
        if arr[0]>arr[1]:
            min=arr[1]
            max=arr[0]
        else:
            max=arr[1]
            min=arr[0]
    else:
        min=arr[0]
        max=arr[0]
        for i in range(0,n):
            if arr[i]>max:
                max=arr[i]
            if arr[i]<min:
                min=arr[i]

    return min, max

test_arr=[1,23,4,5,6,8]
print(test_arr)
print(minmax(test_arr))


from openfisca_aotearoa.entities import Person, Family
from openfisca_core.model_api import *

class social_security__eligible_for_orphans_benefit(Variable):
    value_type = bool
    entity = Person
    definition_period = MONTH
    label = u"Eligible for Orphan's benefit"
    reference = """http://www.legislation.govt.nz/act/public/1964/0136/latest/whole.html#DLM361606
        Orphans' benefits
        A person who is a principal caregiver in respect of a dependent child shall be entitled to receive an orphan's benefit in respect of that child if
        (a) each of the child's natural or adoptive parents is dead, or cannot be found, or suffers a serious long-term disablement which renders him or her unable to care for the child and
        (b) the applicant is likely to be the principal caregiver in respect of the child for at least 1 year from the date of application for the benefit; and
        (c) the applicant is aged 18 years or over; and
        (d) either
        (i) the child is both resident and present in New Zealand or
        (ii) the applicant has been both resident and present in New Zealand for a continuous period of 12 months at any time.
    """

    def formula(persons, period, parameters):
        resident_or_citizen = persons('is_citizen_or_resident', period)
        normally_in_nz = persons("social_security__is_ordinarily_resident_in_new_zealand", period)

        age_test = persons('age', period.start) >= 18

        not_the_parent = not_(
            persons('social_security__is_the_parent_of_dependent_child', period))
        one_year = persons(
            'social_security__is_principal_carer_for_one_year_from_application_date', period)

        is_principal_carer = persons.has_role(Family.PRINCIPAL_CAREGIVER)

        has_orphaned_child_in_family = persons.family(
            'social_security__has_orphaned_child_in_family', period)

        return resident_or_citizen * normally_in_nz * age_test * not_the_parent * one_year * is_principal_carer * has_orphaned_child_in_family

class social_security__has_orphaned_child_in_family(Variable):
    value_type = bool
    entity = Family
    definition_period = MONTH
    reference = "http://www.legislation.govt.nz/act/public/1964/0136/latest/whole.html#DLM361606"
    label = "Family is caring for an orphan as per Social Security Act 1964"

    def formula(families, period, parameters):
        children = families.members(
            'social_security__is_a_child', period)
        orphaned = families.members('social_security__is_orphaned', period)
        resident_or_citizen = families.members(
            'is_citizen_or_resident', period)

        return families.any((children * orphaned * resident_or_citizen), role=Family.CHILD)

class social_security__is_orphaned(Variable):
    value_type = bool
    entity = Person
    definition_period = MONTH
    label = u"each of the child's natural or adoptive parents is dead, or cannot be found, or suffers a serious long-term disablement which renders him or her unable to care for the child"


import email
from email.header import decode_header
import re
from resources.sql_manager import Alerts
from resources.login import Login

class Parse:
    def __init__(self) -> None:
        self.LG = Login()
        self.AL = Alerts()

    @property
    def info(self) -> bool:
        return self.LG.username

    @property
    def messages(self) -> bool:
        return self.LG.messages()

    def __del__(self) -> None:
        print(f"{__name__} being deleted")

    def convert(self, time):
        if int(time[:2]) < 12:
            return time + " AM"
        else:
            if int(time[:2]) == 12:
                return time + " PM"
            elif (int(time[:2])-12) < 10:
                return f"0{(int(time[:2])-12)}{time[2:]} PM"
            return f"{(int(time[:2])-12)}{time[2:]} PM"

    def extraction(self, body, ind):
        """Function to extract and clean alerts, based on meteocontrol bluelog
        firmware version. if ind is 3 its an error"""

        #Using regular expresion to extract coincidences
        if ind == 1:
            time, description, state, device, name = re.findall("(?<=Timestamp: )(.*?)(?= \r)", body), \
            re.findall("(?<=Description: )(.*?)(?= \r)", body), re.findall("(?<=State: )(.*?)(?= \r)", body), \
                re.findall("(?<=Device: )(.*?)(?= \r)", body), re.findall("(?<=Plant name: )(.*)", body)
        elif ind == 2:
            time, description, state, device, name = [(body.split("\r\n\r\n")[0].split("\r\n")[0])], \
                ["".join(re.findall("(?<=Code: )(.*?)(?=\r)", body)), \
                "".join(re.findall("(?<=Additional information: )(.*)", body))] , \
                [None], [re.findall("(?<=Logger name: )(.*?)(?=\r)", body), re.findall("(?<=Device: )(.*?)(?=\r)", body)], \
                    re.findall("(?<=Plant name: )(.*?)(?=\r)", body)

            # Joining multiples information due to old firmware version
            time = time[0].replace(" ", ", ").split("-")
            time = [f"{time[1]}/{time[2].split(', ')[0]}/{time[0]}, {self.convert(time[2].split(', ')[1])}"]
            description = [" ".join(description)]
            device = [": ".join([" ".join(list_) for list_ in device])]

        else:
            time = state = device = name = 404
            description = body

        return(time, description, state, device, name)

    def storing(self, info_pack, id):
        """Temporary non-formatted way of storing"""

        time, description, state, device, name = info_pack
        print(name)

        self.AL.storing_alerts(id, "".join(time), description,
                                state, device, name)

    def clasifing(self, body):
        """Function to clasify the alert base on the firmware version."""
        # eliminating useless information
        body = body.split("This is an automatically generated message")[0].strip()

        amount_multiple_message_body = body.split("\r\n\r\n")
        raw_id = amount_multiple_message_body[0]

        if "sn: " in raw_id:
            id, ind = int(raw_id.split("sn: ")[1].rsplit(")")[0]), 1
        elif "The blue'Log with the serial number" in raw_id:
            id, ind = "".join(re.findall("([0-9]+)", raw_id)), 2
        else:
            id, ind = "error", 3
        return(id, ind, amount_multiple_message_body)

    def sub_main(self, body, debug=True):
        """This function controls the general flow of the program behavior
        after email body are parsed"""

        #Extrating id and amount of message on a single body
        id, ind, amount_multiple_message_body = self.clasifing(body)
        self.AL.creating_table(id)

        if debug:
            if ind == 3:
                print(body)

        # Iterating through every message on a body
        for i in range(1, len(amount_multiple_message_body)):
            info_pack = self.extraction(amount_multiple_message_body[i], ind)
            # Temporary way of storing formated data
            self.storing(info_pack, id)

    def parsing(self, total_messages, messages_to_read) -> None:
        for i in range(total_messages, total_messages-messages_to_read, -1):
            msg = self.LG.fetch(i)
            for response in msg:
                if isinstance(response, tuple):
                    # parse a bytes email into a message object
                    msg = email.message_from_bytes(response[1])
                    # decode email sender
                    from_clean = (re.findall("\S+@\S+[\w-]+\.+[\w-]{2,100}",
                        str(decode_header(msg.get("From"))))[0].split("<"))[1]
                    # just review emails comming from a particular source
                    if (from_clean.strip() == self.LG.monitoring_email):
                        # if the email message is multipart
                        if msg.is_multipart():
                            # iterate over email parts
                            for part in msg.walk():
                                # extract content type of email
                                content_type = part.get_content_type()
                                try:
                                    # get the email body
                                    body = part.get_payload(decode=True).decode()
                                except:
                                    pass
                                if content_type == "text/plain":
                                    self.sub_main(body)
                        else:
                            # extract content type of email
                            content_type = msg.get_content_type()
                            # get the email body
                            body = msg.get_payload(decode=True).decode()
                            if content_type == "text/plain":
                                self.sub_main(body)

if __name__ == "__main__":
    x = Parse()
    x.parsing(10)


import numpy as np
from PIL import Image
import glob
import cv2
from tqdm import tqdm

BASE_PATH = 'datasets/simple/dataset-0/train'

def get_stat(img):
    mean = [0,0,0]
    std = [0,0,0]
    for chanel in range(3):
        mean[chanel] = np.mean(img[:,:,chanel])/255
        std[chanel] = np.std(img[:,:,chanel])/255
    return mean, std
def get_stat_all_imgs(all_imgs):
    count = 0
    total_mean = [0,0,0]
    total_std = [0,0,0]
    for img_path in tqdm(all_imgs):
        try:
            img = cv2.imread(img_path)
            mean ,std = get_stat(img)
            total_mean = np.add(mean,total_mean)
            total_std = np.add(std,total_std)
            count += 1
        except:
            continue
    final_mean = [0,0,0]
    final_std = [0,0,0]
    for i in range(3):
        final_mean[i] = total_mean[i]/count
        final_std[i] = total_std[i]/count
    return final_mean , final_std

def main():
    dir_list = glob.glob(BASE_PATH + "/*")
    all_imgs = []
    for dir in dir_list:
        img_list = glob.glob(dir + "/*")
        all_imgs.extend(img_list)
    final_mean,final_std = get_stat_all_imgs(all_imgs)
    np.save('final_mean',final_mean)
    np.save('final_std',final_std)
    print(final_mean,final_std)

if __name__ == '__main__':
    main()


from bitresource import resource_registry
from bitutils.objects import Currency, Market, Ticker
from dictutils import AttrDict
from http_resource import HttpResource

@resource_registry.register()
class BinanceResource(HttpResource):
    name = 'binance'
    endpoint_url = 'https://www.binance.com/api'

    def results_iter(self, response, **kwargs):
        resp_json = response.json()
        result_rows = []

        if type(resp_json) is dict:
            if 'symbols' in resp_json:
                result_rows = resp_json.get('symbols')
            else:
                yield AttrDict(resp_json)

        if result_rows:
            for result_row in result_rows:
                yield AttrDict(result_row)

    @classmethod
    def get_currencies(cls):
        """
            {'icebergAllowed': True,
             'baseAssetPrecision': 8,
             'symbol': 'ETHBTC',
             'baseAsset': 'ETH',
             'quoteAsset': 'BTC',
             'status': 'TRADING',
             'quotePrecision': 8}
        """
        results = []

        for currency in CurrencyResource.data():
            currency = Currency(code=currency.quoteAsset, decimals=currency.quotePrecision)
            results.append(currency)

        return results

    @classmethod
    def get_markets(cls):
        """
            {'icebergAllowed': True,
             'baseAssetPrecision': 8,
             'symbol': 'ETHBTC',
             'baseAsset': 'ETH',
             'quoteAsset': 'BTC',
             'status': 'TRADING',
             'quotePrecision': 8}
        """
        results = []

        for market in MarketResource.data():
            market_name = '%s-%s' % (market.quoteAsset, market.baseAsset)

            market = Market(code=market_name.replace('-', ''), name=market_name, base=market.quoteAsset,
                            quote=market.baseAsset)
            results.append(market)

        return results

    @classmethod
    def ticker(cls, market, exchange):
        symbol = ''.join([market.name.split('-')[1], market.name.split('-')[0]])
        data = TickerResource.data(symbol=symbol).first()

        return Ticker(last=data.price, market=market.code, exchange=exchange)

CurrencyResource = BinanceResource('v1', 'exchangeInfo')

MarketResource = BinanceResource('v1', 'exchangeInfo')

TickerResource = BinanceResource('v3', 'ticker/price')