import pandas as pd
import os
import torch
from pathlib import Path
import pickle
import logging

import shutil

from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
from torch.utils.data.distributed import DistributedSampler

from transformers import (
    WEIGHTS_NAME,
    BertConfig,
    BertForSequenceClassification,
    BertTokenizer,
    XLMConfig,
    XLMForSequenceClassification,
    XLMTokenizer,
    XLNetConfig,
    XLNetForSequenceClassification,
    XLNetTokenizer,
    RobertaConfig,
    RobertaForSequenceClassification,
    RobertaTokenizer,
    CamembertConfig,
    CamembertForSequenceClassification,
    CamembertTokenizer,
    AlbertConfig,
    AlbertForSequenceClassification,
    AlbertTokenizer,
    DistilBertConfig,
    DistilBertForSequenceClassification,
    DistilBertTokenizer,
)

MODEL_CLASSES = {
    "bert": (BertConfig, BertForSequenceClassification, BertTokenizer),
    "xlnet": (XLNetConfig, XLNetForSequenceClassification, XLNetTokenizer),
    "xlm": (XLMConfig, XLMForSequenceClassification, XLMTokenizer),
    "roberta": (RobertaConfig, RobertaForSequenceClassification, RobertaTokenizer),
    "albert": (AlbertConfig, AlbertForSequenceClassification, AlbertTokenizer),
    "distilbert": (DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer),
    "camembert-base": (CamembertConfig, CamembertForSequenceClassification, CamembertTokenizer)
}

class InputExample(object):
    """A single training/test example for simple sequence classification."""

    def __init__(self, guid, text_a, text_b=None, label=None):
        """Constructs a InputExample.

        Args:
            guid: Unique id for the example.
            text_a: string. The untokenized text of the first sequence. For single
            sequence tasks, only this sequence must be specified.
            text_b: (Optional) string. The untokenized text of the second sequence.
            Only must be specified for sequence pair tasks.
            labels: (Optional) [string]. The label of the example. This should be
            specified for train and dev examples, but not for test examples.
        """
        self.guid = guid
        self.text_a = text_a
        self.text_b = text_b
        if isinstance(label, list):
            self.label = label
        elif label:
            self.label = str(label)
        else:
            self.label = None

class InputFeatures(object):
    """A single set of features of data."""

    def __init__(self, input_ids, input_mask, segment_ids, label_id):
        self.input_ids = input_ids
        self.input_mask = input_mask
        self.segment_ids = segment_ids
        self.label_id = label_id

def _truncate_seq_pair(tokens_a, tokens_b, max_length):
    """Truncates a sequence pair in place to the maximum length."""

    # This is a simple heuristic which will always truncate the longer sequence
    # one token at a time. This makes more sense than truncating an equal percent
    # of tokens from each, since if one sequence is very short then each token
    # that's truncated likely contains more information than a longer sequence.
    while True:
        total_length = len(tokens_a) + len(tokens_b)
        if total_length <= max_length:
            break
        if len(tokens_a) > len(tokens_b):
            tokens_a.pop()
        else:
            tokens_b.pop()

def convert_examples_to_features(
    examples,
    label_list,
    max_seq_length,
    tokenizer,
    output_mode="classification",
    cls_token_at_end=False,
    pad_on_left=False,
    cls_token="[CLS]",
    sep_token="[SEP]",
    pad_token=0,
    sequence_a_segment_id=0,
    sequence_b_segment_id=1,
    cls_token_segment_id=1,
    pad_token_segment_id=0,
    mask_padding_with_zero=True,
    logger=None,
):
    """ Loads a data file into a list of `InputBatch`s
        `cls_token_at_end` define the location of the CLS token:
            - False (Default, BERT/XLM pattern): [CLS] + A + [SEP] + B + [SEP]
            - True (XLNet/GPT pattern): A + [SEP] + B + [SEP] + [CLS]
        `cls_token_segment_id` define the segment id associated to the CLS token (0 for BERT, 2 for XLNet)
    """

    label_map = {label: i for i, label in enumerate(label_list)}

    features = []
    for (ex_index, example) in enumerate(examples):
        if ex_index % 10000 == 0:
            if logger:
                logger.info("Writing example %d of %d" % (ex_index, len(examples)))

        tokens_a = tokenizer.tokenize(example.text_a)

        tokens_b = None
        if example.text_b:
            tokens_b = tokenizer.tokenize(example.text_b)
            # Modifies `tokens_a` and `tokens_b` in place so that the total
            # length is less than the specified length.
            # Account for [CLS], [SEP], [SEP] with "- 3"
            _truncate_seq_pair(tokens_a, tokens_b, max_seq_length - 3)
        else:
            # Account for [CLS] and [SEP] with "- 2"
            if len(tokens_a) > max_seq_length - 2:
                tokens_a = tokens_a[: (max_seq_length - 2)]

        # The convention in BERT is:
        # (a) For sequence pairs:
        #  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
        #  type_ids:   0   0  0    0    0     0       0   0   1  1  1  1   1   1
        # (b) For single sequences:
        #  tokens:   [CLS] the dog is hairy . [SEP]
        #  type_ids:   0   0   0   0  0     0   0
        #
        # Where "type_ids" are used to indicate whether this is the first
        # sequence or the second sequence. The embedding vectors for `type=0` and
        # `type=1` were learned during pre-training and are added to the wordpiece
        # embedding vector (and position vector). This is not *strictly* necessary
        # since the [SEP] token unambiguously separates the sequences, but it makes
        # it easier for the model to learn the concept of sequences.
        #
        # For classification tasks, the first vector (corresponding to [CLS]) is
        # used as as the "sentence vector". Note that this only makes sense because
        # the entire model is fine-tuned.
        tokens = tokens_a + [sep_token]
        segment_ids = [sequence_a_segment_id] * len(tokens)

        if tokens_b:
            tokens += tokens_b + [sep_token]
            segment_ids += [sequence_b_segment_id] * (len(tokens_b) + 1)

        if cls_token_at_end:
            tokens = tokens + [cls_token]
            segment_ids = segment_ids + [cls_token_segment_id]
        else:
            tokens = [cls_token] + tokens
            segment_ids = [cls_token_segment_id] + segment_ids

        input_ids = tokenizer.convert_tokens_to_ids(tokens)

        # The mask has 1 for real tokens and 0 for padding tokens. Only real
        # tokens are attended to.
        input_mask = [1 if mask_padding_with_zero else 0] * len(input_ids)

        # Zero-pad up to the sequence length.
        padding_length = max_seq_length - len(input_ids)
        if pad_on_left:
            input_ids = ([pad_token] * padding_length) + input_ids
            input_mask = (
                [0 if mask_padding_with_zero else 1] * padding_length
            ) + input_mask
            segment_ids = ([pad_token_segment_id] * padding_length) + segment_ids
        else:
            input_ids = input_ids + ([pad_token] * padding_length)
            input_mask = input_mask + (
                [0 if mask_padding_with_zero else 1] * padding_length
            )
            segment_ids = segment_ids + ([pad_token_segment_id] * padding_length)

        assert len(input_ids) == max_seq_length
        assert len(input_mask) == max_seq_length
        assert len(segment_ids) == max_seq_length

        if isinstance(example.label, list):
            label_id = []
            for label in example.label:
                label_id.append(float(label))
        else:
            if example.label is not None:
                label_id = label_map[example.label]
            else:
                label_id = ""

        features.append(
            InputFeatures(
                input_ids=input_ids,
                input_mask=input_mask,
                segment_ids=segment_ids,
                label_id=label_id,
            )
        )
    return features

class DataProcessor(object):
    """Base class for data converters for sequence classification data sets."""

    def get_train_examples(self, filename, size=-1):
        """Gets a collection of `InputExample`s for the train set."""
        raise NotImplementedError()

    def get_dev_examples(self, filename, size=-1):
        """Gets a collection of `InputExample`s for the dev set."""
        raise NotImplementedError()

    def get_test_examples(self, filename, size=-1):
        """Gets a collection of `InputExample`s for the dev set."""
        raise NotImplementedError()

    def get_labels(self):
        """Gets the list of labels for this data set."""
        raise NotImplementedError()

class TextProcessor(DataProcessor):
    def __init__(self, data_dir, label_dir):
        self.data_dir = data_dir
        self.label_dir = label_dir
        self.labels = None

    def get_train_examples(
        self, filename="train.csv", text_col="text", label_col="label", size=-1
    ):

        if size == -1:
            data_df = pd.read_csv(os.path.join(self.data_dir, filename))

            return self._create_examples(
                data_df, "train", text_col=text_col, label_col=label_col
            )
        else:
            data_df = pd.read_csv(os.path.join(self.data_dir, filename))
            #             data_df['comment_text'] = data_df['comment_text'].apply(cleanHtml)
            return self._create_examples(
                data_df.sample(size), "train", text_col=text_col, label_col=label_col
            )

    def get_dev_examples(
        self, filename="val.csv", text_col="text", label_col="label", size=-1
    ):

        if size == -1:
            data_df = pd.read_csv(os.path.join(self.data_dir, filename))
            return self._create_examples(
                data_df, "dev", text_col=text_col, label_col=label_col
            )
        else:
            data_df = pd.read_csv(os.path.join(self.data_dir, filename))
            return self._create_examples(
                data_df.sample(size), "dev", text_col=text_col, label_col=label_col
            )

    def get_test_examples(
        self, filename="val.csv", text_col="text", label_col="label", size=-1
    ):
        data_df = pd.read_csv(os.path.join(self.data_dir, filename))
        #         data_df['comment_text'] = data_df['comment_text'].apply(cleanHtml)
        if size == -1:
            return self._create_examples(
                data_df, "test", text_col=text_col, label_col=None
            )
        else:
            return self._create_examples(
                data_df.sample(size), "test", text_col=text_col, label_col=None
            )

    def get_labels(self, filename="labels.csv"):
        """See base class."""
        if self.labels is None:
            self.labels = list(
                pd.read_csv(os.path.join(self.label_dir, filename), header=None)[0]
                .astype("str")
                .values
            )
        return self.labels

    def _create_examples(self, df, set_type, text_col, label_col):
        """Creates examples for the training and dev sets."""
        if label_col is None:
            return list(
                df.apply(
                    lambda row: InputExample(
                        guid=row.index, text_a=str(row[text_col]), label=None
                    ),
                    axis=1,
                )
            )
        else:
            return list(
                df.apply(
                    lambda row: InputExample(
                        guid=row.index, text_a=str(row[text_col]), label=str(row[label_col])
                    ),
                    axis=1,
                )
            )

class MultiLabelTextProcessor(TextProcessor):
    def _create_examples(self, df, set_type, text_col, label_col):
        def _get_labels(row, label_col):
            if isinstance(label_col, list):
                return list(row[label_col])
            else:
                # create one hot vector of labels
                label_list = self.get_labels()
                labels = [0] * len(label_list)
                labels[label_list.index(row[label_col])] = 1
                return labels

        """Creates examples for the training and dev sets."""
        if label_col is None:
            return list(
                df.apply(
                    lambda row: InputExample(
                        guid=row.index, text_a=row[text_col], label=[]
                    ),
                    axis=1,
                )
            )
        else:
            return list(
                df.apply(
                    lambda row: InputExample(
                        guid=row.index,
                        text_a=row[text_col],
                        label=_get_labels(row, label_col),
                    ),
                    axis=1,
                )
            )

class BertDataBunch(object):
    def __init__(
        self,
        data_dir,
        label_dir,
        tokenizer,
        train_file="train.csv",
        val_file="val.csv",
        test_data=None,
        label_file="labels.csv",
        text_col="text",
        label_col="label",
        batch_size_per_gpu=16,
        max_seq_length=512,
        multi_gpu=True,
        multi_label=False,
        backend="nccl",
        model_type="bert",
        logger=None,
        clear_cache=False,
        no_cache=False,
    ):

        # just in case someone passes string instead of Path
        if isinstance(data_dir, str):
            data_dir = Path(data_dir)

        if isinstance(label_dir, str):
            label_dir = Path(label_dir)

        if isinstance(tokenizer, str):
            _, _, tokenizer_class = MODEL_CLASSES[model_type]
            # instantiate the new tokeniser object using the tokeniser name
            tokenizer = tokenizer_class.from_pretrained(
                tokenizer, do_lower_case=("uncased" in tokenizer)
            )

        self.tokenizer = tokenizer
        self.data_dir = data_dir
        self.train_file = train_file
        self.val_file = val_file
        self.test_data = test_data
        self.cache_dir = data_dir / "cache"
        self.max_seq_length = max_seq_length
        self.batch_size_per_gpu = batch_size_per_gpu
        self.train_dl = None
        self.val_dl = None
        self.test_dl = None
        self.multi_label = multi_label
        self.n_gpu = 1
        self.no_cache = no_cache
        self.model_type = model_type
        self.output_mode = "classification"
        if logger is None:
            logger = logging.getLogger()
        self.logger = logger
        if multi_gpu:
            self.n_gpu = torch.cuda.device_count()

        if clear_cache:
            shutil.rmtree(self.cache_dir, ignore_errors=True)

        if multi_label:
            processor = MultiLabelTextProcessor(data_dir, label_dir)
        else:
            processor = TextProcessor(data_dir, label_dir)

        self.labels = processor.get_labels(label_file)

        if train_file:
            # Train DataLoader
            train_examples = None
            cached_features_file = os.path.join(
                self.cache_dir,
                "cached_{}_{}_{}_{}_{}".format(
                    self.model_type,
                    "train",
                    "multi_label" if self.multi_label else "multi_class",
                    str(self.max_seq_length),
                    os.path.basename(train_file),
                ),
            )

            if os.path.exists(cached_features_file) is False or self.no_cache is True:
                train_examples = processor.get_train_examples(
                    train_file, text_col=text_col, label_col=label_col
                )

            train_dataset = self.get_dataset_from_examples(
                train_examples, "train", no_cache=self.no_cache
            )

            self.train_batch_size = self.batch_size_per_gpu * max(1, self.n_gpu)
            train_sampler = RandomSampler(train_dataset)
            self.train_dl = DataLoader(
                train_dataset, sampler=train_sampler, batch_size=self.train_batch_size
            )

        if val_file:
            # Validation DataLoader
            val_examples = None
            cached_features_file = os.path.join(
                self.cache_dir,
                "cached_{}_{}_{}_{}_{}".format(
                    self.model_type,
                    "dev",
                    "multi_label" if self.multi_label else "multi_class",
                    str(self.max_seq_length),
                    os.path.basename(val_file),
                ),
            )

            if os.path.exists(cached_features_file) is False:
                val_examples = processor.get_dev_examples(
                    val_file, text_col=text_col, label_col=label_col
                )

            val_dataset = self.get_dataset_from_examples(
                val_examples, "dev", no_cache=self.no_cache
            )

            # no grads necessary, hence double val batch size
            self.val_batch_size = self.batch_size_per_gpu * 2 * max(1, self.n_gpu)
            val_sampler = SequentialSampler(val_dataset)
            self.val_dl = DataLoader(
                val_dataset, sampler=val_sampler, batch_size=self.val_batch_size
            )

        if test_data:
            # Test set loader for predictions
            test_examples = []
            input_data = []

            for index, text in enumerate(test_data):
                test_examples.append(InputExample(index, text))
                input_data.append({"id": index, "text": text})

            test_dataset = self.get_dataset_from_examples(
                test_examples, "test", is_test=True, no_cache=self.no_cache
            )

            self.test_batch_size = self.batch_size_per_gpu * max(1, self.n_gpu)
            test_sampler = SequentialSampler(test_dataset)
            self.test_dl = DataLoader(
                test_dataset, sampler=test_sampler, batch_size=self.test_batch_size
            )

    def get_dl_from_texts(self, texts):

        test_examples = []
        input_data = []

        for index, text in enumerate(texts):
            test_examples.append(InputExample(index, text, label=None))
            input_data.append({"id": index, "text": text})

        test_dataset = self.get_dataset_from_examples(
            test_examples, "test", is_test=True, no_cache=True
        )

        test_sampler = SequentialSampler(test_dataset)
        return DataLoader(
            test_dataset, sampler=test_sampler, batch_size=self.batch_size_per_gpu
        )

    def save(self, filename="databunch.pkl"):
        tmp_path = self.data_dir / "tmp"
        tmp_path.mkdir(exist_ok=True)
        with open(str(tmp_path / filename), "wb") as f:
            pickle.dump(self, f)

    def get_dataset_from_examples(
        self, examples, set_type="train", is_test=False, no_cache=False
    ):

        if set_type == "train":
            file_name = self.train_file
        elif set_type == "dev":
            file_name = self.val_file
        elif set_type == "test":
            file_name = (
                "test"
            )  # test is not supposed to be a file - just a list of texts

        cached_features_file = os.path.join(
            self.cache_dir,
            "cached_{}_{}_{}_{}_{}".format(
                self.model_type,
                set_type,
                "multi_label" if self.multi_label else "multi_class",
                str(self.max_seq_length),
                os.path.basename(file_name),
            ),
        )

        if os.path.exists(cached_features_file) and no_cache is False:
            self.logger.info(
                "Loading features from cached file %s", cached_features_file
            )
            features = torch.load(cached_features_file)
        else:
            # Create tokenized and numericalized features
            features = convert_examples_to_features(
                examples,
                label_list=self.labels,
                max_seq_length=self.max_seq_length,
                tokenizer=self.tokenizer,
                output_mode=self.output_mode,
                # xlnet has a cls token at the end
                cls_token_at_end=bool(self.model_type in ["xlnet"]),
                cls_token=self.tokenizer.cls_token,
                sep_token=self.tokenizer.sep_token,
                cls_token_segment_id=2 if self.model_type in ["xlnet"] else 0,
                # pad on the left for xlnet
                pad_on_left=bool(self.model_type in ["xlnet"]),
                pad_token_segment_id=4 if self.model_type in ["xlnet"] else 0,
                logger=self.logger,
            )

            # Create folder if it doesn't exist
            if no_cache is False:
                self.cache_dir.mkdir(exist_ok=True)
                self.logger.info(
                    "Saving features into cached file %s", cached_features_file
                )
                torch.save(features, cached_features_file)

        # Convert to Tensors and build dataset
        all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
        all_input_mask = torch.tensor(
            [f.input_mask for f in features], dtype=torch.long
        )
        all_segment_ids = torch.tensor(
            [f.segment_ids for f in features], dtype=torch.long
        )

        if is_test is False:  # labels not available for test set
            if self.multi_label:
                all_label_ids = torch.tensor(
                    [f.label_id for f in features], dtype=torch.float
                )
            else:
                all_label_ids = torch.tensor(
                    [f.label_id for f in features], dtype=torch.long
                )

            dataset = TensorDataset(
                all_input_ids, all_input_mask, all_segment_ids, all_label_ids
            )
        else:
            all_label_ids = []
            dataset = TensorDataset(all_input_ids, all_input_mask, all_segment_ids)

        return dataset


import pickle, argparse, os, sys, bcolz
from sklearn.metrics import accuracy_score
import numpy as np
import random
import torch
import torch.nn as nn
import torch.functional as F
from torch.nn.modules import loss
import torch.optim as optim
import re

def preprocess(training_file, glove_path = '.'):
    # First read in the file and preprocess:
    with open(training_file, 'rb') as f:
        contents = f.readlines()

    words = []
    tags = []
    vocab = {}
    uniq_word = {}
    uniq_tag = {}
    tag_idx = 1
    uniq_tag['<PAD>'] = 0
    uniq_word['<PAD>'] = 0
    for k, sentence in enumerate(contents):
        # omit the last '\n' and the first 'b''
        sentence = str(sentence)
        sentence = sentence[2:-3]
        # get rid of possible lines starting with space
        i = 0
        while sentence[i] == ' ':
            i += 1
        sentence = sentence[i:]
        parsed_sen = re.split(r'\s+', sentence)
        sub_tags = []
        for i in range(len(parsed_sen)//2):
            # vocab counts the word apprearance in the training set, to determine which words should be labelled as "UNKA"
            if parsed_sen[2*i] not in vocab:
                vocab[parsed_sen[2*i]] = 1
                # word_idx += 1
                # sub_words.append(vocab[parsed_sen[2*i]])
            else:
                vocab[parsed_sen[2*i]] += 1
            # uniq_tag assigns every tag a number, tags saves each tag sequence
            # print(k, i, parsed_sen[2*i+1])
            if parsed_sen[2*i+1] not in uniq_tag:
                uniq_tag[parsed_sen[2*i+1]] = tag_idx
                tag_idx += 1
                sub_tags.append(uniq_tag[parsed_sen[2*i+1]])

            else:
                sub_tags.append(uniq_tag[parsed_sen[2*i+1]])
        # append the tag sequence to the tags
        tags.append(sub_tags)
    uniq_word['UNKA'] = 1
    word_idx = 2
    for sentence in contents:
        sentence = str(sentence)
        sentence = sentence[2:-3]
        # get rid of possible lines starting with space
        i = 0
        while sentence[i] == ' ':
            i += 1
        sentence = sentence[i:]
        parsed_sen = re.split(r'\s+',sentence)
        sub_words = []
        for i in range(len(parsed_sen)//2):
            if vocab[parsed_sen[2*i]] <= 3:
                sub_words.append(uniq_word['UNKA'])
            elif parsed_sen[2*i] not in uniq_word:
                uniq_word[parsed_sen[2*i]] = word_idx
                word_idx += 1
                sub_words.append(uniq_word[parsed_sen[2*i]])
            else:
                sub_words.append(uniq_word[parsed_sen[2*i]])

        words.append(sub_words)

    # now create the embedding matrix using GloVe pre-trained model data
    # read the pre-stored GloVe files
    vectors = bcolz.open(f'{glove_path}/6B.50.dat')[:]
    glove_words = pickle.load(open(f'{glove_path}/6B.50_words.pkl', 'rb'))
    word2idx = pickle.load(open(f'{glove_path}/6B.50_idx.pkl', 'rb'))

    glove = {w: vectors[word2idx[w]] for w in glove_words}
    emb_dim = 50
    matrix_len = len(uniq_word)
    weights_matrix = np.zeros((matrix_len, 50))
    words_found = 0

    for word in uniq_word:
        try:
            weights_matrix[uniq_word[word]] = glove[word]
            words_found += 1
        except KeyError:
            weights_matrix[uniq_word[word]] = np.random.normal(scale=0.6, size=(emb_dim, ))

    return words, tags, uniq_word, uniq_tag, weights_matrix

class RNNTagger(nn.Module):
    def __init__(self, training_file, embedding_dim = 50, hidden_dim = 100):
        super(RNNTagger, self).__init__()
        self.hidden_dim = hidden_dim
        _, _, self.vocab, self.tags, self.weights_matrix = preprocess(training_file)
        n_vocab_words = len(self.vocab)
        padding_idx = self.vocab['<PAD>']
        # embedding layer, use the pretrained weight matrix, make sure the weights are not trainable
        # let the embedding know that padding is '0'
        self.word_embedding = nn.Embedding(num_embeddings=n_vocab_words,embedding_dim=embedding_dim,padding_idx=padding_idx)
        self.word_embedding.weight.data.copy_(torch.from_numpy(self.weights_matrix))
        self.word_embedding.weight.requires_grad = False
        # The LSTM takes word embeddings as inputs, and outputs hidden states
        # with dimensionality hidden_dim.
        self.lstm = nn.LSTM(embedding_dim, hidden_dim)

        # The linear layer that maps from hidden state space to tag space
        self.hidden2tag = nn.Linear(hidden_dim, len(self.tags))
        self.logsoftmax = nn.LogSoftmax(dim=-1)
    def forward(self, sentence):
        embeds = self.word_embedding(sentence)
        # input sentence here is already the embedded sentence, batch_size*len(sentence)*embedding_dim
        lstm_out, _ = self.lstm(embeds.view(len(sentence), 1, -1))
        tag_space = self.hidden2tag(lstm_out.view(len(sentence), -1))
        tag_scores = self.logsoftmax(tag_space)
        return tag_scores

def train(training_file):
    assert os.path.isfile(training_file), 'Training file does not exist'

    # Your code starts here
    # check whether the GloVe embedding is already processed, if not, process it.
    glove_path = '.'
    if not os.path.isfile(f'{glove_path}/6B.50_words.pkl'):
        words = []
        idx = 0
        word2idx = {}
        vectors = bcolz.carray(np.zeros(1), rootdir=f'{glove_path}/6B.50.dat', mode='w')

        with open(f'{glove_path}/glove.6B.50d.txt', 'rb') as f:
            for l in f:
                line = l.decode().split()
                word = line[0]
                words.append(word)
                word2idx[word] = idx
                idx += 1
                vect = np.array(line[1:]).astype(np.float)
                vectors.append(vect)

        vectors = bcolz.carray(vectors[1:].reshape((400000, 50)), rootdir=f'{glove_path}/6B.50.dat', mode='w')
        vectors.flush()
        pickle.dump(words, open(f'{glove_path}/6B.50_words.pkl', 'wb'))
        pickle.dump(word2idx, open(f'{glove_path}/6B.50_idx.pkl', 'wb'))

    training_X, training_Y, _, _, _ = preprocess(training_file)
    model = RNNTagger(training_file, hidden_dim=100)
    loss_function = nn.NLLLoss()
    optimizer = optim.SGD(model.parameters(), lr = 0.05)

    with torch.no_grad():
        inputs = torch.tensor(training_X[0], dtype=torch.long)
        tag_scores = model(inputs)
        print(tag_scores)

    for j in range(10):
        print("current training epoch #", j)
        for i in range(len(training_X)):
            # clear out the gradients from last step
            model.zero_grad()

            tag_scores = model(torch.tensor(training_X[i], dtype=torch.long))

            loss = loss_function(tag_scores, torch.tensor(training_Y[i], dtype=torch.long))
            if i%2000 == 0:
                print("current loss is: ", loss)
            loss.backward()
            optimizer.step()
    # Your code ends here

    return model

def test(model_file, data_file, label_file):
    assert os.path.isfile(model_file), 'Model file does not exist'
    assert os.path.isfile(data_file), 'Data file does not exist'
    assert os.path.isfile(label_file), 'Label file does not exist'

    # Your code starts here
    # check whether the GloVe embedding is already processed, if not, process it.
    glove_path = '.'
    if not os.path.isfile(f'{glove_path}/6B.50_words.pkl'):
        words = []
        idx = 0
        word2idx = {}
        vectors = bcolz.carray(np.zeros(1), rootdir=f'{glove_path}/6B.50.dat', mode='w')

        with open(f'{glove_path}/glove.6B.50d.txt', 'rb') as f:
            for l in f:
                line = l.decode().split()
                word = line[0]
                words.append(word)
                word2idx[word] = idx
                idx += 1
                vect = np.array(line[1:]).astype(np.float)
                vectors.append(vect)

        vectors = bcolz.carray(vectors[1:].reshape((400000, 50)), rootdir=f'{glove_path}/6B.50.dat', mode='w')
        vectors.flush()
        pickle.dump(words, open(f'{glove_path}/6B.50_words.pkl', 'wb'))
        pickle.dump(word2idx, open(f'{glove_path}/6B.50_idx.pkl', 'wb'))

    model = RNNTagger("./wsj1-18.training", hidden_dim=100)
    model.load_state_dict(torch.load(model_file))
    with open(data_file, 'rb') as f:
        contents = f.readlines()
    test_input = []
    for k, sentence in enumerate(contents):
        # omit the last '\n' and the first 'b''
        sentence = str(sentence)
        sentence = sentence[2:-3]
        # get rid of possible lines starting with space
        i = 0
        while sentence[i] == ' ':
            i += 1
        sentence = sentence[i:]
        parsed_sen = re.split(r'\s+', sentence)
        for w in parsed_sen:
            if w in model.vocab:
                test_input.append(model.vocab[w])
            else:
                # if the word is not present in the vocabulary, treat it as 'UNKA'
                test_input.append(1)
    # input tensor: as a 1 row tensor containing all the words
    test_input_tensor = torch.tensor(test_input, dtype=torch.long)

    prediction = model(test_input_tensor)
    prediction = torch.argmax(prediction,-1).cpu().numpy()

    with open(label_file, 'rb') as f:
        lab_contents = f.readlines()
    ground_truth = []
    for k, sentence in enumerate(lab_contents):
        # omit the last '\n' and the first 'b''
        sentence = str(sentence)
        sentence = sentence[2:-3]
        # get rid of possible lines starting with space
        i = 0
        while sentence[i] == ' ':
            i += 1
        sentence = sentence[i:]
        parsed_sen = re.split(r'\s+', sentence)
        for l in range(len(parsed_sen)//2):
            if parsed_sen[2*l+1] in model.tags:
                ground_truth.append(model.tags[parsed_sen[2*l+1]])
            else:
                # if seeing a tag not seen before, assign it to a random previous tag
                ground_truth.append(random.randint(0,len(model.tags)-1))

    # Your code ends here

    print(f'The accuracy of the model is {100*accuracy_score(prediction, ground_truth):6.2f}%')

def main(params):
    if params.train:
        model = train(params.training_file)
        torch.save(model.state_dict(), params.model_file)
    else:
        test(params.model_file, params.data_file, params.label_file)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="HMM POS Tagger")
    parser.add_argument("--train", action='store_const', const=True, default=False)
    parser.add_argument('--model_file', type=str, default='model.torch')
    parser.add_argument('--training_file', type=str, default='')
    parser.add_argument('--data_file', type=str, default='')
    parser.add_argument('--label_file', type=str, default='')

    main(parser.parse_args())


# 作者: 王道 龙哥
# 2022年06月16日10时10分21秒
from socket import *
import select
import sys

# SOCK_STREAM代表tcp
tcp_server = socket(AF_INET, SOCK_STREAM)
server_addr = ('192.168.1.16', 2000)
tcp_server.bind(server_addr)
tcp_server.listen(10)  # tcp_server对象的缓冲大小
# 如果有新的客户端来链接服务器，那么就产生一个新的套接字专门为这个客户端服务
# client_socket用来为这个客户端服务
# tcp_server就可以省下来专门等待其他新客户端的链接
client_socket, client_addr = tcp_server.accept()

epoll = select.epoll()
# 告诉epoll监控标准输入
epoll.register(sys.stdin.fileno(), select.EPOLLIN)
epoll.register(client_socket.fileno(), select.EPOLLIN)

while True:
    events = epoll.poll(-1, 2)
    for fd, event in events:
        if fd == sys.stdin.fileno():  # 代表标准输入里边有数据
            data = input('请输入数据:')
            client_socket.send(data.encode('utf8'))
        elif fd == client_socket.fileno():  # 代表接收缓冲区里边有数据
            # 接收客户端的数据
            recv_data = client_socket.recv(100)
            if not recv_data:
                print('对方断开了')
                exit()
            print(recv_data.decode('utf8'))

client_socket.close()
tcp_server.close()


import numpy
import pytest
from sigopt import Connection
from sigopt.exception import SigOptException

from sigoptlite.driver import LocalDriver
from sigoptlite.models import FIXED_EXPERIMENT_ID

from test.base_test import UnitTestsBase

class TestSuggestionCreate(UnitTestsBase):
  conn = Connection(driver=LocalDriver)

  @pytest.fixture
  def experiment_meta(self):
    return self.get_experiment_feature("default")

  def test_suggestion_id_increment(self, experiment_meta):
    experiment_meta["type"] = "random"
    e = self.conn.experiments().create(**experiment_meta)
    num_obs = 5
    for i in range(num_obs):
      suggestion = self.conn.experiments(e.id).suggestions().create()
      assert int(suggestion.id) == i + 1
      self.conn.experiments(e.id).observations().create(
        assignments=suggestion.assignments,
        values=[{"name": "y1", "value": numpy.random.rand()}],
      )

  def test_suggestion_duplicates(self, experiment_meta):
    e = self.conn.experiments().create(**experiment_meta)
    s1 = self.conn.experiments(e.id).suggestions().create()
    s2 = self.conn.experiments(e.id).suggestions().create()
    assert s1 == s2

  def test_wrong_suggestion_id(self, experiment_meta):
    e = self.conn.experiments().create(**experiment_meta)
    s = self.conn.experiments(e.id).suggestions().create()
    with pytest.raises(SigOptException) as exception_info:
      self.conn.experiments(e.id).observations().create(
        suggestion=s.id + "1",
        values=[{"name": "y1", "value": 1}],
      )
    msg = f"The suggestion you provided: {s.id + '1'} does not match the suggestion stored: {s.id}"
    assert exception_info.value.args[0] == msg

  def test_suggestion_create_before_experiment_create(self):
    new_conn = Connection(driver=LocalDriver)
    with pytest.raises(SigOptException) as exception_info:
      new_conn.experiments(FIXED_EXPERIMENT_ID).suggestions().create()
    msg = "Need to create an experiment first before creating a suggestion"
    assert exception_info.value.args[0] == msg


# -*- coding: utf-8 -*-
"""
Created on Wed Aug  2 08:53:45 2023

@author: ian.michael.bollinger@gmail.com with the help of ChatGPT 4.0

"""
# Base Python Imports
import pathlib, os

# Required Python Imports
from termcolor import cprint
from datetime import datetime

# Function to Generate a Log File
def generate_log_file(log_file_path, use_numerical_suffix=False):
    """
    Generates or clears a log file based on the given parameters.

    This function either creates a new log file or clears an existing one, depending on the specified parameters.
    If the 'use_numerical_suffix' parameter is True, and the file already exists, a new file with a numerical suffix
    will be created. Otherwise, the existing file will be cleared.

    Parameters:
        log_file_path (str): Path to the log file.
        use_numerical_suffix (bool): If True, creates new files with numerical suffixes if the file exists; otherwise, clears the existing file.

    Returns:
        str: Path to the log file.
    """
    if os.path.exists(log_file_path) and use_numerical_suffix:
        # If using numerical suffixes, increment until a new filename is found
        counter = 1
        new_log_file_path = f"{log_file_path.rsplit('.', 1)[0]}_{counter}.txt"
        while os.path.exists(new_log_file_path):
            counter += 1
            new_log_file_path = f"{log_file_path.rsplit('.', 1)[0]}_{counter}.txt"
        log_file_path = new_log_file_path
    else:
        # Clear the existing log file or create a new one
        open(log_file_path, 'w').close()

    return log_file_path

# Function to color coded print to console and save to log_file information
def log_print(input_message, log_file):
    """
    Logs a message to a file and prints it to the console with appropriate coloring.

    This function takes a message and logs it to the specified file. Additionally, the message is printed to the
    console, potentially with specific coloring depending on the context.

    Parameters:
        input_message (str): Message to be logged and printed.
        log_file (str): Path to the log file.
    """
    # Establish current date-time
    now = datetime.now()
    message = f'[{now:%Y-%m-%d %H:%M:%S}]\t{input_message}'

    # Determine the print color based on the input_message content
    message_type_dict = {'ERROR': ['red'],
                         'WARN': ['yellow'],
                         'NOTE': ['blue'],
                         'CMD': ['cyan'],
                         'PASS': ['green']}
    print_color = ['white']  # Default color
    for key, value in message_type_dict.items():
        if key.lower() in input_message.lower():
            print_color = value
            break

    # Writing the message to the log file
    with open(log_file, 'a') as file:
        print(message, file=file)

    # Handling different message types for colored printing
    try:
        cprint(message, print_color[0])
    except (KeyError, IndexError):
        cprint(message, print_color[1] if len(print_color) > 1 else 'white')

## Debuging Main Space & Example
if __name__ == '__main__':
    print('log_print.py DEBUG')

    # Establish Environment
    input_file = 'E:/EGAP/test.fastq'
    try:
        with open(input_file, 'rb') as f:
            print('WINDOWS ENVIRONMENT')
    except FileNotFoundError:
        print('LINUX/WSL/MAC ENVIRONMENT')
        input_file = input_file.replace(input_file.split('/')[0],'/mnt/e')

    # Generate log file with the desired behavior
    file_extension = pathlib.Path(input_file).suffix
    run_log = input_file.replace(file_extension, '.tsv')
    run_log = generate_log_file(run_log, use_numerical_suffix=False)

    log_print(input_message='PASS: TEST', log_file=run_log)
    log_print(input_message='NOTE: TEST', log_file=run_log)
    log_print(input_message='ERROR: TEST', log_file=run_log)
    log_print(input_message='WARN: TEST', log_file=run_log)
    log_print(input_message='CMD: TEST', log_file=run_log)
    log_print(input_message='TEST', log_file=run_log)


import pandas as pd
import numpy as np
from random import shuffle
import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)
import copy

from math import floor
from sys import exit
from scipy.stats import zscore
from ethoscopy.misc.rle import rle

def max_velocity_detector(data,
                        time_window_length,
                        velocity_correction_coef = 3e-3,
                        masking_duration = 6,
                        optional_columns = 'has_interacted'
                        ):
    """
    Max_velocity_detector is the default movement classification for real-time ethoscope experiments.
    It is benchmarked against human-generated ground truth.

    Params:
    @data = pandas dataframe, containing behavioural variables of a single animal (no id)
    @time_window_length = int, the period of time the data is binned and sampled to
    @velocity_correction_coef = float, a coefficient to correct the velocity data (change for different length tubes), default is 3e-3
    @masking_duration = int, number of seconds during which any movement is ignored (velocity is set to 0) after a stimulus is delivered (a.k.a. interaction),
    for beam_cross column, default is 6
    @optional_columns = string, columns other than ['t', 'x', 'velocity'] that you want included post analysis, default is 'has_interacted'
    @raw = bool, if True the returned datafram will contain mean values per window length for all variables from the ethoscope

    returns a pandas dataframe object with columns such as 'moving' and 'beam_cross'
    """

    if len(data.index) < 100:
        return None

    needed_columns = ['t', 'x', 'y', 'w', 'h', 'phi', 'xy_dist_log10x1000']

    dt = prep_data_motion_detector(data,
                                needed_columns = needed_columns,
                                time_window_length = time_window_length,
                                optional_columns = optional_columns)

    dt['deltaT'] = dt.t.diff()
    dt['dist'] = 10 ** (dt.xy_dist_log10x1000 / 1000)
    # in rethomics it was v = dist / deltaT and then vc = v * (deltaT / vcoef), subrating in the first equation ablates the deltaT leavinf vc = dist / vcoef
    dt['velocity'] = dt.dist / velocity_correction_coef

    dt['beam_cross'] = abs(np.sign(0.5 - dt['x']).diff())
    dt['beam_cross'] = np.where(dt['beam_cross'] == 2.0, True, False)

    if 'has_interacted' not in dt.columns:
        if masking_duration > 0:
            masking_duration = 0
        dt['has_interacted'] = 0

    dt['interaction_id'] = dt['has_interacted'].cumsum()
    dt['mask'] = dt.groupby('interaction_id', group_keys = False)['t'].apply(lambda x: pd.Series(np.where(x < (x.min() + masking_duration), True, False), index=x.index))
    dt['beam_cross'] = dt['beam_cross'] & ~dt['mask']
    dt = dt.drop(columns = ['interaction_id', 'mask'])

    d_small = dt.groupby('t_round').agg(
    x = pd.NamedAgg(column='x', aggfunc='mean'),
    y = pd.NamedAgg(column='y', aggfunc='mean'),
    w = pd.NamedAgg(column='w', aggfunc='mean'),
    h = pd.NamedAgg(column='h', aggfunc='mean'),
    phi = pd.NamedAgg(column='phi', aggfunc='mean'),
    max_velocity = pd.NamedAgg(column='velocity', aggfunc='max'),
    mean_velocity = pd.NamedAgg(column='velocity', aggfunc='mean'),
    distance = pd.NamedAgg(column='dist', aggfunc='sum'),
    interactions = pd.NamedAgg(column='has_interacted', aggfunc='sum'),
    beam_crosses = pd.NamedAgg(column='beam_cross', aggfunc= 'sum')
    )

    d_small['moving'] = np.where(d_small['max_velocity'] > 1, True, False)
    d_small['micro'] = np.where((d_small['max_velocity'] > 1) & (d_small['max_velocity'] < 2.5), True, False)
    d_small['walk'] = np.where(d_small['max_velocity'] > 2.5, True, False)
    d_small.rename_axis('t', inplace = True)
    d_small.reset_index(level=0, inplace=True)

    return d_small

def prep_data_motion_detector(data,
                            needed_columns,
                            time_window_length = 10,
                            optional_columns = None
                            ):
    """
    This function bins all points of the time series column into a specified window.
    Also checks optional columns provided in max_velocity_detector are present.

    Params:
    @data = pandas dataframe, dataframe as entered into the max_velocity_detector function
    @needed_columns = string, columns to be kept and the function enacted upon
    @time_window_length = int, the period of time the data is binned and sampled to, default is 10
    @optional_columns = string, columns other than ['t', 'x', 'xy_dist_log10x1000'] that you want included post analysis, default is None

    returns the same object as entered into the function
    """

    if all(elem in data.columns.values for elem in needed_columns) is not True:
        warnings.warn('data from ethoscope should have columns named {}!'.format(needed_columns))
        exit()

    # check optional columns input are column headings
    if optional_columns != None:

        if isinstance(optional_columns, str):
            check_optional_columns = set(data.columns.tolist()).intersection(list([optional_columns]))
            needed_columns = list(set(list(check_optional_columns) + needed_columns))
        else:
            check_optional_columns = set(data.columns.tolist()).intersection(optional_columns)
            needed_columns = list(set(list(check_optional_columns) + needed_columns))

    dc = copy.deepcopy(data[needed_columns])
    dc['t_round'] = dc['t'].map(lambda t: time_window_length * floor(t / time_window_length))

    def curate_sparse_roi_data(data,
                            window = 60,
                            min_p = 20
                            ):
        """
        Remove rows from table when there are not enough data points per the given window

        Params:
        @data =  pandas dataframe, dataframe containing ethoscope raw data with column 't' containing time series data
        @window = int, the size of the window to search for minimum points, default is 60
        @min_p = int, the minimum number of data points needed in a given window for it not to be removed, default is 20

        returns the same object as entered into the function
        """
        data['t_w'] = data['t'].map(lambda t: window * floor(t / window))
        data['n_points'] = data.groupby(['t_w'])['t_w'].transform('count')
        data = data[data.n_points > min_p]
        data.drop(columns = ['t_w', 'n_points'], inplace = True)

        return data

    dc = curate_sparse_roi_data(dc)

    return dc

def sleep_annotation(data,
                    time_window_length = 10,
                    min_time_immobile = 300,
                    motion_detector_FUN = max_velocity_detector,
                    masking_duration = 6,
                    velocity_correction_coef = 3e-3,
                    ):
    """
    This function first uses a motion classifier to decide whether an animal is moving during a given time window.
    Then, it defines sleep as contiguous immobility for a minimum duration.

    Params:
    @data = pandas dataframe, dataframe containing behavioural variable from many or one multiple animals
    @time_window_length = int, the period of time the data is binned and sampled to, default is 10
    @min_time_immobile = int, immobility bouts longer or equal to this value are considered as asleep, default is 300 (i.e 5 mins)
    @motion_detector_FUN = function, the function to curate raw ethoscope data into velocity measurements, default is max_velocity_detector
    @masking_duration, int, number of seconds during which any movement is ignored (velocity is set to 0) after a stimulus is delivered (a.k.a. interaction),
    @velocity_correction_coef = float, a coefficient to correct the velocity data (change for different length tubes), default is 3e-3
    for beam_cross column, default is 6
    returns a pandas dataframe containing columns 'moving' and 'asleep'
    """

    if len(data.index) < 100:
        return None

    d_small = motion_detector_FUN(data, time_window_length, masking_duration = masking_duration, velocity_correction_coef = velocity_correction_coef)

    if len(d_small.index) < 100:
        return None

    time_map = pd.Series(range(d_small.t.iloc[0],
                        d_small.t.iloc[-1] + time_window_length,
                        time_window_length
                        ), name = 't')

    missing_values = time_map[~time_map.isin(d_small['t'].tolist())]
    d_small = d_small.merge(time_map, how = 'right', on = 't', copy = False).sort_values(by=['t'])
    d_small['is_interpolated'] = np.where(d_small['t'].isin(missing_values), True, False)
    d_small['moving'] = np.where(d_small['is_interpolated'] == True, False, d_small['moving'])

    def sleep_contiguous(moving, fs, min_valid_time = 300):
        """
        Checks if contiguous bouts of immobility are greater than the minimum valid time given

        Params:
        @moving = pandas series, series object comtaining the movement data of individual flies
        @fs = int, sampling frequency (Hz) to scale minimum length to time in seconds
        @min_valid_time = min amount immobile time that counts as sleep, default is 300 (i.e 5 mins)

        returns a list object to be added to a pandas dataframe
        """
        min_len = fs * min_valid_time
        r_sleep =  rle(np.logical_not(moving))
        valid_runs = r_sleep[2] >= min_len
        r_sleep_mod = valid_runs & r_sleep[0]
        r_small = []
        for c, i in enumerate(r_sleep_mod):
            r_small += ([i] * r_sleep[2][c])

        return r_small

    d_small['asleep'] = sleep_contiguous(d_small['moving'], 1/time_window_length, min_valid_time = min_time_immobile)

    return d_small

def puff_mago(data, start_response_window = 0, response_window_length = 10, add_false = False, velocity_correction_coef = 3e-3):
    """
    Puff_mago finds interaction times from raw ethoscope data to detect responses in a given window.
    This function will only return data from around interaction times and not whole movement data from the experiment.

    Params:
    @data = pandas dataframe, dataframe containing behavioural variable from many or one multiple animals
    @response_window = int, the period of time (seconds) after stimulus to check for a response (movement), default is 10 seconds
    @add_false = bool / int, If not False then an int which is the percentage of the total of which to add false interactions, recommended is 10
    @velocity_correction_coef = float, a coefficient to correct the velocity data (change for different length tubes), default is 3e-3

    returns  a pandas dataframe object with columns such as 'interaction_t' and 'has_responded'
    """

    if start_response_window == response_window_length or start_response_window > response_window_length:
        raise ValueError("start_response_window must be less than response_window_length")

    # check for has_interaction column, as is removed during loading of roi if all false
    if any('has_interacted' in ele for ele in data.columns.tolist()) is False:
        print('ROI was unable to load due to there being no interactions in the database')
        return None

    data['deltaT'] = data.t.diff()
    data['dist'] = 10 ** (data.xy_dist_log10x1000 / 1000)
    data['velocity'] = data.dist / velocity_correction_coef
    data.drop(columns = ['deltaT', 'dist'], inplace = True)

    if add_false is not False:
        if add_false <= 0 or add_false >= 101:
            raise ValueError("add_false must be between 1 and 100")
        int_list = [2] * (int(len(data)*(add_false/100)))
        int_list_2 = [0] * (len(data) - len(int_list))
        int_list_all = int_list + int_list_2
        shuffle(int_list_all)
        data['has_interacted2'] = int_list_all
        data['has_interacted'] = np.where(data['has_interacted'] == 1, data['has_interacted'], data['has_interacted2'])
        data = data.drop(columns = ['has_interacted2'])

    #isolate interaction times
    interaction_dt = data['t'][(data['has_interacted'] == 1) | (data['has_interacted'] == 2)].to_frame()
    # interaction_dt.rename(columns = {'t' : 't_int'}, inplace = True)

    #check some interactions took place, return none if empty
    if len(interaction_dt.index) < 1:
        return None

    interaction_dt['start'] = interaction_dt.t
    interaction_dt['end'] = interaction_dt.t + response_window_length
    interaction_dt['int_id'] = np.arange(1, len(interaction_dt) + 1)

    # ints = data.t.values
    starts = interaction_dt.start.values
    ends = interaction_dt.end.values

    #### Old method, too memory intensive - the np.where is massive
    # search all time values and retrieve the indexes of values between start and end times
    # creates two lists. first with indices for data dataframe and the second is the indices for interaction_dt
    # create dataframe of two
    # i, j = np.where((ints[:, None] >= starts) & (ints[:, None] <= ends))
    # df = pd.DataFrame(
    #     np.column_stack([data.values[i], interaction_dt.values[j]]),
    #     columns= data.columns.append(interaction_dt.columns)
    # )

    # New method, but can be slow
    df = pd.concat([data[(data['t'] >= i) & (data['t'] < q)] for i, q in zip(starts, ends)])
    df = df.join(interaction_dt, rsuffix = '_int').fillna(method = 'ffill')

    # find relative time to interaction and check for movement
    df['t_rel'] = df.t - df.t_int
    df = df[(df['t_rel'] > start_response_window) | (df['t_rel'] == 0)]
    df.rename(columns = {'t_int' : 'interaction_t'}, inplace = True)
    df['has_responded'] = np.where((df['t_rel'] > 0) & (df['velocity'] > 1), True, False)
    df['has_walked'] = np.where((df['t_rel'] > 0) & (df['velocity'] > 2.5), True, False)
    df.drop(columns = ['xy_dist_log10x1000', 'start', 'end'], inplace = True)

    response_rows = []
    # is any response take the interaction row and change response to True and t_rel to time till movement
    def find_interactions(response_data):
        if any(response_data['has_responded']):
            response_dict = response_data[response_data['t_rel'] == 0].to_dict('records')[0]
            response_dict['has_responded'] = True
            response_dict['t_rel'] = response_data['t_rel'][response_data['has_responded'] == True].iloc[0]
            response_dict.pop('int_id')
            response_rows.append(response_dict)
        else:
            response_dict = response_data[response_data['t_rel'] == 0].to_dict('records')[0]
            response_dict.pop('int_id')
            response_rows.append(response_dict)

    df.groupby('int_id').apply(find_interactions)

    return pd.DataFrame(response_rows)

def find_motifs(data, window = 300, response_window = 10, velocity_correction_coef = 3e-3):
    """
    Find_motifs is a modification of puff_mago. It only takes data with a populated has_interacted column.
    The function will take a response window (in seconds) to find the variables recorded by the ethoscope in this window prior to an
    interaction taking place. Each run is given a unique ID per fly, however it is not unique to other flies. To do so, combine the
    fly ID with run ID after.

    Params:
    @data = pandas dataframe, dataframe containing behavioural variable from many or one multiple animals
    @window = int, the period of time (seconds) prior to the stimulus you want data retrieved for, default is 300
    @velocity_correction_coef = float, a coefficient to correct the velocity data (change for different length tubes), default is 3e-3

    returns  a pandas dataframe object with columns such as 't_count' and 'has_responded'
    """

    # check for has_interaction column, will be moved in prior download if all interactions are false
    if any('has_interacted' in ele for ele in data.columns.tolist()) is False:
        print('ROI was unable to load due to no interactions in the database')
        return None

    data['deltaT'] = data.t.diff()
    data['dist'] = 10 ** (data.xy_dist_log10x1000 / 1000)
    data['velocity'] = data.dist / velocity_correction_coef
    data.drop(columns = ['deltaT', 'dist'], inplace = True)

    #isolate interaction times
    interaction_dt = data['t'][data['has_interacted'] == 1].to_frame()
    interaction_dt.rename(columns = {'t' : 'int_t'}, inplace = True)

    #check some interactions took place, return none if empty
    if len(interaction_dt.index) < 1:
        return None

    interaction_dt['start'] = interaction_dt.int_t - window
    interaction_dt['end'] = interaction_dt.int_t + response_window

    ints = data.t.values
    starts = interaction_dt.start.values
    ends = interaction_dt.end.values

    i, j = np.where((ints[:, None] >= starts) & (ints[:, None] <= ends))
    df = pd.DataFrame(
        np.column_stack([data.values[i], interaction_dt.values[j]]),
        columns = data.columns.append(interaction_dt.columns)
    )

    df['t_rel'] = df.t - df.int_t
    df.rename(columns = {'int_t' : 'interaction_t'}, inplace = True)
    df['has_responded'] = np.where((df['t_rel'] > 0) & (df['velocity'] > 1), True, False)
    df['has_walked'] = np.where((df['t_rel'] > 0) & (df['velocity'] > 2.5), True, False)
    df.drop(columns = ['start', 'end'], inplace = True)

    # filter by window ahead of interaction time and find any postive response, return new df with only interaction time == 0 rows with response
    df['t'] = np.floor(df['t'])
    start_list = np.floor(interaction_dt['start'].to_numpy()).astype(int)
    end_list = np.ceil(interaction_dt['end'].to_numpy()).astype(int)

    response_df = pd.DataFrame()

    def format_window(response):
        if response is True:
            r = 1
        else:
            r = 0

        motif_df = filtered_df[filtered_df['t_rel'] <= 0]
        motif_df['has_responded'] = np.where(motif_df['t_rel'] == 0, response, motif_df['has_responded'])
        cols = motif_df.columns.tolist()
        motif_df[cols] = motif_df[cols].apply(pd.to_numeric, errors='coerce', axis=1)
        d_small = motif_df.groupby('t').agg(**{
                        't' : ('t', 'mean'),
                        'x' : ('x', 'mean'),
                        'y' : ('y', 'mean'),
                        'w' : ('w', 'mean'),
                        'h' : ('h', 'mean'),
                        'phi' : ('phi', 'mean'),
                        'xy_dist_log10x1000' : ('xy_dist_log10x1000', 'mean'),
                        'velocity' : ('velocity', 'mean')
                })
        t_range = list(range(0, len(d_small['t'] + 1)))
        t_range.reverse()
        d_small['t_count'] = t_range
        if len(d_small) != window + 1:
            return None
        else:
            d_small['response'] = [response] * len(d_small)
            id = f'{c}_{r}'
            d_small['run_id'] = [id] * len(d_small)
            return d_small

    for c, (i,q) in enumerate(zip(start_list, end_list)):

        filtered_df = df[df.t.isin(list(range(i, q + 1)))]
        response_data = filtered_df[filtered_df['t_rel'] >= 0]

        if any(response_data['has_responded']):
            formatted_small = format_window(response = True)
            if formatted_small is not None:
                response_df = response_df.append(formatted_small)

        else:
            formatted_small = format_window(response = False)
            if formatted_small is not None:
                response_df = response_df.append(formatted_small)

    return response_df

def isolate_activity_lengths(data, intervals, window, inactive = True, velocity_correction_coef = 3e-3):
    """
    Isolate activity lengths is a loading function that will find consecutive runs of inactivity or activity and segment them into same sized windows
    at intervals stated by the user. This function

    Params:
    @data = pandas dataframe, a dataframe object as provided from the read_single_roi fucntion with a column of time 't' in seconds
    @intervals = list of ints, a list with the timestamps you want the window to work back from, must be in minutes
    @window = int, the time frame you want to work back from each interval
    @inactive = bool, whether to search for runs of activity or inactivity
    @velocity_correction_coef - float, coeffient to find the velocity over time

    returns a pandas dataframe with every run according the the requirements and all data values
    """
    assert(isinstance(intervals, list))
    assert(all(isinstance(item, int) for item in intervals))

    if len(data.index) < 100:
        return None

    window = window * 60

    data['deltaT'] = data.t.diff()
    data['dist'] = 10 ** (data.xy_dist_log10x1000 / 1000)
    data['velocity'] = data.dist / velocity_correction_coef
    data.drop(columns = ['deltaT', 'dist'], inplace = True)
    data['t'] = np.floor(data['t'])
    data = data.groupby('t').agg(**{
                'x' : ('x', 'mean'),
                'y' : ('y', 'mean'),
                'w' : ('w', 'mean'),
                'h' : ('h', 'mean'),
                'phi' : ('phi', 'mean'),
                'xy_dist_log10x1000' : ('xy_dist_log10x1000', 'max'),
                'velocity' : ('velocity', 'max')
        })
    data.reset_index(inplace = True)
    data['moving'] = np.where(data['velocity'] > 1, 1, 0)

    def norm_1_0(data, var):
        v_min = data[var].min()
        v_max = data[var].max()
        data[var] = data[var].map(lambda v: (v - v_min) / (v_max - v_min))
        return data

    data = norm_1_0(data, var = 'x')

    for i in ['w', 'h']:
        data[f'{i}_z'] = np.abs(zscore(data[i].to_numpy())) < 3
        data[i] = np.where(data[f'{i}_z'] == True, data[i], np.nan)
        data[i] = data[i].fillna(method = 'ffill')
        data[i] = data[i].fillna(method = 'bfill')
        data.drop(columns = [f'{i}_z'], inplace = True)
        data = norm_1_0(data, var = i)

    def find_inactivity(data, inactive = inactive):
                if inactive == True:
                    elem = 0
                else:
                    elem = 1
                inactive_count = []
                data_list = []
                counter = 1
                for c, q in enumerate(data):
                    if c == 0 and q != elem:
                        inactive_count.append(np.NaN)
                        data_list.append(q)

                    elif c == 0 and q == elem:
                        inactive_count.append(counter)
                        data_list.append(q)
                        counter += 1

                    else:
                        if q == elem:
                            inactive_count.append(counter)
                            data_list.append(q)
                            counter += 1

                        else:
                            inactive_count.append(np.NaN)
                            data_list.append(q)
                            counter = 1

                return inactive_count

    data['inactive_count'] =  find_inactivity(data['moving'].to_numpy())
    data = data[data['inactive_count'] <= max(intervals) * 60]

    inactivity_df = pd.DataFrame()

    for interval in intervals:
        #isolate interaction times
        interval = interval * 60
        interaction_dt = data['t'][data['inactive_count'] == interval].to_frame()
        interaction_dt.rename(columns = {'t' : 'int_t'}, inplace = True)

        #check some interactions took place, return none if empty
        if len(interaction_dt.index) < 1:
            return None

        interaction_dt['start'] = interaction_dt.int_t - (window-1)
        interaction_dt['end'] = interaction_dt.int_t

        ints = data.t.values
        starts = interaction_dt.start.values
        ends = interaction_dt.end.values

        i, j = np.where((ints[:, None] >= starts) & (ints[:, None] <= ends))

        df = pd.DataFrame(
            np.column_stack([data.values[i], interaction_dt.values[j]]),
            columns = data.columns.append(interaction_dt.columns)
        )
        df.drop(columns = ['end', 'int_t', 'inactive_count'], inplace = True)

        gb = df.groupby('start').size()
        filt_gb = gb[gb == window]
        filt_df = df[df['start'].isin(filt_gb.index.tolist())]
        filt_df['t_rel'] = list(range(interval - window, interval)) * len(filt_gb.index)
        inactivity_df = pd.concat([inactivity_df, filt_df], ignore_index = False)

    return inactivity_df


from string import ascii_letters

__author__ = "josebermudez"

def sentence_to_hashtag(sentence):
    """
    This function convert a sentence in a hashtag.
    Non-alphabetic characters are ignored.
    E.g. 'thi9s Is &&So cool' -> '#ThisIsSoCool'.
    :param sentence: sentence
    :return: hashtag
    """
    result = None

    if sentence and type(sentence) is str:
        result = '#'

        for word in sentence.split():
            # clean non-alphabetic characters
            aux = "".join([ch for ch in word if ch in ascii_letters])

            # add capitalize word
            result += aux.title()

    if result == '#':
        result = None

    return result


import socket
import os
#import datetime
import json
import traceback, sys
import csv
#import pandas as pd

List_of_RFCs = 'C:\\Internet_Protocols\\List_of_RFCs'
csv_receivedlist = 'C:\\Internet_Protocols\\Received_list\\csv_receivedlist.csv'
peer_list = []

Port_RS = 23473
Host_RS = '192.168.0.161'

hostname = socket.gethostname()
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(("8.8.8.8", 80))
ip1 = (s.getsockname()[0])
print(hostname)
portnumber = 10000

# Create a socket object
#s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
#s.connect((Host_RS,Port_RS ))

class peerserver():
    def __init__(self, ip: str, port: int):
        self.ip = ip
        self.port = port

def registration():
    x = 'Register_to_RS'
    message = json.dumps({"msg" : x , "Hostname" : hostname , "Portnumber" : str(portnumber)})
    s.send(message.encode())
    received_message = s.recv(2048).decode()
    print(received_message)
    recv_msg = json.loads(received_message)
    print(recv_msg["msg"])
    s.close()

def check_rfc(request):
    r = []
    for rfcs in request.split(","):
            if rfcs not in peer_list:
                rfcs = rfcs.strip()
                r.append(rfcs)
                #print(r)
    return r

def requesting_peerlist():
    rfc_request = input("Enter the rfc to be requested: ")
    rfc = check_rfc(rfc_request)
    #print(rfc)
    x = 'Active_peer'
    message = json.dumps({"msg" : x , "Hostname" : hostname , "Portnumber" : str(portnumber)})
    s.send(message.encode())
    received_message = s.recv(2048).decode()
    print(received_message)
    recv_message = received_message.split('\n')
    #print(type(recv_message), len(recv_message), recv_message)
    if len(recv_message) == 1:
        print("No peers active")
        s.close()
    else:
        new_msg = recv_message[1]
        #print(type(new_msg))
        new_msg = new_msg.strip('[').strip(']').replace('}, {', '}!{').split('!')
        res = []
        for item in new_msg:
            res.append(json.loads(item))
        print(res)
        for element in res:
            peer = peerserver(**element)
            print(peer)
            try:
                if(requesting_rfcs(peer, rfc)):
                    break
            except Exception:
                traceback.print_exc(sys.stdout)
        if(len(res) != 0):
            print("No active peer left to check")
    s.close()

def checking_list(msg):
    f = open(csv_receivedlist)
    csv_f = csv.reader(f)
    list1 = []
    attendee_emails = []
    for row in csv_f:

        d = row[0].split(";")
        attendee_emails.append(d)
    #print('abc',attendee_emails)
    #print('msg',msg)
    for i in msg:
        #b = i[0]
        c = i[0].split(";")

        if c in attendee_emails:
            pass
        else:
            count = 0
            for j in attendee_emails:
                if c[0] == j[0] and c[1] == j[1] and c[2] == j[2]:
                    #print("in loop")
                    ind = attendee_emails.index(j)
                    attendee_emails[ind] = c
                    count +=1

            if count == 0:
                attendee_emails.append(c)
                list1.append(c)
   # print(list1)
    return list1

def requesting_rfcs(peer: peerserver, rfc):
    print(rfc)
    s1 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    port = int(peer.port)
    ip = peer.ip
    s1.connect((ip, port))
    x = 'listofrfcs'
    message = json.dumps({"msg" : x , "Hostname" : hostname , "Portnumber" : str(portnumber)})
    s1.send(message.encode())
    received_message = s1.recv(2048).decode()
    #print(received_message)
    recv_message = received_message.split('\n')
    #print(type(recv_message), len(recv_message), recv_message)
    new_msg = recv_message[2]
    #print(new_msg)
    r_msg = json.loads(new_msg)
    print(r_msg)
    res_msg = checking_list(r_msg)
    print("-------------------------------------------")
    print(res_msg)
    print("-------------------------------------------")
    for IPList in res_msg:
        #print("Here")
        #IPList = item[0].split(',')
        #print(IPList)
        texttype = IPList[0].split('.')[0]
        ip_1 = IPList[1]
        port_1 = IPList[2]
        flag = IPList[3]
        reg_rfc_in_csv(texttype, ip_1, port_1, flag)

    if new_msg == '[]':
        res = 0
    else:
        print("Im here")
        res = json.loads(new_msg)
        #print(res)
        for element in res:
            IPList = element[0].split(';')
            texttype = IPList[0].split('.')[0]
            ip_1 = IPList[1]
            port_1 = IPList[2]
            print(texttype)
            for i in rfc:
                if texttype == i:
                    #print(ip_1)
                    #print(port_1)
                    try:
                        s2 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                        s2.connect((ip_1,int(port_1)))
                        x = 'Requesting_files'
                        message = json.dumps({"msg" : x ,"file" : i, "Hostname" : hostname , "Portnumber" : str(portnumber)})
                        print(message)
                        s2.send(message.encode())
                        flag = '1'
                        reg_rfc_in_csv(i, ip1, portnumber, flag)
                        print("Im")
                        recv_rfc_file(i, s2)
                    except Exception:
                        traceback.print_exc(sys.stdout)
                    finally:
                        s2.close
    s1.close()
    return False

def recv_rfc_file(rfc, s):
    completeName = os.path.join(List_of_RFCs, str(rfc) + ".txt")
    with open(completeName, 'wb') as file:
        while True:
            data = s.recv(2048)
            if not data:
                break
            file.write(data)
    file.close()

def reg_rfc_in_csv(rfc, ip: str, port: str, flag):
    with open(csv_receivedlist, mode='a', newline ='') as myfile:
        wr = csv.writer(myfile, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
        wr.writerow([str(rfc) +'.txt;' + ip + ';' + str(port) + ';' + flag])

def leave_message():
    x = 'Leave_the_server'
    message = json.dumps({"msg" : x , "Hostname" : hostname , "Portnumber" : str(portnumber)})
    s.send(message.encode())
    received_message = s.recv(2048).decode()
    #recv_msg = json.loads(received_message)
    print(received_message)
    s.close()

def peer_list_files():
    global exisiting_files
    for root, dirs, files in os.walk(List_of_RFCs):
        #print(files)
        for file in files:
            if '.txt' in file:
                peer_list.append(file.split(".")[0])

peer_list_files()

choice = None
while True:
    print("1. Registering with RS\n2. Requesting Peer List and RFC's\n3. Leave the network")
    choice = int(input("Enter your request: "))
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.connect((Host_RS,Port_RS))
    if choice == 1:
        registration()
    elif choice == 2:
        requesting_peerlist()
    else:
        leave_message()


#!/usr/bin/env python3

import os
import sys
import re
import pandas as pd
from GP_file_functions import file_check, suffix_from_filename, string_to_list, find_filetype_files

#------------------------------------------------------------------------------------------------
# COLLECTING OMEGA DATA IF IT MEETS THE CRITERIA SET---------------------------------------------
#------------------------------------------------------------------------------------------------

def filepath_to_omega_dict_list(omega_filepath_list, omega_criteria_list=[]):
    """
    Converts a list of filepaths to omega dictionaries that meet specified criteria.
    Returns a list of omega dictionaries that meet the criteria.
    """
    # Convert the input parameter filepath list and criteria list from strings to lists
    omega_filepath_list = string_to_list(omega_filepath_list)
    omega_criteria_list = string_to_list(omega_criteria_list)

    # Initialize an empty list to store the parameter dictionaries that meet the criteria
    omega_dict_list = []

    # Loop through each parameter filepath in the list
    for omega_filepath in omega_filepath_list:
        # Convert the parameter filepath to a dictionary
        omega_dict = omega_filepath_to_dict(omega_filepath)

        # Check if the omega dictionary meets the criteria
        meets_criteria = all(eval(condition, omega_dict) for condition in omega_criteria_list)
        if meets_criteria or (omega_criteria_list == []):
            # If the parameter dictionary meets the criteria or no criteria were specified, add it to the list
            omega_dict_list.append(omega_dict)

    # Return the list of parameter dictionaries that meet the criteria
    return omega_dict_list

#------------------------------------------------------------------------------------------------
# BASE FUNCTION TO CONVERT OMEGA TO DICT---------------------------------------------------------
#------------------------------------------------------------------------------------------------

def omega_filepath_to_dict(omega_filepath):
    """
    Converts a 'omega' file at a given filepath to a omega dictionary.
    Returns the omega dictionary.
    """
    # Check if the filepath is a valid 'omega' file
    check_param = file_check(omega_filepath, 'omega')

    if check_param:
        omega_dict = {}

        # Get the omega filename and directory path
        omega_file = os.path.basename(omega_filepath)
        omega_directory = os.path.dirname(omega_filepath)
        suffix = suffix_from_filename(omega_filepath)

        # Change the current working directory to the omega directory
        os.chdir(omega_directory)

        # Read in the omega file
        with open(omega_file, 'r') as file:
            lines = file.readlines()

        # Split lines and put values into omega dict
        for line in lines:
            items = line.split()

            omega_dict['kymin'] = {'ky*rho_s': float(items[0])}
            omega_dict['gamma'] = {'cs/a': float(items[1])}
            omega_dict['omega'] = {'cs/a': float(items[2])}

        # Add the filename, filepath, and suffix to the omega dictionary
        omega_dict['filename'] = omega_file
        omega_dict['filepath'] = omega_directory
        omega_dict['suffix'] = suffix

        return omega_dict

    else:
        # If the filepath is not a valid 'parameters' file, print an error message and return None
        print('Filepath given is not a omega file, is empty, or is not a file. Filepath:\n', omega_filepath)
        return None

#------------------------------------------------------------------------------------------------
# OUTPUTTING OF DATA ----------------------------------------------------------------------------
#------------------------------------------------------------------------------------------------

def filepath_list_to_omega_df(omega_filepath_list, omega_criteria_list = []):
    """
    Converts a list of filepaths to a pandas DataFrame of omega dictionaries that meet specified criteria.
    Displays the DataFrame in the console.
    """
    # If omega_filepath_list does not contain omega files, search for them up to a maximum depth of 2 directories
    max_depth = 2
    omega_filepath_list = find_filetype_files(omega_filepath_list, 'omega', max_depth)

    if len(omega_filepath_list) == 0:
        # If no 'parameters' files are found, print an error message
        print(f"No 'omega' files found at maximum depth of {max_depth} directories down. Try giving a more specific filepath or moving your simulations up directories.")
    else:
        # Convert omega filepath to dict list
        omega_dict_list = filepath_to_omega_dict_list(omega_filepath_list, omega_criteria_list)
        if len(omega_dict_list) == 0:
            # If no omega dictionaries meet the criteria, print an error message
            print(f"No 'omega' files matching the given criteria. Please check criteria again or relax the criteria given.")
        else:
            # Create a pandas DataFrame from the omega dictionary list
            df = pd.DataFrame(omega_dict_list)

            # Convert omega criteria list to only have strings (avoids comprehension reading errors)
            pattern = re.compile(r'^\w+')
            omega_criteria_list = [re.findall(pattern, criteria)[0] for criteria in omega_criteria_list]

            if omega_criteria_list == []:
                # If omega list is empty, show the values that are unique between all

                unique_counts = df.nunique()
                # Select only the columns where the count is greater than 1
                columns_to_show = unique_counts[unique_counts > 1].index.tolist()

                # remove_column_list = []
                # columns_to_show = [col for col in columns_to_show if col not in remove_column_list]

                if len(omega_dict_list) == 1:
                    standard_columns = ['filepath', 'filename', 'suffix']
                    columns_to_show = omega_criteria_list + standard_columns

            else:
                # If omega list is not empty, show only the specified columns
                omega_criteria_list = string_to_list(omega_criteria_list)

                standard_columns = ['filepath', 'filename', 'suffix']
                columns_to_show = omega_criteria_list + standard_columns

            # Use the loc accessor to select only the specified columns
            df_subset = df.loc[:, columns_to_show]

            # Display the DataFrame in the console
            print(df_subset)

#------------------------------------------------------------------------------------------------
#------------------------------------------------------------------------------------------------
#------------------------------------------------------------------------------------------------

if __name__=="__main__":
    filepath = os.getcwd()

    filepath_list_to_omega_df(filepath)

# def omega_dict_to_Hz(simulation_dict):
#     param_dict = simulation_dict['parameters']
#     omega_dict = simulation_dict['omega']

#     # collect filename and relevant values
#     omega_cs = omega_dict['omega (cs/a)']
#     gamma_cs = omega_dict['gamma (cs/a)']

#     # collect parameter values to convert omega values
#     TJ = param_dict['Tref']      #ion temperature in (J)
#     mi = param_dict['mref']      #ion mass
#     Lf = param_dict['Lref']      #reference length of the device
#     cs = np.sqrt(TJ/mi)              #speed of sound in a plasma
#     om_ref = cs/Lf                   #reference omega

#     # Convert omega and gamma to kHz range
#     omega_dict['omega (kHz)'] = omega_cs*om_ref/1000.0/(2.0*np.pi) #convert omega to kHz
#     omega_dict['gamma (kHz)'] = gamma_cs*om_ref/1000.0/(2.0*np.pi) #convert omega to kHz

#     # Add global kymin values to omega dict
#     omega_dict['n0_global'] = param_dict['n0_global']

#     return simulation_dict


import sys
import time
import unittest
import xapian

from gi.repository import Gtk

sys.path.insert(0,"../..")
sys.path.insert(0,"..")

from softwarecenter.db.enquire import AppEnquire
from softwarecenter.enums import SortMethods

from softwarecenter.testutils import (get_test_db,
                                      get_test_pkg_info,
                                      get_test_gtk3_icon_cache,
                                      )

class TestAppView(unittest.TestCase):
    """ test the app view """

    def setUp(self):
        self.cache = get_test_pkg_info()
        self.icons = get_test_gtk3_icon_cache()
        self.db = get_test_db()

    def test_app_view(self):
        from softwarecenter.ui.gtk3.views.appview import get_test_window
        enquirer = AppEnquire(self.cache, self.db)
        enquirer.set_query(xapian.Query(""),
                           sortmode=SortMethods.BY_CATALOGED_TIME,
                           limit=10,
                           nonblocking_load=False)

        # get test window
        win = get_test_window()
        appview = win.get_data("appview")
        # set matches
        appview.clear_model()
        appview.display_matches(enquirer.matches)
        self._p()
        # verify that the order is actually the correct one
        model = appview.tree_view.get_model()
        docs_in_model = [item[0] for item in model]
        docs_from_enquirer = [m.document for m in enquirer.matches]
        self.assertEqual(len(docs_in_model),
                         len(docs_from_enquirer))
        for i in range(len(docs_in_model)):
            self.assertEqual(self.db.get_pkgname(docs_in_model[i]),
                             self.db.get_pkgname(docs_from_enquirer[i]))

    def _p(self):
        for i in range(5):
            time.sleep(0.1)
            while Gtk.events_pending():
                Gtk.main_iteration()

if __name__ == "__main__":
    import logging
    logging.basicConfig(level=logging.DEBUG)
    unittest.main()


import numpy as np
from keras import Sequential
from keras.layers import Dense
from sklearn.model_selection import StratifiedKFold
from sklearn.preprocessing import LabelEncoder

csvname = './Data_3D_2classes.csv'

print('Reading "' + csvname + '":')
data = np.loadtxt(csvname, delimiter=';')

x = data[:, 0:3]
y = data[:, 3]

encoder = LabelEncoder()
encoder.fit(y)
encoded_Y = encoder.transform(y)

k = 10
seed = 1337

kfoldcv = StratifiedKFold(n_splits=k, shuffle=True, random_state=seed)

def build_model():
    model = Sequential()

    model.add(Dense(input_dim=3, units=1, activation="sigmoid"))
    model.compile(loss="binary_crossentropy",
                  optimizer="adam", metrics=["accuracy"])
    return model

cvscores = []
for training, test in kfoldcv.split(x, encoded_Y):
    model = build_model()
    model.fit(x[training], encoded_Y[training], epochs=200, batch_size=10, verbose=0)
    cvscore = model.evaluate(x[test], encoded_Y[test])[1]
    cvscores.append(cvscore)

print("Model accuracy: {0}% stdev: {1}%".format(np.mean(cvscores)*100, np.std(cvscores)*100))


import datetime

import locale

from beerthday.entry import Entry
from beerthday.senders import SENDERS

from dateutil.relativedelta import relativedelta

from freezegun import freeze_time

import pytest

@pytest.mark.gen_test
def test_entry():
    locale.setlocale(locale.LC_ALL, 'ru_RU.UTF8')
    assert Entry(1, 'TestName', (SENDERS.EMAIL, 'test@test.com'), datetime.datetime.strptime('01.02.1990', '%x')) == \
           Entry(1, 'OtherName', (SENDERS.EMAIL, 'test@test.com'), datetime.datetime.strptime('01.02.1990', '%x'))

@freeze_time('Feb 29th, 2016')
@pytest.mark.parametrize('years,days', [
    (-4, -7),
    (-4, 1),
    (-5, -6),
    (-5, 1)])
@pytest.mark.gen_test
def test_entry_leap(years, days):
    assert Entry(1,
                 'TestName',
                 ('email', 'test@test.com'),
                 datetime.datetime.now().date() + relativedelta(years=years, days=days)).is_birthday()

@freeze_time('Feb 28th, 2016')
@pytest.mark.parametrize('years,days', [
    (-4, -7),
    (-4, 1),
    (-5, -7),
    (-5, 0)])
@pytest.mark.gen_test
def test_entry_nonleap(years, days):
    assert Entry(1,
                 'TestName',
                 ('email', 'test@test.com'),
                 datetime.datetime.now().date() + relativedelta(years=years, days=days)).is_birthday()
    assert not Entry(1,
                     'TestName',
                     ('email', 'test@test.com'),
                     None).is_birthday()


import math

def traceBack(cameFromMap, current):
    result = [current]
    while current in cameFromMap.keys():
        current = cameFromMap[current]
        result.append(current)
    return result[::-1]

# heuristic may never over-estimate
# f = g + h
def dosomeastarxxx(start, goal, getNbrsOf, nbrDistance, heuristic, isGoalReached=lambda a,b:a==b):
    inf = float("inf")
    closedSet = []
    openSet = [start]
    # for each node: the most efficient preceding node
    cameFromMap = {}
    # for each node: the cost of getting there from start
    gScoreMap = {start: 0}
    # for each node: the total cost of getting from start to goal through that node, partly heuristic
    fScoreMap = {start: heuristic(start, goal)}

    while len(openSet):
        current = min(openSet, key=lambda node: fScoreMap.get(node, inf))
        if isGoalReached(current, goal):
            return [fScoreMap[current], traceBack(cameFromMap, current)]
        openSet.remove(current)
        closedSet.append(current)

        for nbr in getNbrsOf(current):
            if nbr in closedSet:
                continue
            if not nbr in openSet:
                openSet.append(nbr)
            newGScore = gScoreMap[current] + nbrDistance(current, nbr)
            if newGScore < gScoreMap.get(nbr,inf):
                cameFromMap[nbr] = current
                gScoreMap[nbr] = newGScore
                fScoreMap[nbr] = newGScore + heuristic(nbr, goal)
    return [0,[]]

# ALLES HIER ONDER IS NIET NODIG

boardString = """
..s#....
.###..#g
...#.###
...#....
....#...
....#...
........
........
"""

boardString = """
+--+--+--+--+--+--+--+--+--+--+
s  |           |        |     |
+  +  +--+--+  +  +--+  +  +  +
|     |           |     |  |  |
+--+--+  +--+  +--+  +  +  +  +
|        |     |     |  |  |  |
+  +--+--+--+--+  +--+  +  +  +
|        |     |  |     |  |  |
+--+--+  +  +  +  +--+--+  +--+
|     |  |  |  |  |     |     |
+  +  +  +--+  +  +  +  +  +  +
|  |  |     |  |     |     |  |
+  +--+--+  +  +--+--+--+--+  +
|           |        |        |
+--+--+  +--+--+  +  +  +--+--+
|        |     |  |  |     |  |
+  +--+--+--+  +--+  +--+  +  +
|  |  |     |  |     |     |  |
+  +  +  +  +  +  +  +  +--+  +
|        |        |  |        g
+--+--+--+--+--+--+--+--+--+--+
"""

boardString = open("maze.txt",'r').read()

boardLines = boardString.replace(" ",".").split()
width = len(boardLines[0])
length = len(boardLines)

for y, line in enumerate(boardLines):
    for x, char in enumerate(boardLines[y]):
        if char == 's':
            start = (x,y)
            boardLines[y] = boardLines[y].replace('s','.')
        elif char == 'g':
            goal = (x,y)
            boardLines[y] = boardLines[y].replace('g','.')

board = [[0 if (c=='.' or c==' ') else 1 for c in line] for line in boardLines]

moves = [(0,1), (1,0),(0,-1),(-1,0)]

def getAt(x,y):
    if x<0 or y<0 or x>=width or y>=length:
        return -1
    return board[y][x]

def nFunc(node):
    return list(filter(lambda p: getAt(p[0],p[1])==0, [(node[0]+m[0],node[1]+m[1]) for m in moves]))

def h(a, b):
    #return max(abs(a[0]-b[0]), abs(a[1]-b[1]))
    #return abs(a[0]-b[0]) + abs(a[1]-b[1])
    return math.sqrt((a[0]-b[0])**2 + (a[1]-b[1])**2)

for i in range(1):
    dist, path = dosomeastarxxx(start, goal, nFunc, lambda a,b: 1.0, h)

for point in path:
    boardLines[point[1]] = str.join("",['x' if i==point[0] else c for i,c in enumerate(boardLines[point[1]])]).replace("."," ")

print(str.join("\n",boardLines))


import string
from datetime import datetime

def is_hex(s):
     hex_digits = set(string.hexdigits)
     # if s is long, then it is faster to check against a set
     return all(c in hex_digits for c in s)

def to_user (command, old, new):
    if old is None:
        if new is True:
            new = 'On'
        if new is False:
            new = 'Off'
        return ('%s%s' % (command.ljust(13), str(new).ljust(10)))
    else:
        # old is NOT None
        if new is True:
            new = 'On'
        if new is False:
            new = 'Off'
        if old is True:
            old = 'On'
        if old is False:
            old = 'Off'
        return ('%s was %s' % (command, old))

def displaydatetime (dt, completer):
    if completer.options['DAYUSA'].Value:
        if completer.options['AMONTH'].Value:
            return dt.strftime ('%d-%b-%Y %H:%M:%S')
        else:
            return dt.strftime ('%m/%d/%Y %H:%M:%S')
    else:
        if completer.options['AMONTH'].Value:
            return dt.strftime ('%d-%b-%Y %H:%M:%S')
        else:
            return dt.strftime ('%d/%m/%Y %H:%M:%S')

def datetimenow (completer):
    if completer.options['UTC'].Value:
        return datetime.utcnow()
    else:
        return datetime.now()


from .abstract_machine import AutoNormalizedAutoregressiveMachine
from ..layers import VectorToComplexNumber
from ..deepar.layers import ToFloat32, DownShiftLayer, RightShiftLayer, WeightNormalization, ExpandInputDim

from tensorflow.keras.layers import Activation, Add, Concatenate, Conv2D, ZeroPadding2D

class ConvNetAutoregressive2D(AutoNormalizedAutoregressiveMachine):
    """docstring for ConvNetAutoregressive2D"""

    def __init__(self, keras_input_layer, depth, num_of_channels, kernel_size=3, strides=1, activation='relu',
                 weights_normalization=True, exponential_norm=True, **kwargs):
        self.depth = depth
        self.num_of_channels = num_of_channels
        self.kernel_size = kernel_size
        self.padding = kernel_size - 1
        self.strides = strides
        self.activation = activation
        self.weights_normalization = weights_normalization
        self.exponential_norm = exponential_norm
        self._build_unnormalized_conditional_log_wave_function(keras_input_layer)
        super(ConvNetAutoregressive2D, self).__init__(keras_input_layer, **kwargs)

    def _conv2d(self, filters, kernel_size):
        conv_layer = Conv2D(filters=filters, kernel_size=kernel_size, strides=1)
        if self.weights_normalization:
            conv_layer = WeightNormalization(conv_layer, exponential_norm=self.exponential_norm)
        return conv_layer

    def _activation(self):
        return Activation(self.activation)

    def _causal_conv_2d(self, vertical_x, horizontal_x, use_horizontal_mask=False):
        filters = self.num_of_channels
        if self.padding > 0:
            vertical_x = ZeroPadding2D(padding=((self.padding, 0), (self.padding // 2, self.padding // 2)))(vertical_x)
            horizontal_x = ZeroPadding2D(padding=((0, 0), (self.padding, 0)))(horizontal_x)
        vertical_x = self._conv2d(filters=filters, kernel_size=self.kernel_size)(vertical_x)
        x = self._conv2d(filters=filters, kernel_size=(1, self.kernel_size))(horizontal_x)
        x = self._activation()(x)
        if use_horizontal_mask:
            x = RightShiftLayer()(x)
        x = self._conv2d(filters=filters // 2, kernel_size=1)(x)
        y = DownShiftLayer()(self._activation()(vertical_x))
        y = self._conv2d(filters=filters // 2, kernel_size=1)(y)
        x, y = self._activation()(x), self._activation()(y)
        x = Concatenate(axis=-1)([x, y])
        if self.padding > 0:
            x = ZeroPadding2D(padding=((self.padding, 0), (self.padding, 0)))(x)
        horizontal_x = self._conv2d(filters=filters, kernel_size=self.kernel_size)(x)
        return vertical_x, horizontal_x

    def _resiual_block(self, vertical_x, horizontal_x, num_of_layers=2):
        vertical_x_input, horizontal_x_input = vertical_x, horizontal_x
        for _ in range(num_of_layers - 1):
            vertical_x, horizontal_x = self._causal_conv_2d(vertical_x, horizontal_x)
            vertical_x, horizontal_x = self._activation()(vertical_x), self._activation()(horizontal_x)
        vertical_x, horizontal_x = self._causal_conv_2d(vertical_x, horizontal_x)
        vertical_x = Add()([vertical_x_input, vertical_x])
        horizontal_x = Add()([horizontal_x_input, horizontal_x])
        vertical_x, horizontal_x = self._activation()(vertical_x), self._activation()(horizontal_x)
        return vertical_x, horizontal_x

    def _build_unnormalized_conditional_log_wave_function(self, keras_input_layer):
        x = ExpandInputDim()(keras_input_layer)
        x = ToFloat32()(x)
        vertical_x, horizontal_x = x, x
        vertical_x, horizontal_x = self._causal_conv_2d(vertical_x, horizontal_x)
        vertical_x, horizontal_x = self._activation()(vertical_x), self._activation()(horizontal_x)
        for i in range(self.depth - 2):
            vertical_x, horizontal_x = self._resiual_block(vertical_x, horizontal_x)
        _, x = self._causal_conv_2d(vertical_x, horizontal_x, use_horizontal_mask=True)
        x = Conv2D(filters=4, kernel_size=1, strides=1)(self._activation()(x))
        self._unnormalized_conditional_log_wave_function = VectorToComplexNumber()(x)

    @property
    def unnormalized_conditional_log_wave_function(self):
        return self._unnormalized_conditional_log_wave_function


import os
import json
import requests
from bs4 import BeautifulSoup

headers_Get = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:49.0) Gecko/20100101 Firefox/49.0',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'Accept-Language': 'en-US,en;q=0.5',
        'Accept-Encoding': 'gzip, deflate',
        'DNT': '1',
        'Connection': 'keep-alive',
        'Upgrade-Insecure-Requests': '1'
    }

def main():
    url = "https://www.google.com.vn/search?tbm=isch&q=%C4%91%E1%BA%A1i%20n%E1%BB%99i"

    response = requests.get(url)
    html = response.text

    soup = BeautifulSoup(html, 'html.parser')

    results = soup.findAll('div', {'class': 'rg_i'})

    imagelinks = []
    for result in results:
        text = result.text
        print(text)

main()


from PIL import Image as im
import os

oDirectory = "/opt/icons/"

def processImage(image, outDirectory):
    with im.open("./images/" + image) as img:
        img = img.convert("RGB")
        img = img.rotate(-90)
        img = img.resize((128,128))
        output = img
        output.save(os.path.join(outDirectory,image),"JPEG")

if __name__ == "__main__":
    fileList = os.listdir("./images/")
    if not (os.path.isdir(oDirectory)):
        os.makedirs(oDirectory)
    for file in fileList:
        if not(file.startswith(".")):
            processImage(file, oDirectory)