import inspect
import os
import numpy as np
import pickle
import regex

import tensorflow as tf
from tensorflow import keras
from transformers import BertConfig, TFBertMainLayer

from .utils import found_package
import transformers
import tensorflow_addons as tfa

from .utils import iobes_iob, parse_lr_method
from .conlleval_perl import evaluate_lines
from .layers import SimpleSequenceScores, RNNSequenceScores, CharacterFeatures
from .losses import PlainLoss
from .loader import get_input_format, dict_to_tf_dataset
from . import bert_optimization

__author__ = 'Tanjin He'
__maintainer__ = 'Tanjin He, Ziqin (Shaun) Rong'
__email__ = 'tanjin_he@berkeley.edu, rongzq08@gmail.com'

# Modified based on the NER Tagger code from arXiv:1603.01360 [cs.CL]

class NERModel(keras.Model):
    """
    Network architecture.
    """

    def __init__(self,
                 model_path=None,
                 model_name=None,
                 to_reload_model=False,
                 tag_scheme='iobes',
                 clean_tag=True,
                 id_to_word=None,
                 id_to_char=None,
                 id_to_tag=None,
                 classifier_type='lstm',
                 bert_path=None,
                 bert_first_trainable_layer=0,
                 word_dim=100,
                 word_lstm_dim=100,
                 word_bidirect=True,
                 word_unroll=False,
                 word_rnn_wrapper=False,
                 char_dim=25,
                 char_lstm_dim=25,
                 char_bidirect=True,
                 char_combine_method='concat',
                 char_unroll=False,
                 char_rnn_wrapper=False,
                 # cap_dim,
                 ele_num=True,
                 only_CHO=True,
                 tar_tag=False,
                 pre_tag=False,
                 # topic_dim,
                 # keyword_dim,
                 rnn_type='gru',
                 lower=False,
                 zeros=False,
                 use_ori_text_char=False,
                 crf=True,
                 crf_begin_end=True,
                 dropout=0.5,
                 pre_embedding=None,
                 lr_method='sgd-lr_.005',
                 loss_per_token=False,
                 batch_size=1,
                 num_epochs=100,
                 steps_per_epoch=3500,
                 **kwargs):
        """
        Initialize the model. We can init a empty model with model_name, or reload
        a pre-trained model from model_path

        :param model_path: File path to reload the model. If specified, model will be
        reloaded from model_path, and model_name will be discarded.
        :param model_name: Name of the model. If specified, the model will save in a
        folder called model_name.
        """
        super().__init__(**kwargs)
        # Model location
        parent_folder = os.path.abspath(
            os.path.join(
                os.path.dirname(__file__),
                '..'
            )
        )
        if not model_path:
            if model_name:
                self.model_path = os.path.join(parent_folder, 'generated',  model_name)
            else:
                self.model_path = os.path.join(parent_folder, 'generated', 'model_1')
            if not os.path.exists(self.model_path):
                os.makedirs(self.model_path)
            self.parameters_path = os.path.join(self.model_path, 'parameters.pkl')
        else:
            # reload model parameters
            self.model_path = model_path
            self.parameters_path = os.path.join(self.model_path, 'parameters.pkl')

        self.last_cp_dir = os.path.join(self.model_path, 'last_cp')
        if not (to_reload_model or os.path.exists(self.last_cp_dir)):
            os.makedirs(self.last_cp_dir)
        self.last_cp_path = os.path.join(self.last_cp_dir, 'cp.ckpt')
        self.opt_cp_dir = os.path.join(self.model_path, 'opt_cp')
        if not (to_reload_model or os.path.exists(self.opt_cp_dir)):
            os.makedirs(self.opt_cp_dir)
        self.opt_cp_path = os.path.join(self.opt_cp_dir, 'cp.ckpt')

        self.tag_scheme = tag_scheme
        self.clean_tag = clean_tag
        self.classifier_type = classifier_type
        self.bert_path = bert_path
        self.bert_first_trainable_layer = bert_first_trainable_layer
        self.word_dim = word_dim
        self.word_lstm_dim = word_lstm_dim
        self.word_bidirect = word_bidirect
        self.word_unroll = word_unroll
        self.word_rnn_wrapper = word_rnn_wrapper
        self.char_dim = char_dim
        self.char_lstm_dim = char_lstm_dim
        self.char_bidirect = char_bidirect
        self.char_combine_method = char_combine_method
        self.char_unroll = char_unroll
        self.char_rnn_wrapper = char_rnn_wrapper
        self.ele_num = ele_num
        self.only_CHO = only_CHO
        self.tar_tag = tar_tag
        self.pre_tag = pre_tag
        self.rnn_type = rnn_type
        self.lower = lower
        self.zeros = zeros
        self.use_ori_text_char = use_ori_text_char
        self.crf = crf
        self.crf_begin_end = crf_begin_end
        self.dropout = dropout
        self.pre_embedding = pre_embedding
        self.lr_method = lr_method
        self.loss_per_token = loss_per_token
        self.batch_size = batch_size
        self.num_epochs = num_epochs
        self.steps_per_epoch = steps_per_epoch
        assert id_to_tag
        if not self.bert_path:
            assert id_to_word and id_to_char and id_to_tag
        self.create_mappings(
            id_to_word=id_to_word,
            id_to_char=id_to_char,
            id_to_tag=id_to_tag,
        )
        if not to_reload_model:
            # TODO: need to think about how to save bert model
            self.save_model()

        # we will use mask defaultly
        # therefore, id of words and tags should starts from 1
        # add embedding for mask
        if self.pre_embedding is not None:
            self.word_dim = np.shape(self.pre_embedding)[1]
            self.pre_embedding_matrix = np.concatenate(
                (
                    np.zeros((1, self.word_dim)),
                    self.pre_embedding
                )
            )

        lr_method_name, lr_method_parameters = parse_lr_method(self.lr_method)
        print('lr_method_name, lr_method_parameters',
              lr_method_name, lr_method_parameters)
        if lr_method_name == 'sgd':
            self.optimizer = keras.optimizers.SGD(
                learning_rate=lr_method_parameters.get('lr', 0.005),
            )
        elif lr_method_name == 'adam':
            self.optimizer = keras.optimizers.Adam(
                learning_rate=lr_method_parameters.get('lr', 5e-5),
            )
        elif lr_method_name == 'adamdecay':
            num_train_steps = self.steps_per_epoch * self.num_epochs
            num_warmup_steps = int(
                num_train_steps * lr_method_parameters.get('warmup', 0.05)
            )
            print('num_train_steps, num_warmup_steps',
                  num_train_steps, num_warmup_steps)
            self.optimizer = bert_optimization.create_optimizer(
                init_lr=lr_method_parameters.get('lr', 5e-5),
                num_train_steps=num_train_steps,
                num_warmup_steps=num_warmup_steps,
                epsilon=lr_method_parameters.get('epsilon', 1e-6),
                weight_decay_rate=lr_method_parameters.get('weight_decay_rate', 0.01),
            )
        else:
            raise ValueError(
                "Not implemented learning method: {}".format(lr_method_name)
            )

        self.loss = PlainLoss(name='loss_layer')

        self.create_constants()
        # add layers
        if self.bert_path:
            if to_reload_model:
                bert_config = BertConfig.from_pretrained(self.bert_path)
                self.bert_layer = TFBertMainLayer(bert_config, name="bert")
            else:
                self.bert_layer = transformers.TFBertModel.from_pretrained(
                    self.bert_path,
                    from_pt=True,
                ).bert
            self.set_bert_trainable_layers(self.bert_layer, self.bert_first_trainable_layer)

            assert self.word_dim == 0
            assert self.char_dim == 0
            assert pre_embedding == None

        if self.pre_embedding is not None:
            # init embedding layer
            # https://stackoverflow.com/questions/47057361/keras-using-tensorflow-backend-masking-on-loss-function
            # mapping ids always start from 1
            # To use mapping directly, mask_zero should always be ture
            self.word_layer = keras.layers.Embedding(
                input_dim=self.n_words+1,
                output_dim=self.word_dim,
                embeddings_initializer=keras.initializers.Constant(
                    self.pre_embedding_matrix
                ),
                mask_zero=True,
                trainable=True,
                name='word_layer',
            )
        elif self.word_dim:
            self.word_layer = keras.layers.Embedding(
                input_dim=self.n_words+1,
                output_dim=self.word_dim,
                embeddings_initializer='glorot_uniform',
                mask_zero=True,
                trainable=True,
                name='word_layer',
            )

        if self.dropout:
            self.dropout_layer = keras.layers.Dropout(self.dropout)

        # TODO: it is better to assign a name to each layer in the future
        if self.char_dim:
            self.char_layer = CharacterFeatures(
                n_chars=self.n_chars,
                char_dim=self.char_dim,
                char_lstm_dim=self.char_lstm_dim,
                char_bidirect=self.char_bidirect,
                char_combine_method=self.char_combine_method,
                batch_size=self.batch_size,
                rnn_type=self.rnn_type,
                unroll=self.char_unroll,
                rnn_wrapper=self.char_rnn_wrapper,
            )

        if self.classifier_type == 'simple':
            print('classifier_type', classifier_type)
            # activation function here is always None
            self.unary_score_layer = SimpleSequenceScores(
                n_tags=self.n_tags,
            )
        elif self.classifier_type == 'lstm':
            print('classifier_type', classifier_type)
            self.unary_score_layer = RNNSequenceScores(
                word_bidirect=self.word_bidirect,
                word_lstm_dim=self.word_lstm_dim,
                n_tags=self.n_tags,
                rnn_type=self.rnn_type,
                unroll=self.word_unroll,
                rnn_wrapper=self.word_rnn_wrapper,
            )
        else:
            raise NotImplementedError('Classifier {} not recognized!'.format(self.classifier_type))

        self.compile(
            optimizer=self.optimizer,
            loss=self.loss,
        )

    def create_mappings(self, id_to_word, id_to_char, id_to_tag):
        """
        We need to save the mappings if we want to use the model later.

        :param id_to_word: mapping from a number (id) to a word in text
        :param id_to_tag: mapping from a number (id) to a tag of word
        """
        self.id_to_word = id_to_word
        self.id_to_char = id_to_char
        self.id_to_tag = id_to_tag

        if self.id_to_word:
            self.n_words = len(self.id_to_word)
        if self.id_to_char:
            self.n_chars = len(self.id_to_char)

        self.tag_to_id = {v: k for (k, v) in self.id_to_tag.items()}
        self.all_tags = np.array(sorted(
            self.tag_to_id.keys(),
            key=lambda x: self.tag_to_id[x]
        ))

        self.all_tag_ids = np.array(
            [self.tag_to_id[t] for t in self.all_tags],
            dtype=np.int32
        )
        self.n_tags = len(self.all_tags)
        self.tag_tag_to_idx = {
            t: i for (i, t) in enumerate(self.all_tags)
        }
        self.tag_id_to_idx = {
            id: i for (i, id) in enumerate(self.all_tag_ids)
        }
        self.tag_id_to_idx_lookup = tf.lookup.StaticHashTable(
            initializer=tf.lookup.KeyValueTensorInitializer(
                keys=self.all_tag_ids,
                values=tf.range(self.n_tags, dtype=tf.int32),
            ),
            default_value=tf.constant(0, dtype=tf.int32),
            name="tag_id_to_idx_lookup",
        )
        # attention: string in tensor is bytes
        # need to use x.decode('utf-8') after lookup
        self.tag_id_to_tag_lookup = tf.lookup.StaticHashTable(
            initializer=tf.lookup.KeyValueTensorInitializer(
                keys=self.all_tag_ids,
                values=self.all_tags,
            ),
            default_value=tf.constant('<MASK>'),
            name="tag_id_to_tag_lookup",
        )

    def set_bert_trainable_layers(self, bert_main_layer, bert_first_trainable_layer=0):
        # get name of encoder layers
        pattern_bert_layer = regex.compile('.*(layer_._[0-9]+).*')
        bert_encoder_layer_ids = []
        for w in bert_main_layer.encoder.weights:
            tmp_m = pattern_bert_layer.match(w.name)
            if tmp_m and tmp_m.group(1) not in bert_encoder_layer_ids:
                bert_encoder_layer_ids.append(tmp_m.group(1))

       # set bert layers trainable and not trainable
        bert_trainable_layers = bert_encoder_layer_ids[bert_first_trainable_layer:]
        pattern_bert_trainable_layer = regex.compile(
            '.*encoder.+({}).*'.format('|'.join(bert_trainable_layers))
        )
        for w in bert_main_layer.weights:
            if (
                len(bert_trainable_layers) > 0
                and pattern_bert_trainable_layer.match(w.name)
            ):
                w._trainable = True
            else:
                w._trainable = False
            # only for test
            if bert_first_trainable_layer == 100:
                w._trainable = True

    def create_constants(self):
        self.small = -1000.0

        if self.crf and self.crf_begin_end:
            self.b_s = np.ones((1, self.n_tags + 2,), dtype=np.float32) * self.small
            self.b_s[:, -2] = 0.0
            self.e_s = np.ones((1, self.n_tags + 2,), dtype=np.float32) * self.small
            self.e_s[:, -1] = 0.0

            self.b_idx = np.array([self.n_tags], dtype=np.int32)
            self.e_idx = np.array([self.n_tags+1], dtype=np.int32)

    def build(self, input_shape):
        """

        :param input_shape: (None, num_eles)
        :return:
        """
        if self.crf_begin_end:
            self.transitions = self.add_weight(
                shape=(self.n_tags+2, self.n_tags+2),
                initializer='glorot_uniform',
                trainable=True,
                name='transitions'
            )
        else:
            self.transitions = self.add_weight(
                shape=(self.n_tags, self.n_tags),
                initializer='glorot_uniform',
                trainable=True,
                name='transitions'
            )

    def call(self, inputs, training=None):
        # preprocess inputs
        processed_inputs = self.prepare_final_inputs(
            inputs, training=training
        )
        _x = processed_inputs['final_inputs']
        word_mask_bool = processed_inputs['word_mask_bool']
        tags_idxes = processed_inputs['tags_idxes']
        sents_len = processed_inputs['sents_len']
        sents_num_tokens = processed_inputs['sents_num_tokens']
        word_mask_float = tf.cast(word_mask_bool, tf.float32)
        # layers
        # tags_scores: (None, seq_len, n_tags)
        tags_scores = self.get_tags_scores(
            final_inputs=_x,
            word_mask_bool=word_mask_bool,
            word_mask_float=word_mask_float,
        )

        # calculate loss
        if self.crf:
            # TODO: it is better to use scatter_nd to rearrange the scores
            #  to drop out masked elements for bert, which makes crf easier
            if self.crf_begin_end:
                observations = self.add_begin_end_score(
                    tags_scores,
                    sents_len,
                    word_mask_float,
                )
                padded_tags_idxes = self.add_begin_end_idx(
                    tags_idxes,
                    sents_len
                )
                padded_sents_len = sents_len + 2
            else:
                observations = tags_scores
                padded_tags_idxes = tags_idxes
                padded_sents_len = sents_len

            # crf loss
            log_likelihood, transition_params = tfa.text.crf_log_likelihood(
                inputs=observations,
                tag_indices=padded_tags_idxes,
                sequence_lengths=padded_sents_len,
                transition_params=self.transitions,
            )
            loss = -log_likelihood
        else:
            # there is already "-" in keras.losses classes
            loss = keras.losses.sparse_categorical_crossentropy(
                y_true=tags_idxes,
                y_pred=tags_scores,
                from_logits=True,
            )
            loss = loss * word_mask_float
            loss = tf.reduce_sum(loss, axis=-1)

        # # TODO: if mean is better, do we need to weight by sent len?
        if self.loss_per_token:
            loss = loss/tf.cast(sents_num_tokens, tf.float32)

        return loss

    def prepare_final_inputs(self, inputs, training=None):
        # # save parameters
        # if training:
        #     saved_locals = locals()
        #     self.parameters = saved_locals
        #     self.parameters.update(self.parameters['kwargs'])
        #     del self.parameters['kwargs']
        #     del self.parameters['self']
        #     # not save pre_emb because it might be very large
        #     # and duplicates the original embedding file
        #     del self.parameters['pre_emb']
        #     del self.parameters['training']
        #     with open(self.parameters_path, 'wb') as f:
        #         pickle.dump(self.parameters, f)

        # word_ids: (None, max_seq_len, )
        word_ids = inputs['words']
        # word_mask_bool: (None, max_seq_len, )
        word_mask_bool = tf.cast(inputs['score_mask'], tf.bool)

        # Final input (all word features)
        _x = []

        # bert_path
        if self.bert_path:
            # note: return_dict=True will be applied by default
            # it is commeted out because TF says "The parameter `return_dict`
            # cannot be set in graph mode and will always be set to `True`."
            bert_output = self.bert_layer(
                word_ids,
                attention_mask=inputs['bert_attention_mask'],
                token_type_ids=inputs['bert_token_type_ids'],
                # return_dict=True
            )
            _x.append(bert_output.last_hidden_state)

        # word dim
        if self.word_dim:
            # attention! word_ids start from 1, and mask_zero==True
            # thus, they are consistent
            # otherwise, a lookup table is needed to convert id to idx in embeddings
            word_input = self.word_layer(word_ids)
            _x.append(word_input)

        # char dim
        if self.char_dim:
            char_ids = inputs['chars']
            char_input = self.char_layer(char_ids)
            _x.append(char_input)

        if self.ele_num:
            ele_num = tf.expand_dims(inputs['ele_num'], axis=2)
            _x.append(ele_num)

        if self.only_CHO:
            only_CHO = tf.expand_dims(
                tf.cast(inputs['only_CHO'], tf.float32),
                axis=2
            )
            _x.append(only_CHO)

        if self.tar_tag:
            # only for coding test
            tar_tag = tf.expand_dims(
                tf.cast(inputs['tar_tag'], tf.float32),
                axis=2
            )
            _x.append(tar_tag)

        if self.pre_tag:
            # only for coding test
            pre_tag = tf.expand_dims(
                tf.cast(inputs['pre_tag'], tf.float32),
                axis=2
            )
            _x.append(pre_tag)

        # Prepare final input
        if len(_x) == 1:
            final_inputs = _x[0]
        else:
            final_inputs = tf.concat(_x, axis=-1)

        if self.dropout:
            final_inputs = self.dropout_layer(final_inputs, training=training)

        # get final_y, which is tags_idxes used to cal cost
        tag_ids = inputs['tags']
        tags_idxes = self.tag_id_to_idx_lookup.lookup(tag_ids)

        # get sentence length
        sents_len = tf.reshape(inputs['length'], (-1,))

        # get number of tokens in batched sentences
        sents_num_tokens = tf.reshape(inputs['num_tokens'], (-1,))

        processed_inputs = {
            'final_inputs': final_inputs,
            'word_mask_bool': word_mask_bool,
            'tags_idxes': tags_idxes,
            'sents_len': sents_len,
            'sents_num_tokens': sents_num_tokens,
        }
        return processed_inputs

    def get_tags_scores(self, final_inputs, word_mask_bool, word_mask_float):
        """
        return tags_scores
        if crf, scores are logits
        if not crf, scores are normalized with softmax

        :param final_inputs:
        :param word_mask_bool:
        :param word_mask_float:
        :return: (None, max_seq_len, n_tags)
        """
        # layers to get scores before crf
        # the score here is not softmax normalized
        tags_scores = self.unary_score_layer(final_inputs, mask=word_mask_bool)
        tags_scores = tags_scores * tf.expand_dims(word_mask_float, axis=2)
        # If not use crf, normalized tags_scores with softmax to get probability
        # at this time, we should set logits=False in sparse_categorical_crossentropy
        # This is same as not use softmax and set logits=True in
        # sparse_categorical_crossentropy. The loss is not changed
        # ref: https://github.com/tensorflow/tensorflow/blob/1cf0898dd4331ba
        # f93fe77205550f2c2e6c90ee5/tensorflow/python/keras/backend.py#L4508
        return tags_scores

    def pad_with_begin_end(self, seq, sentence_len, begin, end):
        """

        :param seq: (max_seq_len, ) or (max_seq_len, dim)
        :param sentence_len: int, real seq len
        :return:
        """
        padded_seq = tf.concat(
            [
                begin,
                seq[: sentence_len],
                end,
                seq[sentence_len:]
            ],
            axis=0
        )
        return padded_seq

    def add_begin_end_score(self,
                            tags_scores,
                            sents_len,
                            word_mask_float):
        """
        add begin/end tag as the last 2 dimensions in tags_scores

        :param tags_scores: (batch_size, max_seq_len, n_tags)
                    or (batch_size=1, real_seq_len, n_tags)
        :param sents_len: (batch_size, )
                    or (batch_size=1, )
                    consistent with tags_score
        :param word_mask_float: (batch_size, max_seq_len)
                    or (batch_size=1, real_seq_len)
                    consistent with tags_score
        :return: (batch_size, max_seq_len+2, n_tags+2)
        """

        observations = tf.concat(
            [
                tags_scores,
                tf.tile(
                    tf.expand_dims(word_mask_float, 2),
                    multiples=[1, 1, 2]
                )*self.small,
            ],
            axis=-1,
        )

        observations, _ = tf.map_fn(
            lambda x: (
                self.pad_with_begin_end(x[0], x[1], self.b_s, self.e_s),
                x[1]
            ),
            (observations, sents_len)
        )

        return observations

    def add_begin_end_idx(self, tags_idxes, sents_len):
        padded_tags_idxes, _ = tf.map_fn(
            lambda x: (
                self.pad_with_begin_end(x[0], x[1], self.b_idx, self.e_idx),
                x[1]
            ),
            (tags_idxes, sents_len)
        )
        return padded_tags_idxes

    def predict(self, x_batches, return_tensor=False, **kwargs):
        """
        return tags_scores, each sentence would be in real length
        rather than the padded ones
        Scores are logits rather than probabilities

        :param x_batches: all data in multiple batches
        :param kwargs:
        :return:
        """
        # here only scores without crf considered
        # use  predict_idx to get the ids
        processed_inputs = x_batches.map(self.prepare_final_inputs)
        scores = processed_inputs.map(
            lambda x: self.get_tags_scores(
                final_inputs=x['final_inputs'],
                word_mask_bool=x['word_mask_bool'],
                word_mask_float=tf.cast(x['word_mask_bool'], tf.float32),
            )
        ).unbatch()
        word_mask = processed_inputs.map(
            lambda x: x['word_mask_bool']
        ).unbatch()
        scores_mask = tf.data.Dataset.zip((scores, word_mask))
        # restore the length from max_seq_len to real_seq_len
        scores = scores_mask.map(lambda s, mask: s[mask])
        # return tensor for further calculation with tf ops
        # if return tensor and crf==True, we return observations here
        # rather than scores for convenience in furthere calculation
        if not return_tensor:
            scores = [s.numpy() for s in scores]
        return scores

    def predict_idx(self, x_batches, **kwargs):
        # here scores_sents is unbatched data
        scores_sents = self.predict(x_batches, return_tensor=True, **kwargs)

        if self.crf:
            if self.crf_begin_end:
                observations = scores_sents.batch(1)
                observations = observations.map(
                    lambda x: self.add_begin_end_score(
                        tags_scores=x,
                        sents_len=tf.shape(x)[1:2],
                        word_mask_float=tf.ones((1, tf.shape(x)[1])),
                    )
                ).unbatch()
            else:
                observations = scores_sents
            observations = [s.numpy() for s in observations]
            tag_idx_sents = list(map(
                lambda x: (tfa.text.crf.viterbi_decode(
                    x,
                    self.transitions
                ))[0],
                observations
            ))
            if self.crf_begin_end:
                tag_idx_sents = list(map(
                    lambda x: x[1:-1],
                    tag_idx_sents
                ))
        else:
            tag_idx_sents = scores_sents.map(
                lambda x: tf.argmax(x, axis=-1, output_type=tf.int32)
            )
            tag_idx_sents = [s.numpy() for s in tag_idx_sents]
        return tag_idx_sents

    # args and kwargs are the same as original predict function
    # https://www.tensorflow.org/api_docs/python/tf/keras/Model
    def predict_id(self, x_batches, **kwargs):
        """

        :param x_batches: all data in multiple batches
        :param kwargs:
        :return:
        """

        tag_idx_sents = self.predict_idx(x_batches, **kwargs)
        tag_ids_sents = list(map(
            lambda x: self.all_tag_ids[x],
            tag_idx_sents
        ))
        return tag_ids_sents

    def predict_label(self, x_batches, **kwargs):
        """

        :param x_batches: all data in multiple batches
        :param kwargs:
        :return:
        """
        tag_idx_sents = self.predict_idx(x_batches, **kwargs)
        label_predictions = list(map(
            lambda x: self.all_tags[x],
            tag_idx_sents
        ))
        if self.clean_tag:
            label_predictions = list(map(self.remove_broken_entities, label_predictions))
        return label_predictions

    def remove_broken_entities(self, tag_seq):
        clean_seq = []
        for tag in tag_seq:
            if (
                tag.startswith('B-')
                or tag.startswith('S-')
                or tag.startswith('O')
            ):
                clean_seq.append(tag)
            elif (
                len(clean_seq) > 0
                and clean_seq[-1][2:] == tag[2:]
                and (not clean_seq[-1].startswith('E-'))
            ):
                clean_seq.append(tag)
            else:
                clean_seq.append('O')
        return clean_seq

    def evaluate_id(self, x_batches, y_true, raw_sentences, **kwargs):
        """

        :param x_batches: all data in multiple batches
        :param y_true:
        :param raw_sentences:
        :param kwargs:
        :return: averaged F1 score
        """

        labels_true = y_true.map(
            lambda x: self.tag_id_to_tag_lookup.lookup(x)
        )

        return self.evaluate_label(x_batches, labels_true, raw_sentences, **kwargs)

    def evaluate_label(self, x_batches, labels_true, raw_sentences, **kwargs):
        tags_pred = self.predict_label(x_batches)

        # labels_true was padded, here we recover it
        labels_true = labels_true.unbatch()
        word_mask = x_batches.map(
            lambda x: tf.cast(x['score_mask'], tf.bool)
        ).unbatch()
        labels_mask = tf.data.Dataset.zip((labels_true, word_mask))
        # restore the length from max_seq_len to real_seq_len
        labels_true = labels_mask.map(lambda l, mask: l[mask])
        labels_true = [l.numpy().astype('U') for l in labels_true]

        predictions = []
        count = np.zeros((self.n_tags, self.n_tags), dtype=np.int32)

        for sent_index, raw_sentence in enumerate(raw_sentences):
            r_tags = labels_true[sent_index]
            r_idx = [self.tag_tag_to_idx[t] for t in r_tags]

            p_tags = tags_pred[sent_index]
            p_idx = [self.tag_tag_to_idx[t] for t in p_tags]

            assert len(p_tags) == len(r_tags)
            assert len(r_tags) == len(raw_sentence)

            if self.tag_scheme == 'iobes':
                p_tags = iobes_iob(p_tags)
                r_tags = iobes_iob(r_tags)
            for i, (i_pred, i_real) in enumerate(zip(p_idx, r_idx)):
                new_line = " ".join(
                    [raw_sentence[i]['text'],  r_tags[i], p_tags[i]]
                )
                predictions.append(new_line)
                count[i_real, i_pred] += 1
            predictions.append("")

        eval_lines = evaluate_lines(predictions)
        eval_lines = eval_lines.rstrip()
        eval_lines = eval_lines.split('\n')
        eval_lines = [l.rstrip() for l in eval_lines]
        for line in eval_lines:
            print(line)

        # Confusion matrix with accuracy for each tag
        print(("{: >2}{: >7}{: >7}%s{: >9}" % ("{: >7}" * self.n_tags)).format(
            "ID", "NE", "Total",
            *([tag for tag in self.all_tags] + ["Percent"])
        ))
        for i in range(self.n_tags):
            print(("{: >2}{: >7}{: >7}%s{: >9}" % ("{: >7}" * self.n_tags)).format(
                str(i), self.all_tags[i], str(count[i].sum()),
                *([count[i][j] for j in range(self.n_tags)] +
                  ["%.3f" % (count[i][i] * 100. / max(1, count[i].sum()))])
            ))

        # Global accuracy
        print("%i/%i (%.5f%%)" % (
            count.trace(), count.sum(),
            100. * count.trace() / max(1, count.sum())
        ))

        return float(eval_lines[1].strip().split()[-1])

    def save_model(self, save_weights=False):
        """
        Write components values to disk.
        """

        signature = inspect.signature(self.__class__.__init__)
        model_config_essential = dict(filter(
            lambda x: x[0] in signature.parameters,
            self.__dict__.items()
        ))
        with open(
            os.path.join(self.model_path, 'model_config.pkl'),
            'wb'
        ) as fw:
            pickle.dump(model_config_essential, fw)
        if save_weights:
            self.save_weights(self.opt_cp_path)

    @classmethod
    def create_scratch_model(cls, model_path, bert_path=None, to_reload_model=True):
        """
        Create model from config file w/o initialization

        :param model_path:
        :return:
        """
        config_path = os.path.join(model_path, 'model_config.pkl')
        with open(config_path, 'rb') as fr:
            model_config = pickle.load(fr)
        model_config['model_path'] = model_path
        if bert_path:
            model_config['bert_path'] = bert_path
        model_config['to_reload_model'] = to_reload_model
        model = cls(**model_config)
        batch_size = model_config['batch_size']
        # run on one sample to build model
        data_dicts = [
            {
                'words': [1, ],
                'chars': [[1,], ],
                'ele_num': [0.0, ],
                'only_CHO': [0, ],
                'tar_tag': [0, ],
                'pre_tag': [0, ],
                'tags': [1, ],
                'length': 1,
                'num_tokens': 1,
                'bert_token_type_ids': [0, ],
                'bert_attention_mask': [1, ],
                'score_mask': [1, ],
            }
        ] * batch_size
        data_type, data_shape, padded_data_shape = get_input_format(model_type='MER')
        data_X, data_Y = dict_to_tf_dataset(
            data_dicts,
            data_type,
            data_shape,
            padded_shape=padded_data_shape,
            column_y=None,
            batch_size=batch_size,
        )
        # the weight has been changed even only fit once
        # need to reload weight to use it
        # TODO: is there anyway to compile the model without calling fit?
        model.fit(
            x=tf.data.Dataset.zip((data_X, data_Y)),
            epochs=1,
            verbose=0,
        )
        return model

    @classmethod
    def reload_model(cls, model_path, cp_path=None, bert_path=None):
        """
        Load components values from disk.
        """
        # TODO: should be able to create scratch bert model with vocab
        #  rather than whole checkpoint file
        model = NERModel.create_scratch_model(
            model_path=model_path,
            bert_path=bert_path,
            to_reload_model=True,
        )
        if cp_path is None:
            cp_path = os.path.join(model_path, 'opt_cp', 'cp.ckpt')
        model.load_weights(cp_path)
        return model


#!/usr/bin/python

import math
import numpy as np
import rospy as rp
from geometry_msgs.msg import Twist

class VelocityController:
    def __init__(self):
        #controller gain
        self.kp = 1.2

        self.goal_angle = 0.0
        self.current_yaw = 0.0

        self.vel_pub = rp.Publisher('/cmd_vel', Twist, queue_size=10)

    def publish_velocity(self):
        """
            Calculate a velocity using a simple proportional gain controller, with
            an error value based on the current yaw and a goal angle.
            Publish as a Twist message.

        """
        command = Twist()

        #calculate error value - if statement accounts for wraparound of angle value
        if self.goal_angle > math.pi/2.0 and self.current_yaw < -math.pi/2.0:
            err = -((math.pi - abs(self.goal_angle)) + (math.pi - abs(self.current_yaw)))
        elif self.goal_angle < -math.pi/2.0 and self.current_yaw > math.pi/2.0:
            err = (math.pi - abs(self.current_yaw)) + (math.pi - abs(self.goal_angle))
        else:
            err = self.goal_angle - self.current_yaw

        #simple proportional controller to set command velocity
        command.angular.z = self.kp * err

        self.vel_pub.publish(command)

        # rp.loginfo('Current Angle: {}; Target: {}'.format(self.current_yaw, self.goal_angle))
        # rp.loginfo('Published angular velocity: {}'.format(command.angular.z))

    def set_goal_angle(self, angle):
        """
            Update the goal angle value. Before updating, the angle is limited to
            the range +-pi

        """

        #limit angle to range -pi to +pi
        angle = np.clip(angle, a_min=-math.pi, a_max=math.pi)

        #update goal angle and publish a command to turn the robot towards that angle
        self.goal_angle = angle
        self.publish_velocity()

        # rp.loginfo('Set goal angle to {} ({} degrees)'.format(angle, math.degrees(angle)))

    def set_current_yaw(self, yaw):
        """
            Update the current yaw value of the robot.
        """
        self.current_yaw = yaw

        # rp.loginfo('Set current yaw to {} ({} degrees)'.format(yaw, math.degrees(yaw)))


from workers.serializers import *
from workers.models import *
from projects.models import *
from rest_framework import viewsets, permissions, status
from rest_framework.response import Response
from rest_framework.views import APIView
from django.db.models import Q
from django.db.models import Prefetch
import datetime
from teams.serializers import *
from teams.models import *

def excluding_keys(dict, keys):
    return {x: dict[x] for x in dict if x not in keys}

class WorkerViewSet(viewsets.ModelViewSet):
    queryset = Worker.objects.all()
    serializer_class = WorkerSerializer
    permission_classes = [permissions.AllowAny]

class WorkerList(APIView):
    permission_classes = [permissions.AllowAny]

    def post(self, request):

        data = request.data
        worker_data = excluding_keys(data, 'team')
        worker_serializer = WorkerSerializer(data = worker_data)
        if worker_serializer.is_valid():
            worker_serializer.save()

            try:
                created_worker = Worker.objects.get(id = worker_serializer.data['id'])
                worker_team = Team.objects.get(id = data['team'])
                new_membership = TeamMembership(worker = created_worker, team = worker_team)
                new_membership.save()
            except:
                return Response(
                    "Something went wrong",
                    status=status.HTTP_400_BAD_REQUEST)

            return Response({
					'worker_data': worker_serializer.data,
					'team': data['team']
				})

        else:
            return Response(worker_serializer.errors, status=status.HTTP_400_BAD_REQUEST)

class WorkerDetail(APIView):
    permission_classes = [permissions.AllowAny]

    def put(self, request, pk):
        editing_data = excluding_keys(request.data, 'team')
        team_id = request.data['team']

        editing_worker = Worker.objects.get(id = pk)
        worker_serializer = WorkerSerializer(editing_worker, editing_data)
        if worker_serializer.is_valid():
            worker_serializer.save()
            try:
                editing_membership = TeamMembership.objects.filter(worker = pk)[0]
                membership_serializer = TeamMembershipSerializer(editing_membership, {
                    'worker': pk, 'team': team_id})
                if membership_serializer.is_valid():
                    membership_serializer.save()
                    return Response({
                        'worker_data': worker_serializer.data,
                        'team_data': membership_serializer.data
                    })
                else:
                    return Response(membership_serializer.errors, status=status.HTTP_400_BAD_REQUEST)

            except TeamMembership.DoesNotExist:
                return Response(f"Membership from worker {pk} not exist", status=status.HTTP_404_NOT_FOUND)

        else:
            return Response(worker_serializer.errors, status=status.HTTP_400_BAD_REQUEST)

class WeekViewSet(viewsets.ModelViewSet):
    queryset = Week.objects.all()
    serializer_class = WeekSerializer
    permission_classes = [permissions.AllowAny]

class JobTimeViewSet(viewsets.ModelViewSet):
    queryset = JobTime.objects.all()
    serializer_class = JobTimeSerializer
    permission_classes = [permissions.AllowAny]

    def create(self, request, *args, **kwargs):
        data = request.data
        many = isinstance(data, list)
        serializer = self.get_serializer(data=data, many=many)
        serializer.is_valid(raise_exception=True)
        self.perform_create(serializer)
        headers = self.get_success_headers(serializer.data)
        return Response(
            serializer.data,
            status=status.HTTP_201_CREATED,
            headers=headers
        )

MONTH_LIST = [
    'Styczeń',
    'Luty',
    'Marzec',
    'Kwiecień',
    'Maj',
    'Czerwiec',
    'Lipiec',
    'Sierpień',
    'Wrzesień',
    'Październik',
    'Listopad',
    'Grudzień']

class JobTimeAvailabilityDetail(APIView):
    permission_classes = [permissions.AllowAny]

    def get(self, request, pk):
        week_id = pk

        try:
            Week.objects.get(id = week_id)

            response_data = []
            workers = Worker.objects.filter(role = 'W')
            for worker in workers:
                worker_id = worker.id
                job_time = JobTime.objects.filter(
                    Q(worker = worker_id) & Q(week = week_id))

                if len(job_time) > 0:
                    availability = job_time[0].availability
                    worker_team = TeamMembership.objects.filter(worker = 5)[0].team

                    response_data.append({
                        'id': worker.id,
                        'name': f"{worker.firstName} {worker.lastName}",
                        'time': availability,
                        'team': {
                            'id': worker_team.id,
                            'name': worker_team.shortcut
                        }
                    })

            return Response(response_data)

        except Week.DoesNotExist:
            return Response(f"Week {week_id} not in db", status=status.HTTP_404_NOT_FOUND)

    def put(self, request, pk):
        full_data = request.data["data"]
        week_id = request.data["week_id"]

        response_data = []

        for single_data in full_data:
            worker_id = single_data["id"]
            time = single_data["time"]

            job_list = JobTime.objects.filter(Q(worker = worker_id) & Q(week = week_id))
            for single_job in job_list:
                data_to_put = {
                    "worker": single_job.worker.id,
                    "project": single_job.project.id,
                    "week": single_job.week.id,
                    "availability": time
                }
                editing_jobtime = JobTime.objects.get(id = single_job.id)
                jobtime_serializer = JobTimeSerializer(editing_jobtime, data_to_put)
                if jobtime_serializer.is_valid():
                    jobtime_serializer.save()
                    response_data.append(jobtime_serializer.data)
                else:
                    return Response(jobtime_serializer.errors, status=status.HTTP_400_BAD_REQUEST)

        return Response({'data': response_data})

class WorkerProjectsDetail(APIView):
    permission_classes = [permissions.AllowAny]

    def put(self, request, pk):
        full_data = request.data["data"]
        week_id = request.data["week_id"]

        response_data = []

        for single_data in full_data:
            worker_id = single_data["worker_id"]
            projects = single_data["projects"]

            for project in projects:
                try:
                    job_time = JobTime.objects.get(
                        Q(worker = worker_id) & Q(week = week_id) & Q(project = project['id']))
                    editing_jobtime = JobTime.objects.get(id = job_time.id)
                    job_time = {
                        'id': job_time.id,
                        'worker': worker_id,
                        'project': project['id'],
                        'workPart': project['time'],
                    }
                    jobtime_serializer = JobTimeSerializer(editing_jobtime, job_time)
                    if jobtime_serializer.is_valid():
                        jobtime_serializer.save()
                        response_data.append(jobtime_serializer.data)
                    else:
                        return Response(jobtime_serializer.errors, status=status.HTTP_400_BAD_REQUEST)

                except JobTime.DoesNotExist:
                    new_job_time = JobTime(
                        worker = Worker.objects.get(id = worker_id),
                        project = Project.objects.get(id = project['id']),
                        week = Week.objects.get(id = week_id),
                        workPart = project['time'])
                    new_job_time.save()
                    response_data.append({
                        'id': new_job_time.id,
                        'worker': worker_id,
                        'project': project['id'],
                        'workPart': project['time'],
                    })

        return Response({'data': response_data})

class WeekList(APIView):
    permission_classes = [permissions.AllowAny]

    def get(self, request, pk):
        weeks = Week.objects.filter(dateEnd__year=pk)
        new_weeks = []
        new_end = datetime.date(1, 1, 1)
        if len(weeks) == 0:
            new_end = datetime.date(pk, 1, 1)
            while new_end.weekday() != 6:
                new_end += datetime.timedelta(days = 1)
        else:
            last_week = weeks[len(weeks) - 1]
            new_end = last_week.dateEnd + datetime.timedelta(days = 7)

        while new_end.year != pk + 1:
            new_start = new_end - datetime.timedelta(days = 6)
            week_name = 'T' + str(new_end.isocalendar().week)
            new_weeks.append({
                'weekName': week_name,
                'dateStart': new_start,
                'dateEnd': new_end,
                'hours': 40
            })
            new_end += datetime.timedelta(days = 7)

        if len(new_weeks) > 0:
            serializer = WeekSerializer(data = new_weeks, many = True)
            if serializer.is_valid():
                serializer.save()
            else:
                return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

        result = Week.objects.filter(
            (Q(dateStart__year = pk) & ~Q(dateEnd__month = 12)) |
            (Q(dateEnd__year = pk) & ~Q(dateEnd__month = 1)))
        serializer = WeekYearSerializer(result, many=True)

        pre_final = [{'month': i, 'weeks': []} for i in MONTH_LIST]
        for record in serializer.data:
            for i in range(1, 13):
                if i in record['months']:
                    pre_final[i - 1]['weeks'].append(excluding_keys(record, 'months'))

        final = []
        for month in pre_final:
            if len(month['weeks']) != 0:
                final.append(month)

        return Response(final)

class WorkerAvailabilityViewSet(viewsets.ModelViewSet):
    queryset = Worker.objects.all()
    serializer_class = WorkerAvailabilitySerializer
    permission_classes = [permissions.AllowAny]

class WeekYearViewSet(APIView):
    permission_classes = [permissions.AllowAny]

    def get(self, request, pk):
        result = Week.objects.filter(
            Q(dateStart__year=pk) | Q(dateEnd__year=pk))

        serializer = WeekYearSerializer(result, many=True)

        return Response(serializer.data)

class PlanningViewSet(viewsets.ModelViewSet):
    serializer_class = PlanningSerializer
    permission_classes = [permissions.AllowAny]

    def get_queryset(self):
        queryset = Worker.objects.filter(role='W').prefetch_related(Prefetch(
            'JT_worker', queryset=JobTime.objects.filter(week=self.kwargs.get('week_id')), to_attr='filtered_jobtimes'))

        return queryset


class Solution:
    # 1-D Dynamic Programming - Bottom-Up Approach
    # T: O(m*n), M: O(n), where m is num of rows, n is num of cols
    def uniquePaths(self, m: int, n: int) -> int:
        prev_row = [0] * n  # Solve last row base case

        for row in range(m-1, -1, -1):
            curr_row = [0] * n
            curr_row[n - 1] = 1 # Solve last col base case

            # Solve curr case using bottom up approach
            for col in range(n-2, -1, -1):
                curr_row[col] = curr_row[col + 1] + prev_row[col]

            prev_row = curr_row

        return prev_row[0]

    '''
    # 2-D Dynamic Programming - Full Grid/Bottom-Up Approach
    # T&M: O(m*n), where m is num of rows, n is num of cols
    def uniquePaths(self, m: int, n: int) -> int:
        grid = [[0] * n for ln in range(m)]

        # Solve base case for last row & last cols
        for row in range(m):
            grid[row][n-1] = 1
        for col in range(n):
            grid[m-1][col] = 1
        print(grid)

        # Solve for the rest of the grid using bottom-up approach
        for row in range(m-2, -1, -1):
            for col in range(n-2, -1, -1):
                grid[row][col] = grid[row+1][col] + grid[row][col+1]
        print(grid)

        return grid[0][0]
    '''

sol = Solution()

# Ex1:
attempt = sol.uniquePaths(m = 3, n = 7)
assert attempt == 28, f'Expected {28}, but got {attempt}'
# Ex2:
attempt = sol.uniquePaths(m = 3, n = 2)
assert attempt == 3, f'Expected {3}, but got {attempt}'


# F06 - Mengubah stok game
def update_stok(role,hasil):
    if role == 'Admin':
        id_game = input('Masukkan ID game: ')
        jumlah = int(input('Masukkan jumlah: '))

        for line in hasil:
            if id_game == line[0]:
                if (line[5] + jumlah) >= 0:
                    line[5] += jumlah
                    print('Stok game', line[1], 'berhasil ditambahkan. Stok sekarang:', line[5])
                    break
                else:
                    print('Stok game', line[1], 'gagal dikurang karena stok kurang. Stok sekarang:', line[5])
                    break
            else:
                print('Tidak ada game dengan ID tersebut')
                break
    else:
        print('Kamu tidak memiliki akses pada menu ini')

# update_stok('admin', hasil)

# F08 - Beli Game
import datetime

date = datetime.datetime.now()
id_game = input('Masukkan ID Game: ')

def buy (role, id_game, user_id, saldo, game, kepemilikan):
    if role == 'User':
        # history = [open("riwayat.csv")]
        for line in kepemilikan :
            if (id_game == line[0]) and (user_id == line[1]):
                print("Anda sudah memiliki Game tersebut!")
                break
            else:
                for line in game:
                    if (id_game == line[0]):
                        if (line[5] > 0):
                            if saldo > line[4]:
                                saldo -= line[4]
                                print("Game", line[1], "berhasil dibeli!")
                                history = [line[0],line[1],line[4],user_id,date.year]
                                return history, saldo # ini nanti harus dipisahin lagi waktu di main programnya
                                # https://note.nkmk.me/en/python-function-return-multiple-values/#:~:text=Return%20multiple%20values%20using%20commas,the%20return%20%2C%20separated%20by%20commas.
                            else:
                                print("Saldo anda tidak cukup untuk membeli game tersebut")
                                break
                        else:
                            print("Stok Game tersebut sedang habis!")
                            break
                    else:
                        print("Tidak ada Game dengan ID tersebut")
                        break

    else:
        print("Anda tidak memiliki akses pada menu ini.")

# F10 - Mencari game yang dimiliki
def search_my_game(role, user_id,kepemilikan):
    if role == "User":
        game_id = input("Masukkan ID Game: ")
        tahun_rilis = input("Masukkan Tahun Rilis Game: ")
        for line in kepemilikan:
            pass

# F17 - Exit
def exit(hasil):
    answer = input("Apakah Anda mau melakukan penyimpanan file yang sudah diubah? (y/n) ")
    while (answer != "y" or answer != "Y" or answer != "n" or answer != "N"):
        answer = input("Apakah Anda mau melakukan penyimpanan file yang sudah diubah? (y/n) ")
    if (answer == "Y" or answer == 'y'):
        # save()
        print("Data berhasil disimpan.")
        print("Anda telah keluar dari program.")
    else:
        print("Data tidak disimpan.")
        print("Anda telah keluar dari program.")
    # close()


import numpy as np

class User:
    def __init__(self, select_num, itemProb):
        self.L = select_num
        self.K = len(itemProb)
        self.itemProb = itemProb

    #for each items, user makes clicked or unclicked with itemProb
    def react(self, items, itemProb_idx):
        assert len(items) == self.L
        clicked = [False for _ in range(self.L)]
        for i in range(self.L):
            if 0 == np.random.binomial(1, self.itemProb[itemProb_idx][items[i]]): continue
            clicked[i] = True
            return clicked, i
        return clicked, len(clicked)-1


import json
import os
import pathlib
from tempfile import TemporaryDirectory
from typing import Any, Dict

from jsonschema import ValidationError
from pep440 import is_canonical

from nbformat import __version__ as nbf_version
from nbformat import current_nbformat, read, write, writes
from nbformat.reader import get_version
from nbformat.validator import isvalid

from .base import TestsBase

def test_canonical_version():
    assert is_canonical(nbf_version)

class TestAPI(TestsBase):
    def test_read(self):
        """Can older notebooks be opened and automatically converted to the current
        nbformat?"""

        # Open a version 2 notebook.
        with self.fopen("test2.ipynb", "r") as f:
            nb = read(f, as_version=current_nbformat)

        # Check that the notebook was upgraded to the latest version automatically.
        (major, minor) = get_version(nb)
        self.assertEqual(major, current_nbformat)

    def test_write_downgrade_2(self):
        """downgrade a v3 notebook to v2"""
        # Open a version 3 notebook.
        with self.fopen("test3.ipynb", "r") as f:
            nb = read(f, as_version=3)

        jsons = writes(nb, version=2)
        nb2 = json.loads(jsons)
        (major, minor) = get_version(nb2)
        self.assertEqual(major, 2)

    def test_read_write_path(self):
        """read() and write() take filesystem paths"""
        path = os.path.join(self._get_files_path(), "test4.ipynb")
        nb = read(path, as_version=4)

        with TemporaryDirectory() as td:
            dest = os.path.join(td, "echidna.ipynb")
            write(nb, dest)
            assert os.path.isfile(dest)

    def test_read_write_pathlib_object(self):
        """read() and write() take path-like objects such as pathlib objects"""
        path = pathlib.Path(self._get_files_path()) / "test4.ipynb"
        nb = read(path, as_version=4)

        with TemporaryDirectory() as td:
            dest = pathlib.Path(td) / "echidna.ipynb"
            write(nb, dest)
            assert os.path.isfile(dest)

    def test_capture_validation_error(self):
        """Test that validation error can be captured on read() and write()"""
        validation_error: Dict[str, Any] = {}
        path = os.path.join(self._get_files_path(), "invalid.ipynb")
        nb = read(path, as_version=4, capture_validation_error=validation_error)
        assert not isvalid(nb)
        assert "ValidationError" in validation_error
        assert isinstance(validation_error["ValidationError"], ValidationError)

        validation_error = {}
        with TemporaryDirectory() as td:
            dest = os.path.join(td, "invalid.ipynb")
            write(nb, dest, capture_validation_error=validation_error)
            assert os.path.isfile(dest)
            assert "ValidationError" in validation_error
            assert isinstance(validation_error["ValidationError"], ValidationError)

        # Repeat with a valid notebook file
        validation_error = {}
        path = os.path.join(self._get_files_path(), "test4.ipynb")
        nb = read(path, as_version=4, capture_validation_error=validation_error)
        assert isvalid(nb)
        assert "ValidationError" not in validation_error

        validation_error = {}
        with TemporaryDirectory() as td:
            dest = os.path.join(td, "test4.ipynb")
            write(nb, dest, capture_validation_error=validation_error)
            assert os.path.isfile(dest)
            assert "ValidationError" not in validation_error


# Definition for singly-linked list.
# class ListNode:
#     def __init__(self, x):
#         self.val = x
#         self.next = None

class Solution:
    def detectCycle(self, head):
        try:
            fast = head.next.next
            slow = head.next
            while fast != slow:
                fast = fast.next.next
                slow = slow.next
            else:
                slow = head # Restart
                while fast != slow:
                    fast = fast.next
                    slow = slow.next
                else:
                    return slow
        except:
            return None


from django.shortcuts import render,redirect
from .forms import *
from .models import *
from django.contrib.auth.decorators import login_required

def gestionarPersonas(request):
    if request.method == 'POST':
        persona_form = PersonaForm(request.POST)
        if persona_form.is_valid():
            persona_form.changeReason="Creacion"
            persona_form.save()
        else:
            print('error')
    persona_form = PersonaForm()
    personas = Persona.objects.all()
    context = {'persona_form':persona_form,'personas':personas}
    return render(request,'gestionarPersonas.html',context)

def editarPersona(request,id):
    persona=Persona.objects.get(id=id)
    if request.method == 'GET':
        persona_form = PersonaForm(instance = persona)
        context = {'persona_form':persona_form}
        return render(request,'editarPersona.html',context)
    else:
        persona_form=PersonaForm(request.POST,instance=persona)
        if persona_form.is_valid():
            persona=persona_form.save(commit=False)
            persona.changeReason="Modificacion"
            persona.save()
    return redirect('/personas/gestionarPersonas')

def eliminarPersona(request,id):
    persona=Persona.objects.get(id=id)
    persona.changeReason="Eliminacion"
    persona.delete()
    return redirect('/personas/gestionarPersonas')

def gestionarSexos(request):
    if request.method == 'POST':
        sexo_form = SexoForm(request.POST)
        if sexo_form.is_valid():
            sexo_form.changeReason="Creacion"
            sexo_form.save()
        else:
            print('error')
    sexo_form = SexoForm()
    sexos = Sexo.objects.all()
    context = {'sexo_form':sexo_form,'sexos':sexos}
    return render(request,'gestionarSexos.html',context)

def editarSexo(request,id):
    sexo=Sexo.objects.get(id=id)
    if request.method == 'GET':
        sexo_form = SexoForm(instance = sexo)
        context = {'sexo_form':sexo_form}
        return render(request,'editarSexo.html',context)
    else:
        sexo_form=SexoForm(request.POST,instance=sexo)
        if sexo_form.is_valid():
            sexo=sexo_form.save(commit=False)
            sexo.changeReason="Modificacion"
            sexo.save()
    return redirect('/personas/gestionarSexos')

def eliminarSexo(request,id):
    sexo=Sexo.objects.get(id=id)
    sexo.changeReason="Eliminacion"
    sexo.delete()
    return redirect('/personas/gestionarSexos')

def auditoriaPersonas(request):
    if permiso(request, 42):
        auditoria_persona = Contrato.history.all()
        context = {'auditoria_persona': auditoria_persona}
        return render(request, 'sistema/auditoriaPersonas.html', context)
    else:
        return redirect('home')


# Jan Faryad
# 3. 7. 2017
#
# extracting information about coreference from onf file

class Onto_coreference_getter:
    def __init__( self, onto_input):
        """ onto input ... onf file """
        self.onto_input = onto_input

    def process_file( self):
        """
        main method, calle from outside
        reads the "Coreference chains" parts in the onf file, where
        the coreference clusters for each section are enumerated
        """
        list_of_clusters = []

        active_chains = False # if we are in the the "Coreference chains" block which comes after each section (text, set of sentences)
        active_cluster = False # if we are in one particular cluster
        for line in self.onto_input:
            if ( "Coreference chains" in line ): # beginning of the enumeration of clusters
                active_chains = True
            elif ( "Plain sentence" in line ): # new sentence -> end of the enumeration
                active_chains = False
            elif ( active_chains and "(IDENT)" in line ): # new cluster
                active_cluster = True
                cluster = Onto_cluster()
                list_of_clusters.append( cluster)
            elif ( active_chains and line == '\n' ): # end of the cluster
                active_cluster = False

            elif ( active_cluster ): # coreferent in the cluster
                processed = self.process_coref_line( line)
                if ( processed != None ):
                    ( position_string, expression ) = processed # coreferenting expression (not only head, but the whole "subtree"
                                                                # position string: a.b-c, a = sentence number, b/c = first/last word number
                    coreferent = Onto_coreferent( position_string, expression)
                    cluster.add_coref( coreferent)

        return list_of_clusters

    def process_coref_line( self, line): # -> ( string, string )
        """ getting of the coreferenting expression and its position """
        line_list = line.split( ' ')
        if ( len( line_list) < 17 ):
            return None
        position = line_list[15] # position string begins at this position and doesn't have spaces
        if ( position == "" ):
            return None
        if ( len( line) < 26 ):
            return None
        expression = line[26:-1] # expression begins at this position and does have spaces
        return ( position, expression )

class Onto_coreferent:
    def __init__( self, position_string, form):
        self.position_string = position_string # string
        self.form = form # string, the coreferenting word
class Onto_cluster:
    def __init__( self):
        self.coreferents = [] # Onto_coreferents, but later (after conversion) Nodes
    def add_coref( self, coref):
        self.coreferents.append( coref)


import numpy as np
import pytest
from numpy import testing as np_testing

import pymanopt

from . import _backend_tests

class TestNumPyBackend:
    @pytest.fixture(autouse=True)
    def setup(self):
        self.n = 10

        @pymanopt.function.numpy(
            _backend_tests.manifold_factory(point_layout=3)
        )
        def nary_cost(x, y):
            return np.sum(x * y)

        self.cost = self.nary_cost = nary_cost

        @pymanopt.function.numpy(
            _backend_tests.manifold_factory(point_layout=3)
        )
        def nested_nary_cost(x, y, z):
            return np.sum(x**2 * y + 3 * z)

        self.nested_nary_cost = nested_nary_cost

    def test_nary_cost_function(self):
        n = self.n
        x, y = [np.random.normal(size=n) for _ in range(2)]
        np_testing.assert_allclose(np.sum(x * y), self.nary_cost(x, y))

    def test_nested_nary_cost_function(self):
        n = self.n
        x, y, z = [np.random.normal(size=n) for _ in range(3)]
        np_testing.assert_allclose(
            np.sum(x**2 * y + 3 * z), self.nested_nary_cost(x, y, z)
        )

    def test_gradient_hessian_exceptions(self):
        with pytest.raises(NotImplementedError):
            self.cost.get_gradient_operator()
        with pytest.raises(NotImplementedError):
            self.cost.get_hessian_operator()


#!/usr/bin/python

import config_default

class Dict(dict):

	def __init__(self,name=(),value=(),**kw):

		super(Dict,self).__init__(**kw)

		for k,v in zip(name,value):

			self[k]=v

	def __getattr__(self,key):

		try:

			return self[key]

		except KeyError:

			raise AttributeError(r"'Dict' object has no attribute '%s'" % key)

	def __setattr__(self,key,value):

		self[key]=value

def merge(defaults,override):

	r={}

	for k,v in defaults.items():

		if k in override:

			if isinstance(v,dict):

				r[k]=merge(v,override[k])

			else:

				r[k]=override[k]

		else:

			r[k]=v

	return r

def toDict(d):

	D=Dict()

	for x,y in d.items():

		D[x]=toDict(y) if isinstance(y,dict) else y

	return D

defaults=config_default.configs

try:

	import config_override

	configs=merge(defaults,config_override.configs)

except ImportError:

	pass

#we can use it as configs.db.host or we just can do like configs['db']['host']
configs=toDict(configs)

# print(configs.db.host)
# print(configs['db']['host'])

return configs


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import math
import re
import random
import collections
from scipy import stats

import jieba
import jieba.posseg as pseg
import wordcloud
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import TruncatedSVD
from sklearn.cluster import KMeans

# ==============导入数据
data = pd.read_csv('tech_bv_info.csv', encoding='utf-8')
print('============data==========')
print(data)
print(data.info())

# ==============数据预处理
# 硬币、收藏、分享
def str_num_trans(x):
    if x=='投币':
        return 0
    if x=='收藏':
        return 0
    if re.search('万',x) is not None:
        return int(float(x.replace('万',''))*10000)
    return int(x)
data['coin'] = data['coin'].map(str_num_trans)
data['collect'] = data['collect'].map(str_num_trans)
data['share'] = data['share'].map(str_num_trans)

# 简介、评论去站内链接、换行
def text_process(x):
    # 去链接
    idx = re.search('<a class((?:.|\n)*)',x)
    if idx is not None:
        idx = idx.span()
        x_a_sub = x[idx[0]:idx[1]]
        a_sub = re.sub('','',x_a_sub.replace('',''))
        x = x.replace(x_a_sub,a_sub)
    # 去
    x = re.sub('<(/*)span>','',x)
    return x
def desc_process(x):
    if pd.isna(x):
        return x
    return text_process(x)
def comment_process(x):
    if pd.isna(x):
        return x
    comment_list = x.split('|$comment-sep$|')
    new_list = []
    for comment in comment_list:
        new_list.append(text_process(comment))
    return '|$comment-sep$|'.join(new_list)
data['describe'] = data['describe'].map(desc_process)
data['comment'] = data['comment'].map(comment_process)

# tag处理，去空格
def tag_process(x):
    tag_list = x.split('|$tag-sep$|')
    new_list = []
    for tag in tag_list:
        new_list.append(tag.replace(' ','').replace('\n',''))
    new_list = list(set(new_list))
    return '|$tag-sep$|'.join(new_list)
data['tag'] = data['tag'].map(tag_process)

# =================数据描述性分析
def count_table(df):
    table = pd.DataFrame()
    for col in df.columns.tolist():
        if df[col].dtype == object:
            continue
        table.loc[col,'max'] = df[col].max()
        table.loc[col,'min'] = df[col].min()
        table.loc[col,'mean'] = round(df[col].mean(),2)
        table.loc[col,'median'] = df[col].median()
    return table
print('============各字段统计量表格==============')
print(count_table(data))

# log变换后，集中在1w-3w
plt.hist(data['play'].map(math.log).to_numpy(),color='#D2B48C')
plt.xlabel('播放量/log')
plt.ylabel('数量')
plt.title('视频播放量分布')
plt.rcParams['font.sans-serif'] = ['SimHei']
plt.show()

# 因为是数码科技专区，男性多可以理解
plt.bar(['男','保密','女'],data['author_sex'].value_counts().tolist(), color=['#BDB76B','#F0E68C','#EEE8AA'])
plt.xlabel('性别')
plt.ylabel('数量')
plt.title('up主性别柱状图')
plt.show()

time_videoNum = data['time'].map(lambda x:x[0:10]).value_counts().sort_index(ascending=True)
plt.bar(x=time_videoNum.index.tolist(), height=time_videoNum.tolist(), color='#FA8072')
plt.xlabel('发布时间')
plt.xticks(time_videoNum.index.tolist()[::15],rotation=70)
plt.ylabel('视频数')
plt.title('视频发布时间的分布')
plt.show()

def if_topic(x):
    if '|$tag-topic$|' in x:
        return 1
    return 0
data['event_join'] = data['tag'].map(if_topic)
plt.boxplot([data.loc[data['event_join']==1,'play'].map(math.log).tolist(),
             data.loc[data['event_join']==0,'play'].map(math.log).tolist()],
            labels=['参加','不参加'],
            flierprops={'markerfacecolor':'#CD5C5C'},
            whiskerprops={'color':'#800000'},
            patch_artist=True,
            boxprops={'facecolor':'#DEB887'},
            medianprops={'color':'#800000'})
plt.xlabel('是否参加创作激励活动')
plt.ylabel('播放量/log')
plt.title('参与活动话题与视频播放量')
plt.show()

print('未参加：{}({:.2f})，参加：{}({:.2f})'.format(data[data['event_join']==0].shape[0],data[data['event_join']==0].shape[0]/data.shape[0],
                                    data[data['event_join']==1].shape[0],data[data['event_join']==1].shape[0]/data.shape[0]))
# t检验
print(
    stats.ttest_ind(data.loc[data['event_join']==0,'play'].map(math.log).to_numpy(), data.loc[data['event_join']==1,'play'].map(math.log).to_numpy())
)

plt.boxplot([data.loc[data['duration']<=60,'play'].map(math.log).tolist(),
             data.loc[(data['duration']>60)&(data['duration']<=300),'play'].map(math.log).tolist(),
             data.loc[(data['duration']>300)&(data['duration']<=600),'play'].map(math.log).tolist(),
             data.loc[(data['duration']>600)&(data['duration']<=900),'play'].map(math.log).tolist(),
             data.loc[(data['duration']>900)&(data['duration']<=1800),'play'].map(math.log).tolist(),
             data.loc[(data['duration']>1800)&(data['duration']<=3600),'play'].map(math.log).tolist(),
             data.loc[data['duration']>3600,'play'].map(math.log).tolist()],
            labels=['1分钟以内','1-5分钟','5-10分钟','10-15分钟','15-30分钟','30-60分钟','1小时以上'],
            flierprops={'markerfacecolor':'#CD5C5C'},
            whiskerprops={'color':'#800000'},
            patch_artist=True,
            boxprops={'facecolor':'#DEB887'},
            medianprops={'color':'#800000'})
plt.xlabel('视频时长')
plt.ylabel('播放量/log')
plt.title('视频时长与视频播放量')
plt.show()

plt.scatter(data['author_fans'].map(lambda x:math.log(x) if x!=0 else 0).tolist(), data['play'].map(math.log).tolist(),
            s=4,c='#BDB76B')
plt.xlabel('粉丝数/log')
plt.ylabel('播放量/log')
plt.title('粉丝量与视频播放量')
plt.show()

with open('my_stop_words.txt', 'r', encoding='utf-8') as f:
    stop_words = f.read().split('\n')
title_list = data['title'].tolist()
title_words = []
for i in title_list:
    for word in jieba.cut(i):
        if word not in stop_words and len(re.findall('[0-9]',word))!=len(word):
            title_words.append(word)
title_word_freq = sorted(collections.Counter(title_words).items(), key=lambda x:x[1], reverse=True)

title_words_wc = ' '.join(title_words)

wc=wordcloud.WordCloud(font_path="C:\Windows\Fonts\simfang.ttf",width=1000, height=800,
                             max_words=100,
                             background_color='white',colormap='summer')
wc.generate_from_text(title_words_wc)

plt.imshow(wc)

plt.bar(['小米','华为','苹果','iPhone','三星','红米','荣耀'],[269,205,191,147,123,95,70],
        color=['#006400','#008000','#008000','#228B22','#32CD32','#00FF00','#00FF7F'])
plt.xlabel('手机品牌')
plt.ylabel('频数')
plt.title('不同手机品牌出现频数')
plt.show()

phone = []
pc = []
gpu = []
others = []
for i in range(data.shape[0]):
    if '手机' in data.loc[i,'title']:
        phone.append(data.loc[i,'play'])
    elif '显卡' in data.loc[i,'title']:
        gpu.append(data.loc[i,'play'])
    elif '笔记本' in data.loc[i,'title']:
        pc.append(data.loc[i,'play'])
    else:
        others.append(data.loc[i,'play'])
plt.bar(height=[np.array(phone).mean(), np.array(pc).mean(), np.array(gpu).mean(), np.array(others).mean()],
            x=['手机','笔记本','显卡','其他'],color=['#4169E1','#6495ED','#B0C4DE','#708090'])
plt.xlabel('视频类型')
plt.ylabel('平均播放量')
plt.show()
# 标题带有手机、笔记本、显卡的视频平均播放量明显较高

# ========================文本分析
hot_topic = []
hot_bv = []
for i in range(data.shape[0]):
    if data.loc[i,'play']>1000000:
        hot_bv.append(data.loc[i,'bv'])
        hot_video = []
        for word in jieba.cut(data.loc[i,'title']):
            if word not in stop_words:
                hot_video.append(word)
        for word in data.loc[i,'tag'].split('|$tag-sep$|'):
            if word not in hot_video:
                hot_video.append(word)
        hot_topic+=hot_video
hot_topic = sorted(collections.Counter(hot_topic).items(), key=lambda x:x[1], reverse=True)
print(hot_topic)

hot_wc = []
for i in range(11):
    topic=['科技猎手','苹果','华为','安卓','创意','DIY','自制','AMD','性能','发布会','三星']
    topic_num=[83,28,15,15,7,7,7,6,6,6,5]
    for k in range(topic_num[i]):
        hot_wc.append(topic[i])
random.shuffle(hot_wc)
hot_wc = ' '.join(hot_wc)
wc=wordcloud.WordCloud(font_path="C:\Windows\Fonts\simfang.ttf",width=500, height=400,
                             max_words=100,
                             background_color='white',colormap='spring')
wc.generate_from_text(hot_wc)

plt.imshow(wc)

def comment_process2(x):
    if pd.isna(x):
        return x
    com_li = x.split('|$comment-sep$|')
    new_li = []
    for com in com_li:
        new_li.append(com.replace('\n','。').replace('|$comment-top$|',''))
    return '。'.join(new_li)
data['comment_processed'] = data['comment'].map(comment_process2)

com_wc = []
for com in data['comment_processed']:
    if not pd.isna(com):
        for word,flag in pseg.cut(com):
            if word not in stop_words and word not in ['…','_',':'] and word not in ['doge','辣','眼睛','藏狐'] and 'n' in flag:
                com_wc.append(word)
comment_word_freq = sorted(collections.Counter(com_wc).items(), key=lambda x:x[1], reverse=True)

comment_wc = ' '.join(com_wc)

wc=wordcloud.WordCloud(font_path="C:\Windows\Fonts\simfang.ttf",width=1000, height=800,
                             max_words=100,
                             background_color='white',colormap='copper')
wc.generate_from_text(comment_wc)

plt.imshow(wc)

com_wc = []
for com in data['comment_processed']:
    if not pd.isna(com):
        for word in jieba.cut(com):
            if word not in stop_words and word not in ['…','_',':']:
                com_wc.append(word)
comment_word_freq = sorted(collections.Counter(com_wc).items(), key=lambda x:x[1], reverse=True)
wf1 = []
wf2 = []
for i in title_word_freq[:50]:
    wf1.append(i[0])
for i in comment_word_freq[:100]:
    wf2.append(i[0])
wf1s = set(wf1)
wf2s = set(wf2)
wf = wf2s-wf1s
com_unique_freq = []
for i in wf:
    if len(re.findall('[0-9]',i))!=len(i) and len(re.findall('[a-zA-Z]',i))!=len(i) and i not in '说好很没会真的人想笑哭做去更下感觉再里后才很多辣眼睛太中脱单藏狐出东西妙一点不错确实真希望建议只能':
        com_unique_freq.append(comment_word_freq[wf2.index(i)])
com_unique_freq = sorted(com_unique_freq, key=lambda x:x[1], reverse=True)

plt.figure(figsize=(10,6))
plt.bar([i[0] for i in com_unique_freq], [i[1] for i in com_unique_freq],
        color=['#BDB76B','#EEE8AA','#F0E68C','#FFFACD','#FFF8DC'])
plt.xlabel('热评关键词')
plt.ylabel('词频')
plt.title('热评')
plt.show()

# ===================聚类推荐系统
def describe_process2(x):
    if pd.isna(x):
        return x
    return x.replace('\n','。')
data['describe_processed'] = data['describe'].map(describe_process2)
def tag_process2(x):
    if pd.isna(x):
        return x
    return x.replace('|$tag-sep$|','。').replace('|$tag-topic$|','')
data['tag_processed'] = data['tag'].map(tag_process2)

document = []
for i in range(data.shape[0]):
    doc = []
    for word in jieba.cut(data.loc[i,'title']):
        if word not in stop_words and word not in ['…','_',':']:
            doc.append(word)
    if not pd.isna(data.loc[i,'tag_processed']):
        for word in jieba.cut(data.loc[i,'tag_processed']):
            if word not in stop_words and word not in ['…','_',':']:
                doc.append(word)
    if not pd.isna(data.loc[i,'describe_processed']):
        for word in jieba.cut(data.loc[i,'describe_processed']):
            if word not in stop_words and word not in ['…','_',':']:
                doc.append(word)
    if not pd.isna(data.loc[i,'comment_processed']):
        for word in jieba.cut(data.loc[i,'comment_processed']):
            if word not in stop_words and word not in ['…','_',':']:
                doc.append(word)
    document.append(' '.join(doc))

tfidf = TfidfVectorizer()
video_mat = tfidf.fit_transform(document)
svd = TruncatedSVD(n_components=240)
video_mat_svd = svd.fit_transform(video_mat)
km = KMeans(n_clusters=100)
video_clu = km.fit_predict(video_mat_svd)
data['clu']=video_clu
print('==================聚类为第一类的视频====================')
print(data.loc[data['clu']==1,['bv','title','play','like','duration']])


class Solution(object):
    def subarraySum(self, nums, k):
        """
        :type nums: List[int]
        :type k: int
        :rtype: int
        """
        dt = collections.defaultdict(int)
        dt[0] = 1
        ss, res = 0, 0
        for n in nums:
            ss += n
            res += dt[ss - k]
            dt[ss] += 1
        return res


import argparse
import pandas as pd
import os
import subprocess

parser = argparse.ArgumentParser()
parser.add_argument('-csv_path', '--csv_path', default='download_list.csv', help='path of download_list.csv')
parser.add_argument('-save_path', '--save_path', default='video/data/', help='dir for saving videos')
parser.add_argument('-only_video', '--only_video', default = 0, help='if you want to download only video, make this option 1')
parser.add_argument('-only_audio', '--only_audio', default = 0, help='if you want to download only audio, make this option 1')
parser.add_argument('-sound_format', '--sound_format', default = 'm4a', help='webm is 48000 Hz, m4a is 44100 Hz')

args = parser.parse_args()
CSV_PATH = args.csv_path
SAVE_PATH = args.save_path
ONLY_VIDEO = args.only_video
ONLY_AUDIO = args.only_audio
AUDIO_FORMAT = args.sound_format

if not os.path.exists(SAVE_PATH):
    os.mkdir(SAVE_PATH)

df = pd.read_csv(CSV_PATH)

for idx, row in df.iterrows():

    file_number, youtube_link, korean_name, roman_name, resolution, fps, audio_sr, video_opt, webm_opt, m4a_opt = row

    ## refresh the cache for safe download
    if (file_number % 3 == 0):
        command_refresh = 'youtube-dl --rm-cache-dir'
        res_refresh = subprocess.call(command_refresh, shell=True)

    audio_opt = None
    if AUDIO_FORMAT == 'm4a':
        audio_opt = m4a_opt
    else:
        audio_opt = webm_opt

    print(file_number, roman_name, youtube_link)

    ## download video + audio
    command = 'youtube-dl -o {save_path} -f {video_op}+{audio_op} ' \
              'https://www.youtube.com/watch?v={link}'.format(save_path=os.path.join(SAVE_PATH, youtube_link + '.mp4'),
                                                              video_op=str(video_opt),
                                                              audio_op=str(audio_opt),
                                                              link=youtube_link)

    ## download only video
    if ONLY_VIDEO:
        command = 'youtube-dl -o {save_path} -f {video_op} ' \
              'https://www.youtube.com/watch?v={link}'.format(save_path=os.path.join(SAVE_PATH, youtube_link + '.mp4'),
                                                              video_op=str(video_opt), link=youtube_link)
    ## download only audio
    if ONLY_AUDIO:
        command = 'youtube-dl -o {save_path} -f {audio_op} ' \
              'https://www.youtube.com/watch?v={link}'.format(save_path=os.path.join(SAVE_PATH, youtube_link + '.m4a'),
                                                              audio_op=str(audio_opt), link=youtube_link)

    res = subprocess.call(command, shell=True)


import os
import PIL
import math
import numpy as np
import pandas as pd
import cv2 as cv
from PIL import ImageFont
from PIL import Image
from PIL import ImageDraw

if __name__ == '__main__':
    # 图片文件夹目录
    path = "image"
    files = os.listdir(path)
    for img_file in files:
        print(img_file)
        # 读入图片
        img = Image.open(path + '/' + img_file)
        # 指定画笔
        draw = ImageDraw.Draw(img)
        xx = np.array(img).shape[1]
        yy = np.array(img).shape[0]
        # 加载字体
        font = ImageFont.truetype('STSONG.ttf', int(xx / 36))
        (x1, y1) = (int(xx / 36), int(yy / 27))
        (x2, y2) = (int(xx / 36), 2 * int(yy / 27))
        (x3, y3) = (int(xx / 36), 3 * int(yy / 27))
        (x4, y4) = (int(xx / 36), 4 * int(yy / 27))
        rgb = (255, 255, 255)
        df = pd.read_csv('dead_pig_predict.csv')
        df = df.loc[:, ~df.columns.str.contains('^Unnamed')]
        for i in range(len(df)):
            if df['file_name_top'].iloc[i] == img_file or df['file_name_side'].iloc[i] == img_file:
                img_index = i
                break
        point_lst = ['nose_coordinate_', 'ear_coordinate_', 'hip_coordinate_', 'tail_coordinate_',
                     'left_top_coordinate_', 'left_bottom_coordinate_', 'right_top_coordinate_', 'right_bottom_coordinate_']
        point_cnt = 8
        info = img_file.split('_')
        for k in point_lst:
            col = k + info[1]
            if pd.isna(df[col].iloc[img_index]):
                point_cnt -= 1
                continue
            center = (df[col].iloc[img_index])[1:-1].split(', ')
            for i in range(len(center)):
                center[i] = int(center[i])
            r = int(xx / 180)
            for i in range(center[0] - r, center[0] + r + 1):
                for j in range(center[1] - r, center[1] + r + 1):
                    if (pow(i - center[0], 2) + pow(j - center[1], 2)) ** 0.5 <= r:
                        draw.point((i, j), fill=rgb)
        is_output = 1
        if pd.isna(df['predict_length'].iloc[img_index]) or pd.isna(df['predict_weight'].iloc[img_index]):
            is_output = 0
        if is_output:
            text1 = "识别结果"
            draw.text((x1, y1), text1, fill=rgb, font=font)
            text2 = "状态点 : %d" % point_cnt
            draw.text((x2, y2), text2, fill=rgb, font=font)
            length = float(df['predict_length'].iloc[img_index])
            text3 = "体长 : %.2fCM" % length
            draw.text((x3, y3), text3, fill=rgb, font=font)
            weight = float(df['predict_weight'].iloc[img_index])
            text4 = "体重 : %.2fKG" % weight
            draw.text((x4, y4), text4, fill=rgb, font=font)

            img.save('image_output/' + img_file)
        del draw  # 删除画笔
        img.close()  # 关闭图片


# Дана строка. Найдите в этой строке второе
# вхождение буквы f, и выведите индекс этого вхождения.
# Если буква f в данной строке встречается только
# один раз, выведите число -1, а если не встречается
# ни разу, выведите число -2.При решении
# этой задачи нельзя использовать метод count.

s = input()
f1 = s.find('f')
f2 = s.find('f', f1 + 1)
if f1 == -1:
    print(-2)
elif f2 == -1:
    print(-1)
else:
    print(f2)

46_Второе вхождение.py


"""
overriden methods of general classes, specific for English
"""
from verb_record import Verb_record
from frame_extractor import Frame_extractor

class En_verb_record( Verb_record):
    def __init__( self, lemma):
        super().__init__( lemma)
        #self.frame_type_class = En_frame_type

class En_frame_extractor( Frame_extractor):
    def __init__( self, pickle_output=None):
        super().__init__( pickle_output)
        self.verb_record_class = En_verb_record
        #self.en_dict_of_verbs = {}
        self.modal_lemmas = [ "can", "could", "may", "might", "shall", "should",
                                 "will", "would", "must" ]

    def _process_frame( self, verb_record, verb_node):  # -> Frame_inst
        """ called from process_node """
        frame_type, frame_inst = self._process_frame_phase_1( verb_node)

        # newly inserted !!
        self._check_modal_verbs( frame_inst, frame_type, verb_node)

        # adding the frame to the verb_record
        verb_record.consider_new_frame_type( frame_type) #, frame_inst)

        return frame_inst

    def _process_frame_phase_1( self, verb_node):
            # -> pair ( Frame_type, Frame_inst )
        # new frame
        frame_type = self.frame_type_class()
        frame_type.set_verb_features( verb_node)
        frame_inst = self.frame_inst_class()

        # creating and adding args to the frame type/inst
        # creating tokens for example sentence and adding them to frame_inst
        self._process_sentence( frame_type, frame_inst, verb_node)

        frame_type.sort_args()  # important for later frame comparision
        frame_type.add_inst( frame_inst)  # connection between frame type and inst

        return frame_type, frame_inst

    def _check_modal_verbs( self, frame_inst, frame_type, verb_node):
        arg_deprels = [ arg.deprel for arg in frame_type.args ]
        if not "nsubj" in arg_deprels:
            for child_node in verb_node.children:
                if child_node.upos == "AUX":
                    lemma_is_modal = child_node.lemma in self.modal_lemmas
                    #lemma_is_have_to = False
                    #if child_node.lemma == "have":
                    #    for other_child_node in verb_node.children:
                    #        if other_child_node.lemma == "to":
                    #            lemma_is_have_to = True
                    if lemma_is_modal:# or lemma_is_have_to:
                        frame_inst.has_modal = True


import cocotb
from cocotb.triggers import Timer, RisingEdge, FallingEdge, ClockCycles
from cocotb.result import TestFailure
from cocotb.clock import Clock
from cocotb.binary import BinaryValue
from cocotb.scoreboard import Scoreboard
from cocotb.monitors import Monitor
from cocotb.regression import TestFactory
from cocotb.drivers.amba import AXI4LiteMaster
from cocotb.drivers.amba import AXIProtocolError
from cocotb.wavedrom import trace, Wavedrom
from cocotb import SimLog, coroutine
from cocotb.drivers import BitDriver

import random
import numpy as np
import math
import sys, os
import matplotlib.pyplot as plt

# Constants
c_BASEADDRESS = 0x00000000
c_VERSION_OFFSET = 0x00000000
c_CONFIG_ID_OFFSET = 0x00000004
c_COUNT_OFFSET = 0x00000008
c_VERSION_VALUE = 0x00000001
c_CONFIG_ID_VALUE = 0x00000001
c_COUNT_VALUE = 0x02000000
c_CLK_PERIOD = 10 #ns
# ==============================================================================
class BitMonitor(Monitor):
    """Observe a single-bit input or output of the DUT."""
    def __init__(self, name, signal, clk, callback=None, event=None):
        self.name = name
        self.signal = signal
        self.clk = clk
        Monitor.__init__(self, callback, event)

    @coroutine
    def _monitor_recv(self):
        clkedge = RisingEdge(self.clk)

        while True:
            # Capture signal at rising edge of clock
            yield clkedge
            vec = self.signal.value
            self._recv(vec)
# ==============================================================================
class outMonitor(Monitor):
    def __init__(self, name, dut, callback=None, event=None):
        self.name = name
        self.dut = dut
        Monitor.__init__(self, callback, event)

    @cocotb.coroutine
    def _monitor_recv(self):
        clkedge = RisingEdge(self.dut.clk)
        while True:
            # Capture signal at rising edge of clock
            yield clkedge
            vec = self.dut.Zybo_Example_leds_rgb_out
            self._recv(vec)
# ==============================================================================
def input_gen():
    """Generator for input data applied by BitDriver.
    Continually yield a tuple with the number of cycles to be on
    followed by the number of cycles to be off.
    """
    while True:
        yield random.randint(1, 5), random.randint(1, 5)
# ==============================================================================
class TB(object):
    def __init__(self, dut):
        # Some internal state
        self.dut = dut
        self.stopped = False

        # Use the input monitor to reconstruct the transactions from the pins
        # and send them to our 'model' of the design.
        self.input_mon = BitMonitor(name="input", signal=dut.Zybo_Example_sw_in, clk=dut.clk,
                                    callback=self.model)

        # Create input driver and output monitor
        self.input_drv = BitDriver(signal=dut.Zybo_Example_sw_in, clk=dut.clk, generator=input_gen())
        self.output_mon = BitMonitor(name="output", signal=dut.Zybo_Example_leds_out, clk=dut.clk)

        # Create a scoreboard on the outputs
        self.expected_output = []
        self.scoreboard = Scoreboard(dut)
        self.scoreboard.add_interface(self.output_mon, self.expected_output)

    def model(self, transaction):
        if not self.stopped:
            self.expected_output.append(transaction)

    def start(self):
        self.input_drv.start()

    def stop(self):
        self.input_drv.stop()
        self.stopped = True
# ==============================================================================
def check(dut, act, exp):
    # Check
    if act != exp:
        raise TestFailure("Register at address 0x%08X should have been: \
                               0x%08X but was 0x%08X" % (c_BASEADDRESS+c_COUNT_OFFSET, c_COUNT_VALUE, int(act)))

    dut._log.info("Read: 0x%08X From Address: 0x%08X" % (int(act), c_BASEADDRESS+c_COUNT_OFFSET))
# ==============================================================================
@cocotb.test(skip = False, stage = 1)
def Zybo_Example_test(dut):
    # Setting up clocks
    clk_100MHz = Clock(dut.clk, c_CLK_PERIOD, units='ns')
    cocotb.fork(clk_100MHz.start(start_high=False))
    axi_aclk_100MHz = Clock(dut.axi_aclk, c_CLK_PERIOD, units='ns')
    cocotb.fork(axi_aclk_100MHz.start(start_high=False))

    # Setting init values
    dut.reset = 1
    dut.Zybo_Example_sw_in = 0
    dut.Zybo_Example_bt_in = 0
    dut.axi_aresetn = 0
    # AXI-Lite Master object
    axil_m = AXI4LiteMaster(dut, "s_axi", dut.axi_aclk)
    # tb
    tb=TB(dut)
    tb.start()
    # Wait one cycle and deactivate resets
    yield Timer(c_CLK_PERIOD, units='ns')
    dut.reset <= 0
    dut.axi_aresetn <= 1
    yield Timer(c_CLK_PERIOD, units='ns')

    # AXI-Lite read VERSION
    dut._log.info("AXI-Lite: Reading address 0x%02X" % (c_BASEADDRESS+c_VERSION_OFFSET))
    s_value_read = yield axil_m.read(c_BASEADDRESS+c_VERSION_OFFSET)
    # Check
    check(dut, s_value_read, c_VERSION_VALUE)

    # AXI-Lite read CONFIG_ID
    dut._log.info("AXI-Lite: Reading address 0x%02X" % (c_BASEADDRESS+c_CONFIG_ID_OFFSET))
    s_value_read = yield axil_m.read(c_BASEADDRESS+c_CONFIG_ID_OFFSET)
    # Check
    check(dut, s_value_read, c_CONFIG_ID_VALUE)

    # AXI-Lite read COUNT
    dut._log.info("AXI-Lite: Reading address 0x%02X" % (c_BASEADDRESS+c_COUNT_OFFSET))
    s_value_read = yield axil_m.read(c_BASEADDRESS+c_COUNT_OFFSET)
    # Check
    check(dut, s_value_read, c_COUNT_VALUE)

    # AXI-Lite write
    yield axil_m.write(c_BASEADDRESS+c_COUNT_OFFSET, 0x00000001)
    yield Timer(c_CLK_PERIOD, units='ns')

    # end tb
    tb.stop()

@cocotb.test(skip = False, stage = 2)
def Wavedrom_test(dut):
    # Setting up clocks
    clk_100MHz = Clock(dut.clk, c_CLK_PERIOD, units='ns')
    cocotb.fork(clk_100MHz.start(start_high=False))
    axi_aclk_100MHz = Clock(dut.axi_aclk, c_CLK_PERIOD, units='ns')
    cocotb.fork(axi_aclk_100MHz.start(start_high=False))

    # AXI-Lite Master object
    axil_m = AXI4LiteMaster(dut, "s_axi", dut.axi_aclk)

    # Setting init values
    dut.reset <= 1
    dut.Zybo_Example_sw_in <= 4
    dut.Zybo_Example_bt_in <= 0
    dut.axi_aresetn <= 0
    # Wait one cycle and deactivate resets
    yield Timer(c_CLK_PERIOD, units='ns')
    dut.reset <= 0
    dut.axi_aresetn <= 1
    yield Timer(c_CLK_PERIOD, units='ns')

    # AXI-Lite write
    yield axil_m.write(c_BASEADDRESS+c_COUNT_OFFSET, 0x00000001)
    yield Timer(c_CLK_PERIOD, units='ns')

    # Wavedrom
    args = [dut.Zybo_Example_sw_in, dut.Zybo_Example_leds_out, dut.Zybo_Example_leds_rgb_out]
    with trace(*args, clk=dut.clk) as waves:
        yield ClockCycles(dut.clk, 12)
        dut._log.info(waves.dumpj(header = {'text':'WaveDrom example', 'tick':0}))
        waves.write('wavedrom.json', header = {'tick':0}, config = {'hscale':3})