import torch
from torchsim.core import get_float, SMALL_CONSTANT
from torchsim.core.models.expert_params import NUMBER_OF_CONTEXT_TYPES, EXPLORATION_REWARD, REWARD_DISCOUNT_FACTOR
from torchsim.core.models.temporal_pooler.buffer import TPFlockBuffer
from torchsim.core.models.temporal_pooler.kernels import tp_process_kernels
from torchsim.core.models.temporal_pooler.process import TPProcess
from torchsim.core.models.temporal_pooler.tp_output_projection import TPOutputProjection
from torchsim.core.utils.tensor_utils import normalize_probs, normalize_probs_, multi_unsqueeze, move_probs_towards_50, \
    weighted_sum_, kl_divergence, id_to_one_hot, safe_id_to_one_hot

class TPFlockForwardAndBackward(TPProcess):
    """The forward pass of temporal pooler."""
    _float_dtype: torch.dtype
    _output_projection: TPOutputProjection

    _buffer: TPFlockBuffer

    def __init__(self,
                 indices: torch.Tensor,
                 do_subflocking: bool,
                 buffer: TPFlockBuffer,
                 cluster_data: torch.Tensor,
                 context_data: torch.Tensor,
                 reward_data: torch.Tensor,
                 frequent_seqs: torch.Tensor,
                 frequent_seq_occurrences: torch.Tensor,
                 frequent_seq_likelihoods_priors_clusters_context: torch.Tensor,
                 frequent_context_likelihoods: torch.Tensor,
                 frequent_rewards_punishments: torch.Tensor,
                 frequent_exploration_attempts: torch.Tensor,
                 frequent_exploration_results: torch.Tensor,
                 projection_outputs: torch.Tensor,
                 action_outputs: torch.Tensor,
                 action_rewards: torch.Tensor,
                 action_punishments: torch.Tensor,
                 passive_predicted_clusters_outputs: torch.Tensor,
                 execution_counter: torch.Tensor,
                 seq_likelihoods_by_context: torch.Tensor,  # Both input and output tensor
                 best_matching_context: torch.Tensor,
                 n_frequent_seqs: int,
                 n_cluster_centers: int,
                 seq_length: int,
                 seq_lookahead: int,
                 context_size: int,
                 n_providers: int,
                 exploration_probability: float,
                 own_rewards_weight: float,
                 cluster_exploration_prob: float,
                 device: str,
                 follow_goals: bool,
                 convolutional: bool = False,
                 produce_actions: bool = False,
                 compute_backward_pass: bool = True,
                 compute_best_matching_context: bool = True):
        super().__init__(indices, do_subflocking)
        self._float_dtype = get_float(device)
        self.device = device

        self.n_frequent_seqs = n_frequent_seqs
        self.n_cluster_centers = n_cluster_centers

        self.seq_length = seq_length
        self.seq_lookahead = seq_lookahead
        self.seq_lookbehind = self.seq_length - self.seq_lookahead

        self.context_size = context_size
        self.n_providers = n_providers

        self.exploration_probability = exploration_probability
        self.follow_goals = follow_goals
        self.own_rewards_weight = own_rewards_weight
        self.cluster_exploration_prob = cluster_exploration_prob

        # Does this flock produce actions?
        self.produce_actions = produce_actions

        self.compute_backward_pass = compute_backward_pass
        self.compute_best_matching_context = compute_best_matching_context

        self.cluster_history = torch.zeros(self._flock_size, self.seq_lookbehind, self.n_cluster_centers,
                                           dtype=self._float_dtype, device=device)

        self.context_history = torch.zeros(self._flock_size, self.seq_lookbehind, self.n_providers,
                                           self.context_size,
                                           dtype=self._float_dtype, device=device)

        # P(S | C) - sequence likelihoods based just on current cluster history
        self.seq_likelihoods_clusters = torch.zeros(self._flock_size, self.n_frequent_seqs,
                                                    dtype=self._float_dtype, device=device)

        # P(S | Prior, C) - sequence likelihoods based just on prior probabilities and current cluster history
        self.seq_likelihoods_priors_clusters = torch.zeros(self._flock_size, self.n_frequent_seqs,
                                                           dtype=self._float_dtype, device=device)

        # P(S | C, Oi) - sequence likelihoods based on current cluster history and each
        # provider independently
        self.seq_likelihoods_for_each_provider = torch.zeros(self._flock_size, self.n_frequent_seqs,
                                                             self.seq_lookbehind, self.n_providers,
                                                             dtype=self._float_dtype, device=device)

        self.predicted_clusters_by_context = torch.zeros(self._flock_size, self.n_providers, self.context_size,
                                                         self.n_cluster_centers,
                                                         dtype=self._float_dtype, device=device)

        # P(S | C, O*) - sequence likelihoods based on, current cluster history and the most informative context
        self.seq_probs_clusters_context = torch.zeros(self._flock_size, self.n_frequent_seqs,
                                                      dtype=self._float_dtype, device=device)

        # region Backward

        # P(S | Prior, C, O*, how much I want to explore it) - sequence likelihoods based on
        # _seq_likelihoods_priors_clusters_context and how much the expert want them to be explored (
        self.seq_likelihoods_exploration = torch.zeros(self._flock_size, self.n_frequent_seqs,
                                                       dtype=self._float_dtype, device=device)

        self.seq_likelihoods_active = torch.zeros((self._flock_size, self.n_frequent_seqs),
                                                  dtype=self._float_dtype, device=device)

        self.active_predicted_clusters = torch.zeros((self._flock_size, self.seq_length, self.n_cluster_centers),
                                                     dtype=self._float_dtype, device=device)
        # indicator which experts are exploring this turn (1) vs acting according to the goalDirected behavior (0)
        self.exploring = torch.zeros((self._flock_size, 1), dtype=self._float_dtype, device=device)
        # endregion

        self.exploration_random_numbers = torch.zeros((self._flock_size, 1), dtype=self._float_dtype, device=device)

        self.context_informativeness = torch.zeros(self._flock_size, self.seq_lookbehind, self.n_providers,
                                                   dtype=self._float_dtype, device=device)

        self.frequent_seqs_scaled = torch.zeros((self._flock_size, self.n_frequent_seqs,
                                                 self.n_cluster_centers), dtype=self._float_dtype, device=device)

        self.seq_rewards_goal_directed = torch.zeros(
            (self._flock_size, self.n_frequent_seqs, NUMBER_OF_CONTEXT_TYPES - 1),
            dtype=self._float_dtype, device=device)

        self._influence_model = torch.zeros(
            (self._flock_size, self.n_frequent_seqs, self.seq_length - 1, self.n_cluster_centers),
            dtype=self._float_dtype, device=device)

        self._buffer = self._get_buffer(buffer)
        self._cluster_data = self._read(cluster_data)
        self._context_data = self._read(context_data)
        self._reward_data = self._read(reward_data)
        if convolutional:
            self._frequent_seqs = self._read_expanded(frequent_seqs).contiguous()
            self._frequent_seq_occurrences = self._read_expanded(frequent_seq_occurrences)
            self._frequent_context_likelihoods = self._read_expanded(frequent_context_likelihoods)
            self._frequent_exploration_attempts = self._read_expanded(frequent_exploration_attempts)
            self._frequent_exploration_results = self._read_expanded(frequent_exploration_results)
            self._frequent_rewards_punishments = self._read_expanded(frequent_rewards_punishments)
        else:
            self._frequent_seqs = self._read(frequent_seqs)
            self._frequent_seq_occurrences = self._read(frequent_seq_occurrences)
            self._frequent_context_likelihoods = self._read(frequent_context_likelihoods)
            self._frequent_exploration_attempts = self._read(frequent_exploration_attempts)
            self._frequent_exploration_results = self._read(frequent_exploration_results)
            self._frequent_rewards_punishments = self._read(frequent_rewards_punishments)
        self._passive_predicted_clusters = self._read_write(passive_predicted_clusters_outputs)
        self._projection_outputs = self._read_write(projection_outputs)
        self._action_outputs = self._read_write(action_outputs)
        self._action_rewards = self._read_write(action_rewards)
        self._action_punishments = self._read_write(action_punishments)
        self._execution_counter = self._read_write(execution_counter)
        self._seq_likelihoods_by_context = self._read_write(seq_likelihoods_by_context)
        self._best_matching_context = self._read_write(best_matching_context)

        # P(S | Prior, C, O*) - sequence likelihoods based on prior probabilities, current cluster history and the most
        #  informative context
        self.seq_likelihoods_priors_clusters_context = self._read_write(
            frequent_seq_likelihoods_priors_clusters_context)

        self._check_dims(self._cluster_data, self._context_data, self._frequent_seqs, self._frequent_seq_occurrences,
                         self._frequent_context_likelihoods, self._passive_predicted_clusters, self._projection_outputs,
                         self._action_outputs)

        self._output_projection = TPOutputProjection(self._flock_size, self.n_frequent_seqs, self.n_cluster_centers,
                                                     self.seq_length, self.seq_lookahead, self.device)

    def _check_dims(self, cluster_data: torch.Tensor,
                    context_data: torch.Tensor,
                    frequent_seqs: torch.Tensor,
                    frequent_seq_occurrences: torch.Tensor,
                    frequent_context_likelihoods: torch.Tensor,
                    passive_predicted_clusters: torch.Tensor,
                    projection_outputs: torch.Tensor,
                    action_outputs: torch.Tensor):
        assert cluster_data.size() == (self._flock_size, self.n_cluster_centers)
        assert context_data.size() == (self._flock_size, self.n_providers, NUMBER_OF_CONTEXT_TYPES, self.context_size)
        assert frequent_seqs.size() == (self._flock_size, self.n_frequent_seqs, self.seq_length)
        assert frequent_seq_occurrences.size() == (self._flock_size, self.n_frequent_seqs)
        assert frequent_context_likelihoods.size() == (
            self._flock_size, self.n_frequent_seqs, self.seq_length, self.n_providers, self.context_size)
        assert projection_outputs.size() == (self._flock_size, self.n_cluster_centers)
        assert action_outputs.size() == (self._flock_size, self.n_cluster_centers)
        assert passive_predicted_clusters.size() == (self._flock_size, self.seq_length,
                                                     self.n_cluster_centers)

    def run(self):
        """Runs the forward and backward passes.

        Calculates the sequence probabilities, predicted clusters and output projection.
        Stores inputs and results in the buffer.
        """
        with self._buffer.next_step():
            seq_likelihoods_passive = self._forward(self._buffer,
                                                    self._cluster_data,
                                                    self._context_data,
                                                    self._reward_data,
                                                    self._projection_outputs,
                                                    self._passive_predicted_clusters,
                                                    self.cluster_history,
                                                    self.context_history,
                                                    self.seq_likelihoods_clusters,
                                                    self.seq_likelihoods_priors_clusters,
                                                    self.seq_likelihoods_for_each_provider,
                                                    self._seq_likelihoods_by_context,
                                                    self.predicted_clusters_by_context,
                                                    self.seq_likelihoods_priors_clusters_context,
                                                    self._best_matching_context,
                                                    self.seq_probs_clusters_context,
                                                    self._frequent_seqs,
                                                    self._frequent_seq_occurrences,
                                                    self._frequent_context_likelihoods,
                                                    self.context_informativeness)

            if self.compute_backward_pass:

                self._backward(seq_likelihoods_passive,
                               self._context_data,
                               self._frequent_exploration_attempts,
                               self._frequent_exploration_results,
                               self.exploring,
                               self.exploration_random_numbers,
                               self.seq_likelihoods_exploration,
                               self.seq_likelihoods_active,
                               self.seq_likelihoods_priors_clusters_context,
                               self.seq_rewards_goal_directed,
                               self._frequent_context_likelihoods,
                               self._frequent_rewards_punishments,
                               self._frequent_seqs,
                               self._frequent_seq_occurrences,
                               self.active_predicted_clusters,
                               self._action_outputs,
                               self._action_rewards,
                               self._action_punishments,
                               self._influence_model,
                               self._buffer)
            else:
                # just fill some dummy actions and 0 exploring
                self._action_outputs.fill_(0)
                self._buffer.actions.store(self._action_outputs)

                self.exploring.fill_(0)
                self._buffer.exploring.store(self.exploring)

                self._action_rewards.fill_(0)
                self._action_punishments.fill_(0)

            self._execution_counter += 1

    def _compute_predicted_clusters(self,
                                    frequent_seqs: torch.Tensor,
                                    seq_likelihoods: torch.Tensor,
                                    predicted_clusters: torch.Tensor,
                                    cluster_rewards: torch.Tensor = None):
        """Calculate the past, current and predicted cluster probabilities based on sequence likelihoods."""
        # Convert each cluster center id to a set of one-hot vectors corresponding to the cluster in
        # the cluster center space.
        # [flock_size, n_frequent_seqs, seq_length, n_cluster_centers]
        frequent_seqs_unrolled = safe_id_to_one_hot(frequent_seqs, self.n_cluster_centers, self._float_dtype)

        # Expand sequence likelihoods so we can multiply the unrolled freq_seq with it
        seq_likelihoods_expanded = seq_likelihoods.view(self._flock_size, self.n_frequent_seqs, 1, 1)
        seq_likelihoods_expanded = seq_likelihoods_expanded.expand(self._flock_size, self.n_frequent_seqs,
                                                                   self.seq_length, 1)

        # Obtain likelihoods of clusters by multiplying by _seq_likelihoods and summing
        cluster_likelihoods = (frequent_seqs_unrolled * seq_likelihoods_expanded).sum(1)
        if cluster_rewards is not None:
            cluster_rewards.copy_(cluster_likelihoods[:, self.seq_lookbehind, :])

        # Normalise.
        normalize_probs_(cluster_likelihoods, 2)

        # Now copy the normalised likelihoods into the predicted_clusters
        predicted_clusters.copy_(cluster_likelihoods)

    # region Forward

    def _forward(self,
                 buffer: TPFlockBuffer,
                 cluster_data: torch.Tensor,
                 context_data: torch.Tensor,
                 reward_data: torch.Tensor,
                 projection_outputs: torch.Tensor,
                 passive_predicted_clusters_outputs: torch.Tensor,
                 cluster_history: torch.Tensor,
                 context_history: torch.Tensor,
                 seq_likelihoods_clusters: torch.Tensor,
                 seq_likelihoods_priors_clusters: torch.Tensor,
                 seq_likelihoods_for_each_context: torch.Tensor,
                 seq_likelihoods_by_context: torch.Tensor,
                 predicted_clusters_by_context: torch.Tensor,
                 seq_likelihoods_priors_clusters_context: torch.Tensor,
                 best_matching_context: torch.Tensor,
                 seq_probs_clusters_context: torch.Tensor,
                 frequent_seqs: torch.Tensor,
                 frequent_seq_occurrences: torch.Tensor,
                 frequent_context_likelihoods: torch.Tensor,
                 context_informativeness: torch.Tensor) -> torch.Tensor:
        # Add small constant to all inputs which mean probabilities
        buffer.clusters.store(normalize_probs(cluster_data, dim=1, add_constant=True))
        # Don't store the rewards and punishments part from the parent... It's not needed.
        buffer.contexts.store(move_probs_towards_50(context_data[:, :, 0, :]))
        # Do store rewards and punishments that _you_ got this timestep
        buffer.rewards_punishments.store(reward_data)

        if self.compute_best_matching_context:
            # Note seq_likelihoods_by_context if from the last step (not computed for the actual one)
            self._compute_predicted_clusters_for_seq_likelihoods_by_context(frequent_seqs,
                                                                            seq_likelihoods_by_context,
                                                                            predicted_clusters_by_context)

            self._compute_best_matching_context(cluster_data, predicted_clusters_by_context, best_matching_context)

        self._compute_seq_likelihoods(buffer,
                                      cluster_history,
                                      context_history,
                                      seq_likelihoods_clusters,
                                      seq_likelihoods_priors_clusters,
                                      seq_likelihoods_for_each_context,
                                      seq_likelihoods_by_context,
                                      seq_likelihoods_priors_clusters_context,
                                      seq_probs_clusters_context,
                                      frequent_seqs,
                                      frequent_seq_occurrences,
                                      frequent_context_likelihoods,
                                      context_informativeness)

        # self._delete_improbable_seqs()

        # decide if we want to use context or without
        passive_seq_likelihoods = seq_likelihoods_priors_clusters_context

        buffer.seq_probs.store(normalize_probs(passive_seq_likelihoods, dim=1))

        self._apply_output_projection(frequent_seqs, passive_seq_likelihoods,
                                      projection_outputs)

        self._compute_predicted_clusters(frequent_seqs, passive_seq_likelihoods,
                                         passive_predicted_clusters_outputs)

        buffer.outputs.store(projection_outputs)

        return passive_seq_likelihoods

    def _compute_predicted_clusters_for_seq_likelihoods_by_context(self,
                                                                   frequent_seqs: torch.Tensor,
                                                                   seq_likelihoods_by_context: torch.Tensor,
                                                                   predicted_clusters_by_context: torch.Tensor
                                                                   ):
        """
        Compute predicted cluster likelihood for each context.

        Args:
            frequent_seqs: ['flock_size', 'tp_n_frequent_seqs', 'tp_seq_length'] -> 'cluster_id'
            seq_likelihoods_by_context: ['flock_size', 'tp_n_frequent_seqs', 'n_providers', 'context_size']
            predicted_clusters_by_context: ['flock_size', 'n_providers', 'context_size', 'n_cluster_centers']
        """
        predicted_cluster_output = torch.empty((self._flock_size, self.seq_length, self.n_cluster_centers),
                                               dtype=self._float_dtype, device=self.device)
        for context_id in range(self.context_size):
            for provider_id in range(self.n_providers):
                # [flock_size, n_frequent_seqs]
                context_likelihoods = seq_likelihoods_by_context[:, :, provider_id, context_id]
                if context_likelihoods.sum() == 0:
                    # Note: correct would be to treat each expert in the flock separately
                    # (i.e. determine all zeros per dim = 0). However, zeros are now just in the first step, this is OK.
                    context_likelihoods += SMALL_CONSTANT
                self._compute_predicted_clusters(frequent_seqs, context_likelihoods, predicted_cluster_output)
                # [flock_size, n_cluster_centers]
                predicted_cluster_output_next_symbol = predicted_cluster_output[:, self.seq_lookbehind, :]
                predicted_clusters_by_context[:, provider_id, context_id, :] = predicted_cluster_output_next_symbol

    def _compute_best_matching_context(self,
                                       cluster_data: torch.Tensor,
                                       predicted_clusters_by_context: torch.Tensor,
                                       best_matching_context: torch.Tensor):
        """
        Args:
            cluster_data: ['flock_size', 'n_cluster_centers']
            predicted_clusters_by_context: ['flock_size', 'n_providers', 'context_size', 'n_cluster_centers']
            best_matching_context: ['flock_size', 'n_providers', 'context_size']
        """
        input_data_expanded = multi_unsqueeze(cluster_data, [1, 2]).expand(predicted_clusters_by_context.shape)
        multiplied = input_data_expanded * predicted_clusters_by_context
        summed = multiplied.sum(dim=3)
        normalize_probs_(summed, dim=2, add_constant=True)
        best_matching_context.copy_(summed)

    def _compute_seq_likelihoods(self,
                                 buffer: TPFlockBuffer,
                                 cluster_history: torch.Tensor,
                                 context_history: torch.Tensor,
                                 seq_likelihoods_clusters: torch.Tensor,
                                 seq_likelihoods_priors_clusters: torch.Tensor,
                                 seq_likelihoods_for_each_context: torch.Tensor,
                                 seq_likelihoods_by_context: torch.Tensor,
                                 seq_likelihoods_priors_clusters_context: torch.Tensor,
                                 seq_probs_clusters_context: torch.Tensor,
                                 frequent_seqs: torch.Tensor,
                                 frequent_seq_occurrences: torch.Tensor,
                                 frequent_context_likelihoods: torch.Tensor,
                                 context_informativeness: torch.Tensor):
        """Calculates the sequence likelihoods."""

        self._compute_seq_likelihoods_priors_clusters(buffer,
                                                      cluster_history,
                                                      seq_likelihoods_clusters,
                                                      seq_likelihoods_priors_clusters,
                                                      frequent_seqs,
                                                      frequent_seq_occurrences)

        self._compute_seq_likelihoods_for_each_provider(buffer,
                                                        context_history,
                                                        seq_likelihoods_priors_clusters,
                                                        frequent_context_likelihoods,
                                                        seq_likelihoods_for_each_context,
                                                        seq_likelihoods_by_context)

        self._compute_provider_informativeness(seq_likelihoods_priors_clusters,
                                               seq_likelihoods_for_each_context,
                                               context_informativeness)

        # Uses the most informative context.
        self._compute_seq_likelihoods_priors_clusters_context(seq_likelihoods_priors_clusters_context,
                                                              seq_likelihoods_for_each_context,
                                                              context_informativeness)

        self._compute_seq_probs_without_priors(seq_likelihoods_priors_clusters_context,
                                               frequent_seq_occurrences,
                                               seq_probs_clusters_context)

    def _compute_seq_likelihoods_priors_clusters(self,
                                                 buffer: TPFlockBuffer,
                                                 cluster_history: torch.Tensor,
                                                 seq_likelihoods_clusters: torch.Tensor,
                                                 seq_likelihoods_priors_clusters: torch.Tensor,
                                                 frequent_seqs: torch.Tensor,
                                                 frequent_seq_occurrences: torch.Tensor):
        """Calculates sequence likelihoods based clusters only, and based on clusters with priors."""
        buffer.clusters.sample_forward_batch(self.seq_lookbehind, out=cluster_history)

        # compute the likelihoods based on clusters only
        tp_process_kernels.compute_seq_likelihoods_clusters_only(cluster_history,
                                                                 frequent_seqs,
                                                                 frequent_seq_occurrences,
                                                                 seq_likelihoods_clusters,
                                                                 self._flock_size,
                                                                 self.n_frequent_seqs,
                                                                 self.seq_lookbehind)

        # multiply by priors to get the likelihoods without context
        torch.mul(seq_likelihoods_clusters, frequent_seq_occurrences, out=seq_likelihoods_priors_clusters)

    def _compute_seq_likelihoods_for_each_provider(self,
                                                   buffer: TPFlockBuffer,
                                                   context_history: torch.Tensor,
                                                   seq_likelihoods_priors_clusters: torch.Tensor,
                                                   frequent_context_likelihoods: torch.Tensor,
                                                   seq_likelihoods_for_each_provider: torch.Tensor,
                                                   seq_likelihoods_by_context: torch.Tensor):
        """Calculates sequence likelihoods based on context history and cluster history.

        Calculates independently for each parent.

        Args:
            seq_likelihoods_for_each_provider - Output
        """

        buffer.contexts.sample_forward_batch(self.seq_lookbehind, out=context_history)

        # Expand the history over the lookbehind of frequent context probs so they can be multiplied.
        context_history_expanded = context_history.unsqueeze(dim=1).expand(self._flock_size,
                                                                           self.n_frequent_seqs,
                                                                           self.seq_lookbehind,
                                                                           self.n_providers,
                                                                           self.context_size)

        likelihoods = frequent_context_likelihoods[:, :, :self.seq_lookbehind, :, :] * context_history_expanded
        # get rid of nans for the contexts which are shorten than the max context size and sum over context size
        likelihoods.masked_fill_(torch.isnan(likelihoods), 0)
        likelihoods = likelihoods.sum(dim=4)

        # Multiply by the likelihoods taken from the history of clusters and priors to get the real distribution of
        # sequences
        likelihoods *= multi_unsqueeze(seq_likelihoods_priors_clusters, [2, 3]).expand(likelihoods.size())

        # Take the sequence lookbehind
        seq_likelihoods_for_each_provider.copy_(likelihoods)

        # compute
        likelihoods_by_context = frequent_context_likelihoods[:, :, self.seq_lookbehind, :, :].clone()
        likelihoods_by_context.masked_fill_(torch.isnan(likelihoods_by_context), 0)
        likelihoods_by_context *= multi_unsqueeze(seq_likelihoods_priors_clusters, [2, 3]).expand(likelihoods_by_context.size())

        # result = normalize_probs(likelihoods_by_context, dim=4, add_constant=False)
        seq_likelihoods_by_context.copy_(likelihoods_by_context)

    @staticmethod
    def _compute_provider_informativeness(seq_likelihoods_priors_clusters: torch.Tensor,
                                          seq_likelihoods_for_each_provider: torch.Tensor,
                                          provider_informativeness: torch.Tensor):
        """Calculates the KL divergence between the prob. dist. over seqs without context and with each context."""

        # At this point we expect that there are no exactly 0 (except for invalid frequent sequencies) or
        # exactly 1 values in the probabilities, so we do not add small constant here. If we did, it would cause
        # the invalid frequent sequences to have probabilities > 0.

        normalized_baseline = multi_unsqueeze(normalize_probs(seq_likelihoods_priors_clusters, dim=1,
                                                              add_constant=False), [2, 3]).expand(seq_likelihoods_for_each_provider.size())

        normalized_seq_likelihoods_for_each_context = normalize_probs(seq_likelihoods_for_each_provider, dim=1,
                                                                      add_constant=False)

        kl_divergence(normalized_baseline, normalized_seq_likelihoods_for_each_context,
                      output=provider_informativeness, dim=1)

    def _compute_seq_likelihoods_priors_clusters_context(self,
                                                         seq_likelihoods_priors_clusters_context: torch.Tensor,
                                                         seq_likelihoods_for_each_context: torch.Tensor,
                                                         context_informativeness: torch.Tensor):
        """Pick the most informative context for each expert and store it as seq_likelihoods_priors_clusters_context.
        Args:
            seq_likelihoods_priors_clusters_context: Output
        """

        # Find maximum over both last dimensions: lookbehind and context_size
        most_informative_context = torch.argmax(context_informativeness.view(self._flock_size, -1), dim=1)

        each_context_viewed = seq_likelihoods_for_each_context.view(
            self._flock_size, self.n_frequent_seqs, -1)

        indexes = most_informative_context.view(self._flock_size, 1, 1).expand(self._flock_size, self.n_frequent_seqs,
                                                                               1)

        torch.gather(input=each_context_viewed, dim=2, index=indexes,
                     out=seq_likelihoods_priors_clusters_context.unsqueeze(2))

    @staticmethod
    def _compute_seq_probs_without_priors(seq_likelihoods_priors_clusters_context: torch.Tensor,
                                          frequent_seq_occurrences: torch.Tensor,
                                          seq_probs_clusters_context: torch.Tensor):
        # Compute the sequence probabilities without the priors
        torch.div(input=seq_likelihoods_priors_clusters_context,
                  other=frequent_seq_occurrences,
                  out=seq_probs_clusters_context)

        # ... and get rid of any nans (invalid freq_seqs have 0 occurrences)
        seq_probs_clusters_context.masked_fill_(torch.isnan(seq_probs_clusters_context), 0)

        # normalize to get probabilities
        normalize_probs_(seq_probs_clusters_context, dim=1)

    def _apply_output_projection(self,
                                 frequent_seqs: torch.Tensor,
                                 # [flock_size, n_frequent_seqs, seq_length]
                                 seq_likelihoods: torch.Tensor,
                                 outputs: torch.Tensor):
        """Transfer the sequence likelihoods into the cluster space with more weight on clusters near in time."""
        self._output_projection.compute_output_projection(frequent_seqs, seq_likelihoods, outputs)

    # endregion

    # region Backward

    def _backward(self,
                  seq_likelihoods_passive: torch.Tensor,
                  context_data: torch.Tensor,
                  frequent_exploration_attempts: torch.Tensor,
                  frequent_exploration_results: torch.Tensor,
                  exploring: torch.Tensor,
                  exploration_random_numbers: torch.Tensor,
                  seq_likelihoods_exploration: torch.Tensor,
                  seq_likelihoods_active: torch.Tensor,
                  seq_likelihoods_priors_clusters_context: torch.Tensor,
                  seq_rewards_goal_directed: torch.Tensor,
                  frequent_context_likelihoods: torch.Tensor,
                  frequent_rewards_punishments: torch.Tensor,
                  frequent_seqs: torch.Tensor,
                  frequent_seq_occurrences: torch.Tensor,
                  active_predicted_clusters: torch.Tensor,
                  action_outputs: torch.Tensor,
                  action_rewards: torch.Tensor,
                  action_punishments: torch.Tensor,
                  influence_model: torch.Tensor,
                  buffer: TPFlockBuffer):

        # The influence model is the normalised probabilities of the exploration results
        influence_model[:, :, -self.seq_lookahead:] = normalize_probs(frequent_exploration_results, dim=3)

        # Normalise the sequence likelihoods to obtain probabilities
        seq_probs_priors_clusters_context = normalize_probs(seq_likelihoods_priors_clusters_context, dim=1)

        # NOTE: Punishements (and thus summations with rewards) are disabled
        # seq_likelihoods_goal_directed = seq_rewards_goal_directed.sum(dim=2)

        # decide which likelihoods to use passive vs goal_directed
        if self.follow_goals:
            self._compute_rewards(frequent_seqs,
                                  frequent_seq_occurrences,
                                  context_data,
                                  frequent_context_likelihoods,
                                  seq_probs_priors_clusters_context,
                                  frequent_rewards_punishments,
                                  influence_model,
                                  seq_rewards_goal_directed)

            # We only need the rewards here
            seq_likelihoods_active.copy_(seq_rewards_goal_directed[:, :, 0])
        else:
            seq_likelihoods_active.copy_(seq_likelihoods_passive)

        self._exploration_sequence(exploring, exploration_random_numbers, seq_likelihoods_active, buffer)

        self._compute_predicted_clusters(frequent_seqs,
                                         seq_likelihoods_active,
                                         active_predicted_clusters,
                                         action_rewards)

        # Store the actions that we care about
        # NOTE: Punishments to be considered in the future
        #
        # action_outputs.copy_(id_to_one_hot(torch.argmax(active_predicted_clusters[:, self.seq_lookbehind, :]), vector_len=self.n_cluster_centers))
        action_outputs.copy_(active_predicted_clusters[:, self.seq_lookbehind, :])

        # decide if exploring and if yes, overwrite the action_outputs
        self._exploration_cluster(exploration_random_numbers, action_outputs, action_rewards)

        # Store punishments to the punishments part too
        # NOTE: Punishments here are for sequences,  should be converted to clusters first
        # action_punishments.copy_(seq_rewards_goal_directed[:, :, 1])

        # store it to buffer
        buffer.actions.store(action_outputs)

    def _exploration_sequence(self, exploring: torch.Tensor, exploration_random_numbers: torch.Tensor,
                             sequence_likelihoods_active: torch.Tensor, buffer: TPFlockBuffer):
        """Decide if exploring. If yes, overwrite the sequence_likelihoods by a random sequence which will get
         converted to a corresponding cluster center later. Update the rewards with a corresponding exploration reward.

           Also stores the exploration flag into the buffer."""

        # Randomly decide which experts will be exploring
        exploration_random_numbers.uniform_(0, 1.0)
        exploring.copy_(exploration_random_numbers < self.exploration_probability)

        buffer.exploring.store(exploring)

        random_indices = torch.randint(low=0, high=self.n_frequent_seqs, size=(self._flock_size,), device=self.device)
        # one-hot representation of the sequences we want to explore right now
        exploration_sequences = id_to_one_hot(random_indices, self.n_frequent_seqs, dtype=self._float_dtype)

        exploration_sequences *= EXPLORATION_REWARD

        # overwrite the actions for experts which are exploring by exploration actions
        weighted_sum_(tensor_a=exploration_sequences, weight_a=exploring,
                      tensor_b=sequence_likelihoods_active, weight_b=1.0 - exploring,
                      output=sequence_likelihoods_active)

    def _exploration_cluster(self, exploration_random_numbers: torch.Tensor,
                             action_outputs: torch.Tensor, action_rewards: torch.Tensor):
        """For all those exploring, they have a self.exploration_probability chance of a totally random cluster being picked to explore instead
         of the sequence they were going to do in _exploration_sequence. Update rewards accordingly.

         """
        cluster_exploration = (exploration_random_numbers < self.cluster_exploration_prob * self.exploration_probability).float()

        random_indices = torch.randint(low=0, high=self.n_cluster_centers, size=(self._flock_size,), device=self.device)
        # one-hot representation of the actions (=clusters) we want to explore right now
        exploration_actions = id_to_one_hot(random_indices, self.n_cluster_centers, dtype=self._float_dtype)
        exploration_rewards = exploration_actions * EXPLORATION_REWARD

        # overwrite the actions for experts which are exploring by exploration actions
        weighted_sum_(tensor_a=exploration_actions, weight_a=cluster_exploration,
                      tensor_b=action_outputs, weight_b=1.0 - cluster_exploration,
                      output=action_outputs)

        # Do the same for the rewards
        weighted_sum_(tensor_a=exploration_rewards, weight_a=cluster_exploration,
                      tensor_b=action_rewards, weight_b=1.0 - cluster_exploration,
                      output=action_rewards)

    # endregion

    def _compute_rewards(self,
                         frequent_sequences: torch.Tensor,  # [flock_size, n_frequent_seqs, seq_len]
                         frequent_seq_occurrences: torch.Tensor,
                         context_data: torch.Tensor,  # [flock_size, n_providers, 3, context_size]
                         frequent_context_likelihoods: torch.Tensor,  # [flock_size, n_frequent_seqs, seq_length, n_providers, context_size)
                         seq_probs_priors_clusters_context: torch.Tensor,
                         frequent_rewards_punishments: torch.Tensor, # [flock_size, n_frequent_seqs, seq_length, 2]
                         influence_model: torch.Tensor,  # [flock_size, n_frequent_seqs, seq_lookahead, n_cluster_centers]
                         seq_rewards_goal_directed: torch.Tensor):  # [flock_size, n_frequent_seqs, 2]
        """Computes the expected rewards from each context provider for the next transition of all frequent sequences.

        We traverse the future clusters backwards using the current rewards context and remembered context probs.
        We calculate the expected rewards for reaching each cluster in each sequence using the current rewards context,
        then proceed to iteratively work backwards through the lookahead of the sequence updating the expected rewards
        by introducing the influence model and applying its effects to our model of the expected values.

        Once the expected rewards are calculated for all the lookahead transitions of each sequence for each provider,
        the rewards are discounted by the prior likelihoods and the expected rewards for the next step are saved."""
        influence_model[torch.isnan(influence_model)] = 0
        context_data[torch.isnan(context_data)] = 0

        # Unsqueeze and expand both the context data and the context on probs so that they can be multiplied
        context_data_rewards_punishments = multi_unsqueeze(context_data[:, :, 1:],
                                                           [1, 2]).expand(self._flock_size, self.n_frequent_seqs,
                                                                          self.seq_length, self.n_providers, 2,
                                                                          self.context_size)

        context_on_probs = frequent_context_likelihoods.unsqueeze(dim=4).expand(self._flock_size, self.n_frequent_seqs,
                                                                                self.seq_length, self.n_providers, 2,
                                                                                self.context_size)

        # Calculate the rewards/punishments for each cluster in each freq_seq lookahead assuming every
        # transition is perfect. AKA the undiscounted rewards from the parents.
        current_rewards = (context_data_rewards_punishments * context_on_probs).sum(dim=5)

        # Get the average freq_reward that this expert has seen by dividing the seen rewards by the occurrences
        frequent_rewards_punishments_scaled =\
            frequent_rewards_punishments / multi_unsqueeze(frequent_seq_occurrences.float() + SMALL_CONSTANT, [2, 3]).expand(frequent_rewards_punishments.size())

        # Expand the scaled rewards for addition to the current rewards
        # NOTE: This can be done in extract frequent sequence in the learning process instead of here
        frequent_rewards_punishments_scaled = frequent_rewards_punishments_scaled.unsqueeze(dim=3).expand(self._flock_size, self.n_frequent_seqs, self.seq_lookahead,
                                                                 self.n_providers, 2) * self.own_rewards_weight

        # Add in the scaled rewards to the lookahead part of the current rewards.
        current_rewards[:, :, self.seq_lookbehind:, :, :] += frequent_rewards_punishments_scaled

        # We iterate backwards through the possibilities of sequences to transform the undiscounted cluster rewards into
        # expected_values of following transitions

        # Allocate some temp storage for the kernel - This is the expected rewards/punishments for each destination
        # cluster when following the sequence from the current cluster, for all parents and all sequences
        cluster_rewards = torch.zeros((self._flock_size, self.n_frequent_seqs, self.n_providers, 2,
                                       self.n_cluster_centers), dtype=self._float_dtype, device=self.device)

        n_transitions = self.seq_length - 1
        # If this is the bottom level, we process all lookahead transitions, otherwise only down to
        # sequence_lookahead - 1. We do this so that higher levels of abstraction don't 'double discount' the expected
        # rewards of the lowest level for the next transition.
        if self.produce_actions:
            last_processed_transition = n_transitions - self.seq_lookahead
        else:
            last_processed_transition = n_transitions - (self.seq_lookahead - 1)

        # Now we iterate over the transitions of the sequence, starting from the last one (at n_transitions),
        # and moving towards the current point in the sequence (marked by last_processed_transition). The indices
        # for processing this range are a bit funky, as the index for the final transition is n_transitions-1, and the
        # true last_processed_transition index is last_processed_transition - 1.
        for transition in range(n_transitions - 1, last_processed_transition - 1, -1):
            cluster_rewards.fill_(0)

            tp_process_kernels.discount_rewards_iterative(frequent_sequences,
                                                          seq_probs_priors_clusters_context,
                                                          current_rewards,
                                                          influence_model,
                                                          cluster_rewards,
                                                          self._flock_size,
                                                          self.n_frequent_seqs,
                                                          self.n_cluster_centers,
                                                          transition,
                                                          self.n_providers)
            # Get the EV of trying to get to the next cluster of this sequence.
            transition_rewards = cluster_rewards.sum(dim=4) * REWARD_DISCOUNT_FACTOR

            # Update current rewards with the new EV of this transition (using the EV of rewards gained
            # when attempting this transition)
            current_rewards[:, :, transition + 1] = transition_rewards

            # Take max of transition EV vs origin cluster EV (for each provider) and update current rewards with it
            # Creates lower bound for EV
            max_ev = torch.max(current_rewards[:, :, transition], current_rewards[:, :, transition + 1])
            current_rewards[:, :, transition] = max_ev

        # Scale all rewards by their prior sequence probabilities
        discounted_rewards = current_rewards * multi_unsqueeze(seq_probs_priors_clusters_context,
                                                               [2, 3, 4]).expand(self._flock_size, self.n_frequent_seqs,
                                                                                self.seq_length, self.n_providers, 2)

        # we care about the rewards and punishments for this transition, so get them
        next_potentials = discounted_rewards[:, :, self.seq_lookbehind]

        # Find max over all providers in each sequence in the influence_model
        next_potentials, _ = torch.max(next_potentials, dim=2)

        # Copy this into the return
        seq_rewards_goal_directed.copy_(next_potentials)


# -------------------------------------------------------------------------
# AUTHOR: Joey Cindass
# FILENAME: naive_bayes.py
# SPECIFICATION: Using naive bayes, predict if someone will play tennis given weather conditions
# FOR: CS 4210- Assignment #2
# TIME SPENT: 4 hour
# -----------------------------------------------------------*/

# IMPORTANT NOTE: DO NOT USE ANY ADVANCED PYTHON LIBRARY TO COMPLETE THIS CODE SUCH AS numpy OR pandas. You have to
# work here only with standard dictionaries, lists, and arrays

# importing some Python libraries
from sklearn.naive_bayes import GaussianNB
import csv

# reading the training data in a csv file
# --> add your Python code here
db = []
with open('weather_training.csv', 'r') as csvfile:
    reader = csv.reader(csvfile)
    for i, row in enumerate(reader):
        if i > 0:  # skipping the header
            db.append(row[1:])

# transform the original training features to numbers and add them to the 4D array X.
# For instance Sunny = 1, Overcast = 2, Rain = 3, so X = [[3, 1, 1, 2], [1, 3, 2, 2], ...]]
# --> add your Python code here
# X =
X = []
for i in range(len(db)):
    X_temp = []
    for j in range(len(db[i])):
        ones = ['Sunny', 'Cool', 'Normal', 'Weak']
        twos = ['Overcast', 'Mild', 'High', 'Strong']
        threes = ['Rain', 'Hot']
        if db[i][j] in ones:
            X_temp.append(1)
        elif db[i][j] in twos:
            X_temp.append(2)
        elif db[i][j] in threes:
            X_temp.append(3)
    X.append(X_temp)

# transform the original training classes to numbers and add them to the vector Y.
# For instance Yes = 1, No = 2, so Y = [1, 1, 2, 2, ...]
# --> add your Python code here
# Y =
Y = []
for i in range(len(db)):
    if db[i][-1] == "Yes":
        Y.append(1)
    else:
        Y.append(0)

# fitting the naive bayes to the data
clf = GaussianNB()
clf.fit(X, Y)

# reading the test data in a csv file
# --> add your Python code here
db2 = []
with open('weather_test.csv', 'r') as csvfile:
    reader = csv.reader(csvfile)
    for i, row in enumerate(reader):
        if i > 0:  # skipping the header
            db2.append(row[1:])

# printing the header os the solution
print("Day Outlook  Temperature Humidity Wind    PlayTennis Confidence")

# use your test samples to make probabilistic predictions. For instance: clf.predict_proba([[3, 1, 2, 1]])[0]
# --> add your Python code here

X2 = []
for i in range(len(db2)):
    X2_temp = []
    for j in range(len(db2[i])):
        ones = ['Sunny', 'Cool', 'Normal', 'Weak']
        twos = ['Overcast', 'Mild', 'High', 'Strong']
        threes = ['Rain', 'Hot']
        if db2[i][j] in ones:
            X2_temp.append(1)
        elif db2[i][j] in twos:
            X2_temp.append(2)
        elif db2[i][j] in threes:
            X2_temp.append(3)
    X2.append(X2_temp)

conf_list = []
pred_list = []
for instance in X2:
    confidence = clf.predict_proba([instance])[0]
    pred = clf.predict([instance])
    if pred == 1:
        pred_list.append("Yes")
    else:
        pred_list.append("No")
    conf_list.append(confidence.max())

for i in range(len(db2)):
    if conf_list[i] >= 0.75:
        print(f"D{i+15} {db2[i][0]:8} {db2[i][1]:11} {db2[i][2]:8} {db2[i][3]:4}    {pred_list[i]} {conf_list[i]:25}")


import nltk
from nltk.tokenize import TweetTokenizer
from nltk.corpus import stopwords
from nltk.corpus import opinion_lexicon
import string
from nltk.stem.porter import *
from sklearn.feature_extraction.text import CountVectorizer
import math
import csv

class TweetEnricher:
    """
    Class that adds features to tweets collected
    """
    def __init__(self):
        self.tokenizer = TweetTokenizer()
        self.stemmer = PorterStemmer()
        self.stopset = set(stopwords.words('english'))
        self.negative_opinions = opinion_lexicon.negative()
        self.positive_opinions = opinion_lexicon.positive()
        self.brexit_keywords = [line.rstrip('\n') for line in open('../Data/Lists/BrexitKeywords')]
        self.vulgar_words = [line.rstrip('\n').lower() for line in open('../Data/Lists/VulgarWordsList')]
        self.twitter_jargons = [line.rstrip('\n') for line in open('../Data/Lists/TwitterSlangsAndAbbreviations')]
        self.web_abbreviations = [line.rstrip('\n').lower() for line in open('../Data/Lists/WebAcronymns')]
        self.emoticons_list = [line.rstrip('\n') for line in open('../Data/Lists/EmojiList')]
        self.pos_emoticons_list = [line.rstrip('\n') for line in open('../Data/Lists/PositiveEmojiList')]
        self.neg_emoticons_list = [line.rstrip('\n') for line in open('../Data/Lists/NegativeEmojiList')]
        self.first_person_pronouns = [line.rstrip('\n') for line in open('../Data/Lists/FirstPersonPronouns')]
        self.speech_act_verbs = [line.rstrip('\n') for line in open('../Data/Lists/StemmedSpeechActVerbs')]
        self.trusted_domains = [line.rstrip('\n') for line in open('../Data/Lists/TrustedDomains')]
        self.verb_tags = ['VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ']
        self.n_gram_count_matrix = {}
        self.vectorizer = CountVectorizer(ngram_range=(1, 3), tokenizer=self.tokenizer.tokenize,
                                          stop_words=list(self.stopset)
                                                     + self.web_abbreviations
                                                     + list(string.punctuation)
                                                     + ["…", "...", "..", ")", "(", "-->", "->", ">>", "#", "RT", "@"])
        self.vectorizer_unigram = CountVectorizer(ngram_range=(1, 1), tokenizer=self.tokenizer.tokenize,
                                                  stop_words=list(self.stopset)
                                                             + self.web_abbreviations
                                                             + list(string.punctuation))
        self.positive_ops = [x.lower() for x in self.positive_opinions]
        self.negative_ops = [x.lower() for x in self.negative_opinions]

    def tweetFeatures(self):
        self.tweet_features = ["VulgarWords", "Emoticons", "Interrogation", "Exclamation", "Abbreviations",
                               "TwitterJargons", "#", "# position", "@", "@ position", "RT", "RT position", "Link"]
        for w in self.speech_act_verbs:
            self.tweet_features.append(w)
        self.tweet_features.append(["NegativeOpnions", "PositiveOpnions","NumbersPresent","Non-Ascii Characters","GoodLink"])
        self.basic_features = self.tweet_features
        for w in self.n_gram_count_matrix:
            self.tweet_features.append(w)
        return self.basic_features, self.tweet_features, self.n_gram_count_matrix

    def tokenize(self,tweet):
        """
        Tokens created using nltk tokenizer
        :return: tokens from tweet
        """
        return self.tokenizer.tokenize(tweet)

    def removeStopWords(self,tokens):
        """
        Removes stop words from tokens
        :return: tokens without stopwords
        """
        tokens_without_stopwords = [w for w in tokens if w not in self.stopset]
        return tokens_without_stopwords

    def hasNegativeOpinions(self,tokens):
        """
        Checks if negative opinions from nltk corpus present
        :return: 1 if negative opinions present. 0 otherwise
        """
        count = 0
        negative = 0
        negative_ops = [x.lower() for x in self.negative_opinions]
        for w in tokens:
            if w.lower() in negative_ops:
                count += 1
        if count > 0:
            negative = 1
        return count, negative

    def hasPositiveOpinions(self,tokens):
        '''
        Checks if positive opnions present
        :return: 1 if positive opinions present. 0 otherwise
        '''
        count = 0
        positive = 0
        positive_ops = [x.lower() for x in self.positive_opinions]
        for w in tokens:
            if w.lower() in positive_ops:
                count += 1
        if count > 0:
            positive = 1
        return count, positive

    def hasVulgarWords(self,tokens):
        '''
        Checks if vulgar words from online list present
        :return: 1 if present
        '''
        for w in tokens:
            if w.lower() in self.vulgar_words:
                return 1
        return 0

    def hasAbbreviations(self,tokens):
        '''
        Checks if abbreviations from online list present
        :return: 1 if present
        '''
        for w in tokens:
            if w.lower() in self.web_abbreviations:
                return 1
        return 0

    def hasTwitterJargons(self,tokens):
        '''
        Checks if twitter specific abbreviations and jargons from online list present
        :return: 1 if present
        '''
        for w in tokens:
            if w in self.twitter_jargons:
                return 1
        return 0

    def hasEmoticons(self,tokens):
        '''
        Checks if abbreviations from online list present
        :return: 1 if present
        '''
        for w in tokens:
            if w in self.emoticons_list:
                return 1
        return 0

    def isInterrogative(self,tokens):
        '''
        Checks if ? present
        :return: 1 if present
        '''
        for term in tokens:
            if term in string.punctuation:
                if term == "?":
                    return 1
        return 0

    def isExclamatory(self,tokens):
        '''
        Checks if ! present
        :return: 1 if present
        '''
        for term in tokens:
            if term in string.punctuation:
                if term == "!":
                    return 1
        return 0

    def hasHash(self,tokens):
        '''
        Checks if # present
        :return: 1 if present, 0 otherwise; 1 if present in the beginning, 0 oherwise
        '''
        for index, term in enumerate(tokens):
            if re.match("#\w*", term):
                if 2 * index < len(tokens):
                    return 1,1
                else:
                    return 1,0
        return 0,0

    def hasRT(self,tokens):
        '''
        Checks if RT present
        :return: 1 if present, 0 otherwise; 1 if present in the beginning, 0 oherwise
        '''
        for index, term in enumerate(tokens):
           if term == "RT":
                if 2 * index < len(tokens):
                    return 1,1
                else:
                    return 1,0
        return 0,0

    def hasATag(self,tokens):
        '''
        Checks if @ present
        :return: 1 if present, 0 otherwise; 1 if present in the beginning, 0 oherwise
        '''
        for index, term in enumerate(tokens):
            if re.match("@\w*", term):
                if 2 * index < len(tokens):
                    return 1,1
                else:
                    return 1,0
        return 0,0

    def hasALink(self,tokens):
        """
        Checks if the tweet has a url
        :return: 1 if url present
        """
        for w in tokens:
            if re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', w):
                return 1
        return 0

    def hasManyNumbers(self,tokens):
        """
        Checks if the tweet many numbers
        :return: 1 if more than 5 numbers present
        """
        count = 0
        for w in tokens:
            if re.match('[\d]+',w):
               count = count + 1
        if count > 2:
            return 1
        else:
            return 0

    def hasManyNonAscii(self, tokens):
        """
        Checks if the tweet has non-ascii characters
        :return: 1 if more than 5 such present
        """
        count = 0
        for token in tokens:
            if re.match('[^\x00-\x7F]',token):
                count = count + 1
        if count > 5:
            return 1
        else:
            return 0

    def hasLinksToReputedDomains(self, url_string):
        """
        Checks if the tweet links to some trusted pages
        :return: 1 if more any such link present
        """
        for i in self.trusted_domains:
            pattern = re.compile(i+'.*')
            if re.findall(pattern,url_string):
                return 1
        return 0

    def hasSpeechActVerbs(self,tokens):
        '''
        Returns 265 binary features for the speech act verbs
        :return: a dictionary
        '''
        speech_act_feature_dict = {}        # dict stores the 265 features
        for w in self.speech_act_verbs:     # initializing
            speech_act_feature_dict[w] = 0
        pos_list = nltk.pos_tag(tokens)
        for entry in pos_list:
            if entry[1] in self.verb_tags:   # check if the tokens that are verbs are speech act verbs
                verb = self.stemmer.stem(entry[0].lower())
                if verb in self.speech_act_verbs:
                    speech_act_feature_dict[verb] = 1
        return speech_act_feature_dict

    def collectNGramFeatures(self,tweet):
        '''
        Collects the n-grams present in tweet that also occur through out the document more than 5 times
        :return: a count matrix
        '''

        # initializing matrix to all 0s
        tweet_n_gram_count_dict = {}

        for gram in self.n_gram_count_matrix:
            tweet_n_gram_count_dict[gram] = 0
            p = re.compile(re.escape(gram))
            if re.findall(p, str(tweet)):
                tweet_n_gram_count_dict[gram] = 1

        return tweet_n_gram_count_dict

    def returnUnigramMatrix(self,collection):
        '''
        Reurns unigrams in document- done to fetch Brexit related keywords
        '''
        X = self.vectorizer_unigram.fit_transform(collection)
        feature_names = self.vectorizer_unigram.get_feature_names()
        term_freqs = X.sum(axis=0).A1
        unigram_count_matrix = dict(zip(feature_names, term_freqs))

        return unigram_count_matrix

    def calculateNGramsEntropy(self, feature_names, SA_tagged_collection ):
        '''
        Calculates entropy of n-grams (uni, bi and tri grams)
        :param feature_names: n-grams
        :param SA_tagged_collection: speect act tagged collection
        '''

        #Remove n-grams with #,@,RT, only numbers
        for i in feature_names:
            if re.findall('#.*', i):
                feature_names.remove(i)
            elif re.findall('[\d]+', i):
                feature_names.remove(i)
            elif re.findall('@.*', i):
                feature_names.remove(i)
            elif re.findall('RT.*', i):
                feature_names.remove(i)
            elif re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+',i):
                feature_names.remove(i)

        # Remove potential Brexit keywords
        for i in feature_names:
            for j in i.split():
                if j in self.brexit_keywords:
                    feature_names.remove(i)
                    break

        for term in nltk.pos_tag(feature_names):
            # Remove proper nouns
            if term[1] == "NNP" or term[1] == "NNPS" or term[1] == "NN":
                feature_names.remove(term[0])

        #Find entropy of each n-gram
        entropy = {}
        for item in feature_names:
            entropy[item] = 0
        total = len(SA_tagged_collection)
        occurrences = {}
        tagged_tweets= {}
        for gram in SA_tagged_collection:
            if SA_tagged_collection.get(gram)[1] != '':
                tagged_tweets[gram] = SA_tagged_collection.get(gram)

        for item in feature_names:
            p = re.compile(re.escape(item))
            for gram in tagged_tweets:
                occurrences[tagged_tweets.get(gram)[1]] = 0
            for gram in tagged_tweets:
                times_in_tweet = len(re.findall(p, str(tagged_tweets.get(gram)[0])))
                occurrences[tagged_tweets.get(gram)[1]] += times_in_tweet
            for gram in tagged_tweets:
                entropy[item] += -(occurrences[tagged_tweets.get(gram)[1]]/total)*math.log((occurrences[tagged_tweets.get(gram)[1]]+1)/total)

        # n-gram entropies
        with open('../Data/test/entropy_raw.csv', 'w+', newline='', encoding='utf8') as out_file:
            writer = csv.writer(out_file)
            for w in sorted(entropy, key=entropy.get):
                writer.writerow([w, entropy.get(w)])

        # entropies greater than 0 and less than 0.02 chosen.
        with open('../Data/test/N_grams_final.csv', 'w+', newline='', encoding='utf8') as out_file:
            writer = csv.writer(out_file)
            for w in entropy:
                if entropy.get(w) > 0 and entropy.get(w) < 0.02 :
                    writer.writerow([w])

    def createNGramCountMatrix(self,collection,SA_tagged_collection, create_n_grams):
        '''
        Creates a n-gram count matrix of all the tweets. N-grams that have no common nouns and occur more than 5 times
        :return: a count matrix
        '''
        X = self.vectorizer.fit_transform(collection)
        feature_names = self.vectorizer.get_feature_names()
        print("Size of n grams =  %d " % len(feature_names))

        if create_n_grams :
            self.calculateNGramsEntropy(feature_names, SA_tagged_collection)
        else:
            # USING good n-grams generated beforehand
            feature_names = [line.rstrip('\n') for line in open('../Data/Lists/N_grams_final.csv', encoding='utf8')]

        term_freqs = X.sum(axis=0).A1
        self.n_gram_count_matrix = dict(zip(feature_names, term_freqs))

        # for w in self.n_gram_count_matrix.copy():
        #   #keep only those n-grams that have a frequency > 5
        #     if(self.n_gram_count_matrix.get(w) < 5):
        #         self.n_gram_count_matrix.pop(w)

        #
        # # Normalize by dividing by log of number of occurences on n-gram in collection
        # for gram in entropy.copy():
        #     if gram in self.n_gram_count_matrix.keys():
        #         entropy[gram] = entropy[gram]/(math.log(self.n_gram_count_matrix.get(gram)))
        #     else:
        #         entropy.pop(gram)
        #

        # # n-gram entropies
        # with open('../Data/test/entropy.csv', 'w+', newline='', encoding='utf8') as out_file:
        #     writer = csv.writer(out_file)
        #     for w in sorted(entropy, key=entropy.get):
        #         writer.writerow([w, entropy.get(w)])

        return self.tweetFeatures()

    def speechActTagTweet(self,tweet):
        tokens = self.tokenize(tweet)
        pos_list = nltk.pos_tag(tokens)
        tag = ''
        for entry in pos_list:
            if entry[1] in self.verb_tags:  # check if the tokens that are verbs are speech act verbs
                verb = self.stemmer.stem(entry[0].lower())
                if verb in self.speech_act_verbs:
                    tag = verb
                    break
        return tag

    def speechActTagCollection(self, collection):
        self.speech_act_tags = {}
        for tweet in collection:
            self.speech_act_tags[tweet] = (collection.get(tweet), self.speechActTagTweet(collection.get(tweet)))
        return self.speech_act_tags

    def enrichTweets(self, tweet, urls):
        """
        Features added to a row of data.
        :return: Returns a row with features added
        """
        row=[]
        basic_row = []
        binary_ngrams_row= []
        #tokenize tweet
        tokens = self.tokenize(tweet)
        # remove stop words
        tokens = self.removeStopWords(tokens)
        # add  Vulgar words feature
        row.append(self.hasVulgarWords(tokens))
        binary_ngrams_row.append(self.hasVulgarWords(tokens))
        basic_row.append(self.hasVulgarWords(tokens))
        # add Emoticons feature
        row.append(self.hasEmoticons(tokens))
        basic_row.append(self.hasEmoticons(tokens))
        binary_ngrams_row.append(self.hasEmoticons(tokens))
        # add Interrogation feature
        row.append(self.isInterrogative(tokens))
        binary_ngrams_row.append(self.isInterrogative(tokens))
        basic_row.append(self.isInterrogative(tokens))
        # add Exclamation feature
        row.append((self.isExclamatory(tokens)))
        binary_ngrams_row.append((self.isExclamatory(tokens)))
        basic_row.append((self.isExclamatory(tokens)))
        # add Abbreviations feature
        row.append(self.hasAbbreviations(tokens))
        basic_row.append(self.hasAbbreviations(tokens))
        binary_ngrams_row.append(self.hasAbbreviations(tokens))
        # add twitter jargons feature
        row.append(self.hasTwitterJargons(tokens))
        basic_row.append(self.hasTwitterJargons(tokens))
        binary_ngrams_row.append(self.hasTwitterJargons(tokens))
        # add Twiiter specific characters' features- presence and position
        presence, position = self.hasHash(tokens)
        row.append(presence)
        row.append(position)

        basic_row.append(presence)
        basic_row.append(position)

        binary_ngrams_row.append(presence)
        binary_ngrams_row.append(position)

        presence, position = self.hasATag(tokens)
        row.append(presence)
        row.append(position)

        basic_row.append(presence)
        basic_row.append(position)

        binary_ngrams_row.append(presence)
        binary_ngrams_row.append(position)

        presence, position = self.hasRT(tokens)
        row.append(presence)
        row.append(position)

        basic_row.append(presence)
        basic_row.append(position)

        binary_ngrams_row.append(presence)
        binary_ngrams_row.append(position)

        # add URL presence feature
        row.append(self.hasALink(tokens))
        basic_row.append(self.hasALink(tokens))
        binary_ngrams_row.append(self.hasALink(tokens))
        #add speech act verbs features
        sac_feature_dict = self.hasSpeechActVerbs(tokens)
        for verb in self.speech_act_verbs:
              row.append(sac_feature_dict.get(verb))
              basic_row.append(sac_feature_dict.get(verb))
              binary_ngrams_row.append(sac_feature_dict.get(verb))

        #more than 5 numbers in tweet
        row.append(self.hasManyNumbers(tokens))
        basic_row.append(self.hasManyNumbers(tokens))
        binary_ngrams_row.append(self.hasManyNumbers(tokens))

        #presence of non-ascii characters
        row.append(self.hasManyNonAscii(tokens))
        basic_row.append(self.hasManyNonAscii(tokens))
        binary_ngrams_row.append(self.hasManyNonAscii(tokens))

        #presence of links to trusted domains
        row.append(self.hasLinksToReputedDomains(urls))
        binary_ngrams_row.append(self.hasLinksToReputedDomains(urls))
        basic_row.append(self.hasLinksToReputedDomains(urls))

        # add positive and negative opinion feature
        pos_count, pos_presence = self.hasNegativeOpinions(tokens)
        neg_count, neg_presence = self.hasPositiveOpinions(tokens)

        # counts
        row.append(pos_count)
        row.append(neg_count)

        # boolean- positive/negative opinion feature
        basic_row.append(pos_presence)
        basic_row.append(neg_presence)

        binary_ngrams_row.append(pos_presence)
        binary_ngrams_row.append(neg_presence)

        #sentiment feature
        pos_per, neg_per = self.sentiment(tweet)
        tweet_sentiment = 1
        if pos_per > neg_per:
            tweet_sentiment = 1
        elif pos_per < neg_per:
            tweet_sentiment = 0

        row.append(tweet_sentiment)
        basic_row.append(tweet_sentiment)

        # n grams from tweet
        n_gram_feature_dict = self.collectNGramFeatures(tweet)
        for w in n_gram_feature_dict:
            row.append(n_gram_feature_dict.get(w))
            binary_ngrams_row.append(n_gram_feature_dict.get(w))

        return row, basic_row, binary_ngrams_row

    def enrichTweetsWithNgrams(self, tweet):
        '''
        Returns only n gram features for tweet
        '''
        row = []
        n_gram_feature_dict = self.collectNGramFeatures(tweet)
        for w in n_gram_feature_dict:
            row.append(n_gram_feature_dict.get(w))
        return row

    def hasPositiveEmoticons(self, tokens):
        '''
        Checks if abbreviations from online list present
        :return: 1 if present
        '''
        for w in tokens:
            if w in self.pos_emoticons_list:
                return 1
        return 0

    def hasNegativeEmoticons(self, tokens):
        '''
        Checks if abbreviations from online list present
        :return: 1 if present
        '''
        for w in tokens:
            if w in self.neg_emoticons_list:
                return 1
        return 0

    def sentiment(self, text):
        length = len(text)
        tokens = self.tokenize(text)

        tokens = [w.lower() for w in tokens if not w in set(stopwords.words('english') + self.web_abbreviations + list(string.punctuation))]
        tokens = list(set(tokens))
        positive_count, negative_count = self.simin_opinion_count(tokens)
        return positive_count, negative_count

    def simin_opinion_count(self, tokens):
        '''
                Checks if positive opnions present
                :return: 1 if positive opinions present. 0 otherwise
                '''
        pos_count = 0
        neg_count = 0

        for w in tokens:
            synonyms = []
            for syn in nltk.corpus.wordnet.synsets(w):
                for l in syn.lemmas():
                   synonyms.append(l.name())
            for syn in synonyms:
                if syn in self.positive_ops:
                    pos_count += 1
                    break
                elif syn in self.negative_ops:
                    neg_count += 1
                    break
            if w in self.neg_emoticons_list:
                neg_count += 1
            if w in self.pos_emoticons_list:
                pos_count += 1
        return pos_count, neg_count

    def hasFirstPersonPronouns(self, tokens):
        for w in tokens:
            if w in self.first_person_pronouns:
                return 1
        return 0


import contextlib
import io
import os
import pathlib
import tempfile
import warnings
import zipfile

import pytest
import tomli_w

from click.testing import CliRunner

"""

In your tests:

  def test_foo(cli_runner):
    r = cli_runner.invoke(mycli, ["mycommand"])
    assert r.exit_code == 0

In `some_command()`, add:

  @cli.command()
  def mycommand():
    import pytest; pytest.set_trace()

Then run via:

  $ pytest -sv --pdb-trace ...

Note any tests checking CliRunner stdout/stderr values will fail when
--pdb-trace is set.

"""

def pytest_addoption(parser) -> None:
    parser.addoption(
        "--pdb-trace",
        action="store_true",
        default=False,
        help="Allow calling pytest.set_trace() in Click's CliRunner",
    )

class MyCliRunner(CliRunner):
    def __init__(self, *args, in_pdb=False, **kwargs) -> None:
        self._in_pdb = in_pdb
        super().__init__(*args, **kwargs)

    def invoke(self, cli, args=None, **kwargs):
        params = kwargs.copy()
        if self._in_pdb:
            params["catch_exceptions"] = False

        return super().invoke(cli, args=args, **params)

    def isolation(self, input=None, env=None, color=False):
        if self._in_pdb:
            if input or env or color:
                warnings.warn(
                    "CliRunner PDB un-isolation doesn't work if input/env/color are passed"
                )
            else:
                return self.isolation_pdb()

        return super().isolation(input=input, env=env, color=color)

    @contextlib.contextmanager
    def isolation_pdb(self):
        s = io.BytesIO(b"{stdout not captured because --pdb-trace}")
        yield (s, not self.mix_stderr and s)

@pytest.fixture
def cli_runner(request) -> MyCliRunner:
    """A wrapper round Click's test CliRunner to improve usefulness."""
    return MyCliRunner(
        # workaround Click's environment isolation so debugging works.
        in_pdb=request.config.getoption("--pdb-trace")
    )

@pytest.fixture
def datadir(request) -> pathlib.Path:
    """Returns the directory in which the test is sitting."""
    return pathlib.Path(request.module.__file__).parents[0] / "data"

def pytest_configure(config):
    """This function is run once for pytest setup."""
    config.addinivalue_line(
        "markers",
        "skip_if_rc_var_not_set(name): this mark skips the test if a certain "
        "~/.config/deepdraw.toml variable is not set",
    )

    config.addinivalue_line("markers", "slow: this mark indicates slow tests")

def pytest_runtest_setup(item):
    """This function is run for every test candidate in this directory.

    The test is run if this function returns ``None``.  To skip a test,
    call ``pytest.skip()``, specifying a reason.
    """
    from deepdraw.utils.rc import load_rc

    rc = load_rc()

    # iterates over all markers for the item being examined, get the first
    # argument and accumulate these names
    rc_names = [
        mark.args[0]
        for mark in item.iter_markers(name="skip_if_rc_var_not_set")
    ]

    # checks all names mentioned are set in ~/.config/deepdraw.toml, otherwise,
    # skip the test
    if rc_names:
        missing = [k for k in rc_names if rc.get(k) is None]
        if any(missing):
            pytest.skip(
                f"Test skipped because {', '.join(missing)} is **not** "
                f"set in ~/.config/deepdraw.toml"
            )

def rc_variable_set(name):
    from deepdraw.utils.rc import load_rc

    rc = load_rc()
    pytest.mark.skipif(
        name not in rc,
        reason=f"RC variable '{name}' is not set",
    )

@pytest.fixture(scope="session")
def temporary_basedir(tmp_path_factory):
    return tmp_path_factory.mktemp("test-cli")

def pytest_sessionstart(session: pytest.Session) -> None:
    """Presets the session start to ensure the STARE dataset is always
    available."""

    from deepdraw.utils.rc import load_rc

    rc = load_rc()

    database_dir = rc.get("datadir.stare")
    if database_dir is not None:
        # if the user downloaded it, use that copy
        return

    # else, we must extract the LFS component (we are likely on the CI)
    archive = (
        pathlib.Path(__file__).parents[0]
        / "data"
        / "deepdraw-ci-assets"
        / "test-database.zip"
    )
    assert archive.exists(), (
        f"Neither datadir.stare is set on the global configuration, "
        f"(typically ~/.config/deepdraw.toml), or it is possible to detect "
        f"the presence of {archive}' (did you git submodule init --update "
        f"this submodule?)"
    )

    stare_tempdir = tempfile.TemporaryDirectory()
    rc.setdefault("datadir.stare", stare_tempdir.name)

    with zipfile.ZipFile(archive) as zf:
        zf.extractall(stare_tempdir.name)

    config_filename = "deepdraw.toml"
    with open(os.path.join(stare_tempdir.name, config_filename), "wb") as f:
        tomli_w.dump(rc.data, f)
        f.flush()

    os.environ["XDG_CONFIG_HOME"] = stare_tempdir.name

    # stash the newly created temporary directory so we can erase it when the
    key = pytest.StashKey[tempfile.TemporaryDirectory]()
    session.stash[key] = stare_tempdir


#GUESS THE NUMBER
game = "start"
while str(game) != "exit" :

    count = 1
    guess = input("GUESS THE NUMBER THAT WILL BE UPCOMING IF YOU CAN : ")
    import random
    for x in range(1):
        r = random.randint(1,10)
        print(r)

    if 10 > int(guess)-int(r) >= 4:
        print("you guessed too high")
    elif (int(guess) - int(r)) <= 3 and (int(guess) - int(r)) != 0:
        print("you guessed very near")
    elif int(guess) == int(r):
        print("woah! you guessed it correct. you are a Genius.")

    game = input("Do you wish to exit the game, if yes type exit : ")
    count += 1

print("you guessed the number " + str(count) + " times")


from typing import Optional, List

class ListNode:
    def __init__(self, val=0, next=None):
        self.val = val
        self.next = next

class Solution:
    # Time Complexity: O(n)
    # Space Complexity: O(1)
    def partition(self, head: Optional[ListNode], x: int) -> Optional[ListNode]:
        left_head = ListNode(0)
        left_cursor = left_head

        right_head = ListNode(0)
        right_cursor = right_head

        while head:
            if head.val < x:
                left_cursor.next = head
                left_cursor = left_cursor.next
            else:
                right_cursor.next = head
                right_cursor = right_cursor.next

            head = head.next

        left_cursor.next = right_head.next
        right_cursor.next = None

        return left_head.next

def getVals(head: Optional[ListNode]) -> List[int]:
    res = []

    while head:
        res.append(head.val)
        head = head.next

    return res

if __name__ == '__main__':
    s = Solution()

    list = [ListNode(1), ListNode(4), ListNode(3), ListNode(2), ListNode(5), ListNode(2)]
    for i in range(0, len(list)-1):
        list[i].next = list[i+1]
    assert getVals(s.partition(list[0], 3)) == [1,2,2,4,3,5]

    list = [ListNode(2), ListNode(1)]
    for i in range(0, len(list)-1):
        list[i].next = list[i+1]
    assert getVals(s.partition(list[0], 2)) == [1,2]


import uuid
from enum import Enum

from sqlalchemy import Column, ForeignKey, Index, String
from sqlalchemy.dialects.postgresql import UUID, ENUM

from models.postgres import Base, SchemaInDb
from models.postgres.base import UUIDMixin, CreatedTimeMixin

class OAuthProviderEnum(str, Enum):
    google: str = "google"
    yandex: str = "yandex"

class UserOAuthAccount(Base, UUIDMixin, CreatedTimeMixin):
    __tablename__ = "user_oauth_account"
    __mapper_args__ = {"eager_defaults": True}

    user_id = Column(UUID(as_uuid=True), ForeignKey("user.id", ondelete="CASCADE"), nullable=False)
    oauth_provider_name = Column(ENUM(OAuthProviderEnum), nullable=False)
    oauth_account_id = Column(String(60), nullable=False)

    __table_args__ = (
        Index("ix_provider_account", "oauth_provider_name", "oauth_account_id", unique=True),
        Index("ix_user_provider", "user_id", "oauth_provider_name", unique=True),
    )

class UserOAuthAccountSchema(SchemaInDb):
    user_id: uuid.UUID
    oauth_provider_name: str | OAuthProviderEnum
    oauth_account_id: str


from flask import Flask, request, jsonify
from dotenv import load_dotenv
import subprocess
import os

import requests
import threading
import time
import json
import platform
import datetime

requests.packages.urllib3.disable_warnings()

load_dotenv()  # take environment variables from .env.
app = Flask(__name__)
fileDir = os.path.dirname(__file__)
servicesDir = os.path.join(fileDir, '../services')
print(f"servicesDir: {servicesDir}")

PORT = os.environ.get("SERVICE_FATHER_PORT", 16000)
EXPECTED_TOKEN_ID = os.environ.get("SERVICE_FATHER_TOKEN_ID", "")
REPORT_IP = os.environ.get("SERVICE_FATHER_MGR_REPORT_IP", None)
REPORT_INTERVAL_SECS = os.environ.get("SERVICE_FATHER_MGR_REPORT_INTERVAL", 2)
HOSTNAME = platform.node()
MAX_KAP_OFFSET = int(os.environ.get("MAX_SERVICES_KAP_OFFSET_SECONDS", 10))

REPORT_DOCTOR_IP = os.environ.get("DEXYNTH_DOCTOR_REPORT_IP", "dd.dexynth.com")
REPORT_DOCTOR_INTERVAL_SECS = int(os.environ.get("DEXYNTH_DOCTOR_REPORT_INTERVAL", 60))

SF_INSTANCE= os.environ.get("SF_INSTANCE", HOSTNAME)

serviceStatus = {}

def getErrorJson(service, action, completedProcess):
    # http 500: internal server error
    err = {'service': service,
           'action': action,
           'message': f"Error while running script '{action}.sh' for service '{service}'.",
           'stderr': completedProcess.stderr.decode('utf-8'),
           'stdout': completedProcess.stdout.decode('utf-8'),
           'returncode': completedProcess.returncode}

    return err

def getResponseJson(service, action, completedProcess):
    rsp = {'service': service,
           'action': action,
           'message': f"Action '{action}' performed for service '{service}'.",
           'stderr': completedProcess.stderr.decode('utf-8'),
           'stdout': completedProcess.stdout.decode('utf-8'),
           'returncode': completedProcess.returncode}
    return rsp

def enableService(service_name):
    with open(f'{servicesDir}/{service_name}/enabled', 'w') as f:
        f.write('')
    return

def disableService(service_name):
    if os.path.isfile(f'{servicesDir}/{service_name}/enabled'):
        os.remove(f'{servicesDir}/{service_name}/enabled')
    return

def getPostResponse(json):
    # print(f"Performing post request: {json}")
    data = json
    service_name = data.get('serviceName')
    action = data.get('action')
    token_id = data.get('token', None)
    action = action.lower()

    if not service_name or not action or not token_id:
        # http 400: bad request
        return {'returncode': 400, 'message': 'Service name, action, and tokenId are required.'}, 400

    if token_id != EXPECTED_TOKEN_ID:
        # http 401: unauthorized
        return {'returncode': 403, 'message': 'Incorrect tokenId. Access denied.'}, 401

    if action == 'enable':
        # print(f"Enabling service '{service_name}'")
        enableService(service_name)
        return {'returncode': 200, 'message': f"Service '{service_name}' enabled."}, 200
    elif action == 'disable':
        disableService(service_name)
        return {'returncode': 200, 'message': f"Service '{service_name}' disabled."}, 200

    script_path = f'{servicesDir}/{service_name}/{action}.sh'
    if not os.path.isfile(script_path):
        return {'returncode': 404, 'message': f"Script '{action}.sh' not found for service '{service_name}'."}, 404

    isEnabled = os.path.isfile(f'{servicesDir}/{service_name}/enabled')
    if not isEnabled:
        return {'returncode': 500, 'message': f"Service '{service_name}' is disabled."}, 500

    # print(f"Running script '{action}.sh' for service '{service_name}'")
    completedProcess = subprocess.run(
        ['bash', script_path], capture_output=True)
    if completedProcess.returncode != 0:
        # http 500: internal server error
        return getErrorJson(service_name, action, completedProcess), 500
    # http 200: okq
    return getResponseJson(service_name, action, completedProcess), 200

@app.route('/api', methods=['POST'])
def perform_post():
    json, http_code = getPostResponse(request.json)
    print("forcing reportStatus")
    reportStatus()
    return jsonify(json), http_code

@app.route('/api/kap', methods=['POST'])
def perform_kap():
    data = request.json
    serviceName = data.get('serviceName')
    status = data.get('status','')
    kapTime = datetime.datetime.now()
    print(f"KAP: {serviceName} {status} {kapTime}")
    serviceStatus[serviceName] = {'status': status, 'time': kapTime}
    return jsonify({'returncode': 200, 'message': f"KAP: {serviceName} {status} {kapTime}"}), 200

def calculateKapStatus():
    for serviceName in serviceStatus:
        kapTime = serviceStatus[serviceName]['time']
        now = datetime.datetime.now()
        delta = now - kapTime
        serviceStatus[serviceName]['kapDelta'] = delta.seconds
        serviceStatus[serviceName]['kapOk'] = delta.seconds <= MAX_KAP_OFFSET

@app.route('/api/kap', methods=['GET'])
def get_kap():
    serviceName = request.args.get("serviceName", None)
    calculateKapStatus()
    if serviceName is None:
        return jsonify(serviceStatus), 200
    else:
        print(f"serviceName: {serviceName}")
        if serviceName not in serviceStatus:
            return jsonify({'kapOk': False, 'status': "unknown", "time" : "unknown"}), 200
        return jsonify(serviceStatus[serviceName]), 200

# https://localhost:26000/api/services
# https://localhost:26000/api/services?kind=enabled
# https://localhost:26000/api/services?kind=disabled

def getServices(kind='all'):
    # print(f"Getting services of kind '{kind}'")
    services = os.listdir(servicesDir)
    outServices = []
    for service in services:
        enabled = False
        if os.path.isfile(f'{servicesDir}/{service}/enabled'):
            enabled = True
        outServices.append({"service": service, "enabled": enabled})

    if (kind == "enabled"):
        outServices = list(
            filter(lambda service: service["enabled"], outServices))
    elif (kind == "disabled"):
        outServices = list(
            filter(lambda service: not service["enabled"], outServices))
    return outServices

@ app.route('/api/services', methods=['GET'])
def get_services():
    kind = request.args.get("kind", "all")
    outServices = getServices(kind)
    return jsonify(outServices), 200

def reportStatus():
    if REPORT_IP is None:
        return
    # url = f'http://{REPORT_IP}/get_my_ip'
    # ip = requests.get(url).json()['ip']
    # print(f"Reporting status to {REPORT_IP} from {ip}", flush=True)
    allServices = getServices()
    globalStatus = []
    for service in allServices:
        serviceName = service['service']
        enabled = service['enabled']
        statusRequest = {"serviceName": serviceName,
                         "action": "status",
                         "token": EXPECTED_TOKEN_ID}
        statusRequest, code = getPostResponse(statusRequest)
        # print(f"service: {service}")
        # print(f"statusRequest: {statusRequest}")
        srvStatus = {
            'hostname': HOSTNAME,
            'port': PORT,
            'service': serviceName,
            'enabled': enabled,
            'statusMessage': statusRequest['message'],
            'returncode': statusRequest['returncode']
        }
        globalStatus.append(srvStatus)

    statusJson = json.dumps(globalStatus)

    # print(f"globalStatus: {statusJson}")
    try:
        response = requests.post(f'http://{REPORT_IP}/status',
                                 json=globalStatus, verify=False, timeout=1)
        # print(f"response: {response}")
    except Exception as e:
        print(f"Exception: {e}")
        print(f"Failed to report status to {REPORT_IP}", flush=True)
        return

"""
REPORT_DOCTOR_IP = os.environ.get("DEXYNTH_DOCTOR_REPORT_IP", None)
REPORT_DOCTOR_INTERVAL_SECS = os.environ.get("DEXYNTH_DOCTOR_REPORT_INTERVAL", 2)

SF_INSTANCE= os.environ.get("SF_INSTANCE", platform.node())
"""
def __reportStatusToDoctor():
    if REPORT_DOCTOR_IP is None:
        return
    url = f'https://{REPORT_DOCTOR_IP}/heartbeat'
    serviceName = f"ServiceFather@{SF_INSTANCE}"
    data = {'service': serviceName}
    try:
        print(f"Reporting status to {REPORT_DOCTOR_IP} from {HOSTNAME}: {data}", flush=True)
        response = requests.post(url, json=data, verify=False, timeout=1)
        print(f"response: {response}", flush=True)
    except Exception as e:
        print(f"Exception: {e}")
        print(f"Failed to report status to {REPORT_DOCTOR_IP}", flush=True)
        return

def startReportStatusThread():
    def reportStatusToDoctorThread():
        while True:
            print(f"  >> reportStatusToDoctorThread", flush=True)
            __reportStatusToDoctor()
            time.sleep(REPORT_DOCTOR_INTERVAL_SECS)

    def reportStatusThread():
        while True:
            reportStatus()
            time.sleep(REPORT_INTERVAL_SECS)

    if REPORT_IP is not None:
        x = threading.Thread(target=reportStatusThread, args=())
        x.start()

    if REPORT_DOCTOR_IP is not None:
        print(f"Starting reportStatusToDoctorThread")
        x = threading.Thread(target=reportStatusToDoctorThread, args=())
        x.start()

if __name__ == '__main__':
    startReportStatusThread()
    app.run(debug=True, ssl_context='adhoc', host='0.0.0.0', port=PORT)


from LineLogParser import LineLogParser

class FileLogParser:
    def __init__(self):
        self.parser = LineLogParser()

    def parseFile(self, filename):
        lines = open(filename, "r")
        parsedLogs = []

        for line in lines:
            parsedLogs.append(self.parser.parseLine(line))

        return parsedLogs


from typing import Any, Callable, Mapping, Sequence, Tuple, Type

from pydantic import BaseModel
from pydantic.fields import FieldInfo

from . import utils
from .validation import create_func_model

__all__: Sequence[str] = (
    "create_model_cls",
    "create_model",
    "bind_arguments",
)

def create_model_cls(
    func: Callable, fields: Mapping[str, Tuple[Any, FieldInfo]]
) -> Type[BaseModel]:
    class Config:
        allow_population_by_field_name: bool = True
        arbitrary_types_allowed: bool = True

    return create_func_model(func, fields, config=Config)

def create_model(
    func: Callable,
    fields: Mapping[str, Tuple[Any, FieldInfo]],
    arguments: Mapping[str, Any],
) -> BaseModel:
    model_cls: Type[BaseModel] = create_model_cls(func, fields)

    return model_cls(**arguments)

def bind_arguments(
    func: Callable, args: Tuple[Any, ...], kwargs: Mapping[str, Any]
) -> Mapping[str, Any]:
    arguments: Mapping[str, Any] = utils.bind_arguments(func, args, kwargs)

    return {
        key: value
        for key, value in arguments.items()
        if not isinstance(value, FieldInfo)
    }


import telebot

bot = telebot.TeleBot("5959592035:AAHFfTbtlGp6v59F4xK9DpO70O3eE2fAiZg", parse_mode=None)
@bot.message_handler(commands=['start', 'help'])
def send_welcome(message):
	bot.reply_to(message, "treasure, how was your day?")

bot.infinity_polling()


import tweepy
from tweepy import Client
from decouple import config

api_key = config('api_key')
api_key_secret= config('api_key_secret')
access_token = config('access_token')
access_token_secret = config('access_token_secret')

auth = tweepy.OAuthHandler(api_key, api_key_secret)
auth.set_access_token(access_token, access_token_secret)

api = tweepy.API(auth)

def extract_tweets(keyword, date_since, date_until, num_tweets):
    tweets = tweepy.Cursor(api.search_tweets, q=keyword, lang="en", since_id=date_since, until=date_until, tweet_mode='extended').items(num_tweets)
    tweet_cont, tweet_rt, tweet_fav = [], [], []
    for tweet in tweets:
        try:
            tweet_cont.append(tweet.retweeted_status.full_text)
            tweet_rt.append(tweet.retweeted_status.retweet_count)
            tweet_fav.append(tweet.retweeted_status.favorite_count)
        except AttributeError:
            tweet_fav.append(tweet.favorite_count)

    data = {
        'tweet_content': tweet_cont,
        'retweet_count': tweet_rt,
        'favorite_count': tweet_fav
    }
    return data

if __name__ == '__main__':
    keyword = 'bitcoin'
    date_since = '2020-01-01'
    date_until = '2020-01-02'
    num_tweets = 100
    data = extract_tweets(keyword, date_since, date_until, num_tweets)
    print(data)


#EXERCICIO10
qtdCigarros = int(input("Qtd cigarros por dia: "))
tmpFumante = int(input("Qtd de anos que fuma: "))

#total de cigarros por ano
qtdTotalCigarros = qtdCigarros*365*tmpFumante

tmpVida = ((qtdTotalCigarros*10)/24)/60
print('Tempo de vida perdido: ', tmpVida)


import doctest as dt

def btod(bin_str):
    assert isinstance(bin_str, str)
    print("Your string is " + bin_str)
    return None

#btod("1100101")
#btod(1100101)

def btod(bin_str):
    assert isinstance(bin_str, str)
    for bit in bin_str:
        assert bit in '01', 'A binary number is needed.'
    print("Your string is " + bin_str)
    return None

#btod('1100101')
#btod('123')

def foo(x, y):
    assert isinstance(x, int), "An integer is needed"
    assert isinstance(y, int), "An integer is needed"
    assert y > 0, "Can't divide by zero"

    return x // y

#print(foo(8, 4))
#print(foo(8.0, 4.0))
#print(foo(8,0))

def double_your_num(number):
    assert isinstance(number, int), 'An integer is needed'
    assert (number >= 1) and (number <= 10), 'Number out of bounds'
    print(number * 2)
    return None

#double_your_num(3)
#double_your_num(3.5)

def incr_li(li, n):
    '''(list of ints, int) -> None

    Increment the first n values in
    a list by 1 and print the new list.
    >>> incr_li([1, 2, 3, 4], 2)
    [2, 3, 3, 4]
    >>> incr_li([1, 2, 3, 4], 5)
    [2, 3, 4, 5]
    '''
    #newli = li[:] # slicing the entire list creates a copy
    #for i in range(n):
    #    newli[i] += 1
    #print(newli)
    #return None

    try:
        newli = li[:]
        for i in range(n):
            newli[i] += 1
        print(newli)
    except IndexError:
        newli1 = li[:]
        for i in range(len(newli1)):
            newli1[i] += 1
        print(newli1)

    return None

def catch_zero_divition(x, y):
    '''
    (number, number) -> number

    Divides x by y

    >>> catch_zero_divition(2, 0)
    Denomenator cannot be zero
    >>> catch_zero_divition(2, 4)
    0.5
    '''

    try:
        output = x / y
        print(output)
    except ZeroDivisionError:
        print("Denomenator cannot be zero")

print(dt.testmod())


from ..responses import ResponseMsg, ResponseCQFunc
from ..permissions import get_permissions
from .general import Session

class CQCommandSession(Session):
    def __init__(self, user_id):
        Session.__init__(self, user_id=user_id)
        self._max_delta = 30
        self.session_type = 'CQ控制台'
        self.strict_commands = ['command', 'cmd', 'console', '控制台', '命令']
        self.description = '使用onebot标准（github.com/botuniverse/onebot）的API控制机器人'
        self.permissions = get_permissions().get('super', {})
        self.is_first_time = True
        self.func_name = ''
        self.kwargs = {}

    def handle(self, request):
        if self.is_first_time:
            self.is_first_time = False
            return ResponseMsg('【%s】请输入函数名' % self.session_type)
        elif self.func_name == '':
            self.func_name = request.msg
            return ResponseMsg('【%s】请输入参数（字典形式）' % self.session_type)
        elif self.kwargs == {}:
            self.kwargs = eval(request.msg)
            self.deactivate()
            return ResponseCQFunc(func_name=self.func_name, kwargs=self.kwargs)

    def process_output(self, output):
        return ResponseMsg('【%s/Output】%s' % (self.session_type, str(output)))

class CQRebootSession(Session):
    def __init__(self, user_id):
        Session.__init__(self, user_id=user_id)
        self._max_delta = 3
        self.session_type = 'CQ重启程序'
        self.strict_commands = ['restart', 'reboot', '重启']
        self.permissions = get_permissions().get('super', {})

    def handle(self, request):
        self.deactivate()
        return ResponseCQFunc(func_name='set_restart', kwargs={})

    def process_output(self, output):
        return ResponseMsg('【%s/Output】%s' % (self.session_type, str(output)))