import torch
import numpy as np
import gymnasium as gym
from gymnasium import spaces
from torch.autograd import Variable
import torch.nn.functional as F
from collections import namedtuple
import memory as mem
from feedforward import Feedforward

torch.set_num_threads(1)

class UnsupportedSpace(Exception):
    """Exception raised when the Sensor or Action space are not compatible

    Attributes:
        message -- explanation of the error
    """
    def __init__(self, message="Unsupported Space"):
        self.message = message
        super().__init__(self.message)

class QFunction(Feedforward):
    def __init__(self, observation_dim, action_dim, upper_bound, hidden_sizes,
                 learning_rate, output_size, activation_fun=torch.nn.LeakyReLU()):
        super().__init__(input_size=observation_dim + action_dim, hidden_sizes=hidden_sizes,
                         output_size=output_size, upper_bound=upper_bound, activation_fun=activation_fun)

    def Q_value(self, observations, actions, log=False):

        logits = self.forward(torch.hstack([observations,actions]), "critic")

        if log:
            return F.log_softmax(logits, dim=-1)
        else:
            return F.softmax(logits, dim=-1)

class OUNoise():
    def __init__(self, shape, theta: float = 0.15, dt: float = 1e-2):
        self._shape = shape
        self._theta = theta
        self._dt = dt
        self.noise_prev = np.zeros(self._shape)
        self.reset()

    def __call__(self) -> np.ndarray:
        noise = (
            self.noise_prev
            + self._theta * ( - self.noise_prev) * self._dt
            + np.sqrt(self._dt) * np.random.normal(size=self._shape)
        )
        self.noise_prev = noise
        return noise

    def reset(self) -> None:
        self.noise_prev = np.zeros(self._shape)

class D4PGAgent(object):
    """
    Agent implementing Q-learning with NN function approximation.
    """
    def __init__(self, observation_space, action_space, env_name, **userconfig):

        if not isinstance(observation_space, spaces.box.Box):
            raise UnsupportedSpace('Observation space {} incompatible ' \
                                   'with {}. (Require: Box)'.format(observation_space, self))
        if not isinstance(action_space, spaces.box.Box):
            raise UnsupportedSpace('Action space {} incompatible with {}.' \
                                   ' (Require Box)'.format(action_space, self))

        self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
        self.env_name = env_name

        self._config = {
            "tau": 0.005,  # for soft update of target parameters
            "eps": 0.1,    # strength of noise
            "eps_decay": 1,
            "eps_min": 0.01,
            "discount": 0.995,
            "rollout": 2,
            "buffer_size": int(1e6),
            "batch_size": 256,
            "lr_actor": 0.0001,
            "lr_critic": 0.0005,
            "weight_decay": 0,
            "gradient_clip": 0,
            "soft_updates_every": 1, # how often to copy weights over to target networks (Gradually)
            "hard_updates_every": 50, # how often to copy weights over to target networks (Instant)
            "hard_update": True, # Hard update OR Soft Update?
            "hidden_sizes_actor": [512, 256],
            "hidden_sizes_critic": [512, 256, 128],
            "use_target_net": True,
            "optimizer_eps": 0.000001,  # Optimizer epsilon: Term added to denominator for numerical stability
            "action_noise_theta": 0.15,  # Describes Ornstein-Uhlenbeck (OU) process noise
            "action_noise_sigma": 0.1, # Describes Ornstein-Uhlenbeck (OU) process noise
            "reward_norm": True,
            "batch_norm": True,
            "noise" : 0.2
        }
        self._observation_space = observation_space
        self._obs_dim = self._observation_space.shape[0]
        self._action_space = action_space
        if env_name == "Hockey":
            self._action_n = int(action_space.shape[0]/2)
        else:
            self._action_n = int(action_space.shape[0])
        self.action_noise = OUNoise((self._action_n))

        #self.buffer = mem.Memory(max_size=self._config["buffer_size"])
        self.buffer = mem.Memory(max_size=self._config["buffer_size"])
        self.replay_sample = namedtuple("ReplaySample", ["state", "action", "reward", "next_state", "done"])

        self.actor_loss = 0
        self.critic_loss = 0
        self.learn_step = 0
        self.episode = 0

        # D4PG variables
        # Lower and upper bounds of critic value output distribution, these will vary with environment
        self.vmin = -10.0  # Min value which atom's action-value function begins
        self.vmax = 10.0  # Max value which atom's action-value function begins
        self.num_atoms = 51  # Multiple atoms can represent the distribution of V
        self.delta = (self.vmax - self.vmin) / float((self.num_atoms - 1))
        self.atoms = torch.linspace(self.vmin, self.vmax, self.num_atoms)
        self.bin_centers = np.array([self.vmin + i * self.delta for i in range(self.num_atoms)]).reshape(-1, 1)

        self._config.update(userconfig)

        #Initialize ACTOR networks
        self.actor = Feedforward(input_size=self._obs_dim,
                                    hidden_sizes=self._config["hidden_sizes_actor"],
                                    output_size=self._action_n,
                                    upper_bound=None,
                                    activation_fun=torch.nn.LeakyReLU(),
                                    output_activation=torch.nn.Tanh(),
                                    batch_norm=self._config["batch_norm"]).to(self.device)

        self.actor_target = Feedforward(input_size=self._obs_dim,
                                    hidden_sizes=self._config["hidden_sizes_actor"],
                                    output_size=self._action_n,
                                    upper_bound=None,
                                    activation_fun=torch.nn.LeakyReLU(),
                                    output_activation=torch.nn.Tanh(),
                                    batch_norm=self._config["batch_norm"]).to(self.device)

        self.hard_update(self.actor, self.actor_target)

        #Initialize CRITIC networks
        self.critic = QFunction(observation_dim=self._obs_dim,
                            action_dim=self._action_n,
                            output_size=self.num_atoms,
                            upper_bound=None,
                            hidden_sizes=self._config["hidden_sizes_critic"],
                            learning_rate=self._config["lr_critic"]).to(self.device)

        # Target Q1 Network
        self.critic_target = QFunction(observation_dim=self._obs_dim,
                                   action_dim=self._action_n,
                                   output_size=self.num_atoms,
                                   upper_bound=None,
                                   hidden_sizes=self._config["hidden_sizes_critic"],
                                   learning_rate=0).to(self.device)

        self.hard_update(self.critic, self.critic_target)

        self.actor_optimizer = torch.optim.Adam(self.actor.parameters(),
                                        lr=self._config["lr_actor"],
                                        eps=self._config["optimizer_eps"],
                                        weight_decay=self._config["weight_decay"])

        self.critic_optimizer = torch.optim.Adam(self.critic.parameters(),
                                        lr=self._config["lr_critic"],
                                        eps=self._config["optimizer_eps"],
                                        weight_decay=self._config["weight_decay"])
        self.train_iter = 0

        print("D4PG Agent initialized")
        print("eps", self._config["eps"])
        print("lr_actor", self._config["lr_actor"])
        print("lr_critic", self._config["lr_critic"])
        print("batch_size", self._config["batch_size"])
        print("batch_norm", self._config["batch_norm"])
        print("reward_norm", self._config["reward_norm"])
        print("hard_update", self._config["hard_update"])
        print("hard_updates_every", self._config["hard_updates_every"])
        print("noise", self._config["noise"])

    def act(self, observation, evaluation=False):

        observation = torch.from_numpy(observation).float().to(self.device)

        if evaluation is False:

            if np.random.random() > self._config["eps"]:

                self.actor.eval()
                with torch.no_grad():
                    action = self.actor.predict(observation).detach().cpu().data.numpy()
                self.actor.train()

                action += self._gauss_noise(self._action_n)# if np.random.rand() < self._config["noise"] else 0
            else:
                if self.env_name == "Hockey":
                    action = self._action_space.sample()[:4]
                else:
                    action = self._action_space.sample()

        else:
            self.actor.eval()
            with torch.no_grad():
                action = self.actor.predict(observation).detach().cpu().data.numpy()

        return action.clip(-1, 1)

    @property
    def eps_(self):
        """
        Anneals the epsilon rate down to a specified minimum to ensure there is
        always some randomness to the actions performed by the agent
        """

        self._config['eps'] = max(self._config["eps_min"], self._config['eps'] * self._config['eps_decay'])
        return self._config['eps']

    def store_transition(self, transition):
        self.buffer.add_transition(transition)

    def state(self):
        return (self.actor.state_dict(), self.actor_optimizer.state_dict(),
                self.critic.state_dict(), self.critic_optimizer.state_dict(), self._config)

    def restore_state(self, state):
        self.actor.load_state_dict(state[0])
        self.actor_optimizer.load_state_dict(state[1])
        self.critic.load_state_dict(state[2])
        self.critic_optimizer.load_state_dict(state[3])
        self.hard_update(self.actor, self.actor_target)
        self.hard_update(self.critic, self.critic_target)

    def reset(self):
        self.action_noise.reset()

    def train_iteration(self, num_iterations=32):
        # Convert a NumPy array to a Torch tensor with float32 data type
        to_torch = lambda x: torch.from_numpy(x.astype(np.float32))

        # Increment episode count and set epsilon
        self.episode += 1
        self._config["eps"] = self.eps_

        for i in range(num_iterations):
            # Sample data from the replay buffer
            #sampled_data, weights, tree_idx_lst = self.buffer.sample(batch_size=self._config['batch_size'])
            sampled_data = self.buffer.sample(batch_size=self._config['batch_size'])
            states = to_torch(np.stack(sampled_data[:, 0]))  # Current state
            actions = to_torch(np.stack(sampled_data[:, 1])[:, None]).squeeze(1)  # Chosen actions
            rewards = to_torch(np.stack(sampled_data[:, 2])[:, None])  # Rewards (batchsize,1)
            next_states = to_torch(np.stack(sampled_data[:, 3]))  # Next states
            dones = to_torch(np.stack(sampled_data[:, 4])[:, None])  # Done signals (batchsize,1)

            # Normalize rewards if reward normalization is enabled
            if self._config["reward_norm"]:
                rewards = (rewards - rewards.mean()) / (rewards.std() + 1e-8)

            atoms = self.atoms.unsqueeze(0)

            # Calculate projected target probabilities using target networks
            actions_target = self.actor_target.predict(next_states)
            probs_target = self.critic_target.Q_value(next_states, actions_target)
            projected_target_probs = self.categorical_projection(rewards, probs_target, dones, states, actions)

            # Calculate log probability distribution using current critic network
            log_probs = self.critic.Q_value(states, actions, log=True)

            # Calculate critic loss (cross-entropy)
            critic_loss = -(projected_target_probs * log_probs).sum(-1).mean()
            #critic_loss = self.huber_loss(projected_target_probs, log_probs).sum(-1).mean()
            #print("critic_loss", critic_loss, critic_loss.shape)

            # Predict action for actor network loss calculation
            predicted_action = self.actor.forward(states, "actor")

            # Predict value distribution using current critic network and predicted action
            probs = self.critic.Q_value(states, predicted_action)

            # Calculate expected reward by summing probabilities and atom values
            expected_reward = (probs * atoms).sum(-1)

            # Calculate actor loss (policy gradient)
            actor_loss = -torch.mean(expected_reward)

            # Perform gradient ascent for actor
            self.actor.train()
            self.actor_optimizer.zero_grad()
            actor_loss.backward()
            if self._config["gradient_clip"] != 0:
                torch.nn.utils.clip_grad_norm_(self.actor.parameters(), self._config["gradient_clip"])
            self.actor_optimizer.step()

            # Perform gradient descent for critic
            self.critic.train()
            self.critic_optimizer.zero_grad()
            critic_loss.backward()
            if self._config["gradient_clip"] != 0:
                torch.nn.utils.clip_grad_norm_(self.critic.parameters(), self._config["gradient_clip"])
            self.critic_optimizer.step()

            # Update target networks (hard or soft)
            self.update_target_networks()

            # Store losses for visualization
            self.actor_loss = actor_loss.item()
            self.critic_loss = critic_loss.item()

            return self.critic_loss, self.actor_loss #, weights, tree_idx_lst

    def categorical_projection(self, rewards, probs_target, dones, states, actions):
        """
        Returns the projected value distribution for the input state/action pair
        """

        vmin = self.vmin
        vmax = self.vmax
        num_atoms = self.num_atoms
        delta_z = (vmax - vmin) / (num_atoms - 1)
        atoms = self.atoms
        gamma = self._config['discount']
        rollout = self._config["rollout"]  # N-Step bootstrapping for Temporal Difference Update Calculations

        projected_atoms = rewards + gamma ** rollout * atoms.unsqueeze(0) + (1 - dones)
        projected_atoms.clamp_(vmin, vmax)

        b = (projected_atoms - vmin) / delta_z

        lower_bound = b.floor()
        upper_bound = b.ceil()

        m_lower = (upper_bound + (lower_bound == upper_bound).float() - b) * probs_target
        m_upper = (b - lower_bound.float()) * probs_target

        projected_probs = torch.tensor(np.zeros(probs_target.size())).to(self.device)

        for idx in range(probs_target.size(0)):
            projected_probs[idx].index_add_(0, lower_bound[idx].double(), m_lower[idx].double())
            projected_probs[idx].index_add_(0, upper_bound[idx].double(), m_upper[idx].double())

        return projected_probs.float()

    def _gauss_noise(self, shape):
        """
        Returns the epsilon scaled noise distribution for adding to Actor
        calculated action policy.
        """

        n = np.random.normal(0, self._config["noise"], shape)
        return n

    def hard_update(self, local_model, target_model):
        """
        Fully copy parameters from active network to target network. To be used
        in conjunction with a parameter "C" that modulated how many timesteps
        between these hard updates.
        """
        target_model.load_state_dict(local_model.state_dict())

    def soft_update(self, local_model, target_model, tau):
        """Soft update model parameters.
        θ_target = τ*θ_local + (1 - τ)*θ_target

        Params
        ======
            local_model: PyTorch model (weights will be copied from)
            target_model: PyTorch model (weights will be copied to)
            tau (float): interpolation parameter
        """
        for target_param, local_param in zip(target_model.parameters(), local_model.parameters()):
            target_param.data.copy_(tau * local_param.data + (1.0 - tau) * target_param.data)

    def to_tensor(self, x, requires_grad=True, dtype=torch.FloatTensor):
        x = torch.from_numpy(x.astype(np.float32))
        x = Variable(x, requires_grad=requires_grad).type(dtype)
        return x

    def copy_gradients(self, model_local, model_global ):
        for param_local, param_global in zip(model_local.parameters(), model_global.parameters()):
            if param_global.grad is not None:
                return
            param_global._grad = param_local.grad

    def sample_from_buffer(self, batch_size):
        return [self.replay_sample(*sample) for sample in self.buffer.sample(batch_size=batch_size)]

    # Huber loss function
    def huber_loss(self, y_pred, y, delta=1.0):
        huber_mse = 0.5 * (y - y_pred) ** 2
        huber_mae = delta * (torch.abs(y - y_pred) - 0.5 * delta)
        return torch.where(torch.abs(y - y_pred) <= delta, huber_mse, huber_mae)


import json
import os

from cbc_api.schedule.tsp import TSP
from cbc_api.schedule.schedule import Schedule
from cbc_api.schedule.capacity_tester import CapacityTester

PATH = os.path.dirname(os.path.realpath(__file__))

def test_google_tsp():
    tsp = TSP()
    soln = tsp.solve_tsp(
        origin=[28.50,-81.25],
        destination=[28.50,-81.25],
        waypoints=[
            [28.53874, -81.37861],
            [28.54052, -81.38118],
            [28.56707, -81.38971],
            [28.4497135606058, -81.4707795871383],
            [28.5414049683169, -81.37367],
            [28.404738, -81.428915]
        ],
        method='google'
    )
    assert 'durations' in soln
    for duration in soln['durations']:
        assert type(duration) in [int,float]
    assert 'order' in soln
    for order in soln['order']:
        assert type(order) == int

def test_pyschedule_tsp():
    tsp = TSP()
    soln = tsp.solve_tsp(
        origin=[28.50,-81.25],
        destination=[28.50,-81.25],
        waypoints=[
            [28.53874, -81.37861],
            [28.54052, -81.38118],
            [28.56707, -81.38971],
            [28.4497135606058, -81.4707795871383],
            [28.5414049683169, -81.37367],
            [28.404738, -81.428915]
        ],
        method='pyschedule'
    )
    assert 'durations' in soln
    for duration in soln['durations']:
        assert type(duration) in [int,float]
    assert 'order' in soln
    for order in soln['order']:
        assert type(order) == int

def test_pyschedule_constrained_tsp():
    tsp = TSP()
    soln = tsp.solve_tsp(
        origin=[28.50,-81.25],
        destination=[28.50,-81.25],
        waypoints=[
            [28.53874, -81.37861],
            [28.54052, -81.38118],
            [28.56707, -81.38971],
            [28.4497135606058, -81.4707795871383],
            [28.5414049683169, -81.37367],
            [28.404738, -81.428915]
        ],
        constraints=[
            {
                'index1' : '4',
                'constraint_type' : 'before',
                'index2' : '2'
            },
            {
                'index1': '0',
                'constraint_type' : 'after',
                'index2' : '2'
            }
        ],
        method='pyschedule'
    )
    assert 'durations' in soln
    for duration in soln['durations']:
        assert type(duration) in [int,float]
    assert 'order' in soln
    for order in soln['order']:
        assert type(order) == int

def test_schedule():
    path = '/'.join(PATH.split('/')[:-1])
    filename = path + '/cbc_api/data/schedule_test.json'
    with open(filename, 'r') as f:
        configs = json.load(f)

    schedule = Schedule(configs)
    schedule.schedule_activities()
    assert 'activities' in schedule.configs
    assert 'distance_matrix' in schedule.configs
    for activity in schedule.configs['activities']:
        assert 'assigned_day' in activity
        assert 'cluster_coords' in activity
        assert 'coordinates' in activity
        assert 'day' in activity
        assert 'duration' in activity
        assert 'from_duration' in activity
        assert 'index' in activity
        assert 'label' in activity
        assert 'order' in activity
        assert 'to_duration' in activity

def test_capacity_tester():
    capacities = [120, 120, 240, 480, 480]
    durations = [87, 101, 132, 139, 161]
    capacity_tester = CapacityTester(capacities, durations)
    extra_days = capacity_tester.find_additional_capacity()
    assert extra_days == 3

    capacities = [120, 240]
    durations = [139, 161]
    capacity_tester = CapacityTester(capacities, durations)
    extra_days = capacity_tester.find_additional_capacity()
    assert extra_days == 0


import duckdb
import logging

from django.conf import settings
from tally_ho.apps.tally.management.commands.utils import (
    build_generic_model_key_values_from_duckdb_row_tuple_data,
    check_for_missing_columns,
    get_ballot_by_ballot_number,
    get_electrol_race_by_ballot_name,
)
from tally_ho.apps.tally.models.ballot import Ballot
from tally_ho.apps.tally.models.candidate import Candidate
from tally_ho.apps.tally.models.electrol_race import ElectrolRace
from tally_ho.apps.tally.models.tally import Tally
from tally_ho.libs.utils.query_set_helpers import BulkCreateManager
from tally_ho.celeryapp import app
from tally_ho.libs.utils.memcache import MemCache

logger = logging.getLogger(__name__)

def create_candidates_from_candidates_file_data(
        duckdb_candidates_data=None,
        tally=None,
        ballots_by_ballot_number=None,
        electrol_races_by_ballot_name=None,
        candidate_id_by_ballot_order_dict=None,
        step_number=None,
        step_name=None,
        command=None,
):
    """Create candidates from candidates file data inside duckdb.

    :param duckdb_candidates_data: candidates file data in duckdb format.
    :param tally: candidates tally.
    :param ballots_by_ballot_number: ballots by ballot name dict.
    :param electrol_races_by_ballot_name: electrol races by ballot name dict.
    :param candidate_id_by_ballot_order_dict: candidate id by ballot order
        dict.
    :param step_number: Step number
    :param step_name: Step name
    :param command: stdout command.
    :returns: None"""
    try:
        instances_count_memcache_key =\
            f"{tally.id}_{step_name}_{step_number}"
        # reset instances count in memcache if exists already
        client = MemCache()
        client.delete(instances_count_memcache_key)
        candidate_foreign_key_fields =\
                    ['ballot', 'electrol_race']
        col_names_to_model_field_map =\
            getattr(settings,
                    'CANDIDATE_FILE_COLS_NAMES_TO_CANDIDATE_MODEL_FIELDS')
        candidates_cols_names_list =\
            list(col_names_to_model_field_map.keys())
        candidates_data =\
                    duckdb_candidates_data.project(
                    ','.join(
            candidates_cols_names_list)).distinct().fetchall()
        bulk_mgr = BulkCreateManager(
            chunk_size=1000,
            cache_instances_count=True,
            cache_key=instances_count_memcache_key,
            memcache_client=client,
        )

        for candidate_vals_tuple in candidates_data:
            kwargs =\
                build_generic_model_key_values_from_duckdb_row_tuple_data(
                    candidate_vals_tuple,
                    col_names_to_model_field_map,
                    candidates_cols_names_list,
                )
            candidate_data_by_index =\
                { item[0]: item[1] for item in enumerate(
                list(candidate_vals_tuple))}
            for index, col_name in enumerate(candidates_cols_names_list):
                field_name = col_names_to_model_field_map.get(col_name)
                # Continue if field is not a foreign key field
                if field_name not in candidate_foreign_key_fields:
                    continue

                field_val = candidate_data_by_index.get(index)
                if field_name == 'ballot':
                    ballot = get_ballot_by_ballot_number(
                        field_val,
                        ballots_by_ballot_number
                    )
                    kwargs['ballot'] = ballot
                    continue

                if field_name == 'electrol_race':
                    electrol_race = get_electrol_race_by_ballot_name(
                        field_val,
                        electrol_races_by_ballot_name,
                    )
                    kwargs['electrol_race'] = electrol_race
                    continue

            if len(kwargs.items()):
                kwargs['tally'] = tally
                kwargs['order'] = candidate_id_by_ballot_order_dict.get(
                    kwargs.get('candidate_id'))
                bulk_mgr.add(Candidate(**kwargs))
        bulk_mgr.done()

        return
    except Exception as e:
        msg = 'Failed to create candidates, error: %s' % e
        if command:
            command.stdout.write(command.style.WARNING(msg))
        if logger:
            logger.warning(msg)
        raise Exception(msg)

def build_candidate_id_by_ballot_order_dict(ballot_order_file_path):
    """
    Builds a dictionary of candidate id by ballot order from the ballot order
    to candidates ids csv file.

    :param ballot_order_file_path: ballot order to candidates ids csv file.
    :returns: candidate ids by ballot order dictionary or error exception.
    """
    try:
        candidate_id_by_ballot_order = {}
        ballot_order_column_list =\
                getattr(settings,
                        'BALLOT_ORDER_COLUMN_NAMES')
        duckdb_ballot_order_data =\
            duckdb.from_csv_auto(ballot_order_file_path, header=True).project(
                    ','.join(
            ballot_order_column_list)).distinct().fetchall()
        candidate_id_by_ballot_order =\
            { item[0]: item[1] for item in duckdb_ballot_order_data }
        return candidate_id_by_ballot_order
    except Exception as e:
        msg = str("Error occured while generating candidate id to"
                  f"ballot order dictionary, error: {e}")
        raise Exception(msg)

@app.task()
def async_import_candidates_from_candidates_file(
        tally_id=None,
        csv_file_path=None,
        command=None,
        **kwargs):
    """Create candidates from a candidates csv file.

    :param tally_id: tally id.
    :param csv_file_path: candidates csv file path.
    :param command: stdout command.
    :returns: candidates count."""
    try:
        tally = Tally.objects.get(id=tally_id)
        candidates_file_path = csv_file_path
        duckdb_candidates_data =\
            duckdb.from_csv_auto(candidates_file_path, header=True)
        candidate_col_names =\
            getattr(settings,
                    'CANDIDATE_COLUMN_NAMES')
        check_for_missing_columns(
            candidate_col_names,
            duckdb_candidates_data.columns,
            'candidates'
        )
        ballot_order_file_path = kwargs.get('ballot_order_file_path')
        candidate_id_by_ballot_order_dict =\
            build_candidate_id_by_ballot_order_dict(
                ballot_order_file_path
            )
        ballots_by_ballot_number =\
            {
                ballot.number:\
                ballot for ballot in Ballot.objects.filter(tally=tally)
            }
        electrol_races_by_ballot_name =\
            {
                electrol_race.ballot_name:\
                electrol_race for electrol_race in\
                    ElectrolRace.objects.filter(tally=tally)
            }
        create_candidates_from_candidates_file_data(
            duckdb_candidates_data=duckdb_candidates_data,
            tally=tally,
            ballots_by_ballot_number=ballots_by_ballot_number,
            electrol_races_by_ballot_name=electrol_races_by_ballot_name,
            candidate_id_by_ballot_order_dict=
            candidate_id_by_ballot_order_dict,
            step_name=kwargs.get('step_name'),
            step_number=kwargs.get('step_number'),
            command=command,
        )

        return duckdb_candidates_data.shape[0]
    except Exception as e:
        msg = f'Error occured while trying to create candidates: {e}'
        if command:
            command.stdout.write(command.style.WARNING(msg))
        if logger:
            logger.warning(msg)
        raise Exception(msg)


from functools import partial
from logging import getLogger, StreamHandler, Formatter, DEBUG, INFO, WARNING, ERROR, CRITICAL
from logging.handlers import QueueHandler
from sys import stdout
from multiprocessing import Process, Queue, current_process
from time import sleep
import atexit

def start_listener(queue):
	listener = getLogger('compile')
	handler = StreamHandler(stdout)
	handler.setFormatter(Formatter('%(asctime)s %(processName)-10s %(name)s %(levelname)-8s %(message)s'))
	listener.addHandler(handler)
	while True:
		try:
			record = queue.get()
			if record is None:
				break
			# level or filter
			listener.handle(record)
		except Exception:
			import sys, traceback
			print('The logger had a problem!', file=sys.stderr)
			traceback.print_exc(file=sys.stderr)

def start_sender(queue):
	sender = getLogger('compile')
	sender.addHandler(QueueHandler(queue))
	for k in range(5):
		sleep(.070)
		sender.log(DEBUG, '{0:s}: message #{1:d}'.format(current_process().name, k))

def kill_logger(proc):
	proc.join(timeout=0.1)
	proc.terminate()

def main():
	queue = Queue(-1)
	listener = Process(target=start_listener, args=(queue,))
	listener.start()
	atexit.register(partial(kill_logger, listener))
	senders = []
	for k in range(5):
		sleep(.150)
		worker = Process(target=start_sender, args=(queue,))
		senders.append(worker)
		worker.start()
		atexit.register(partial(kill_logger, worker))
	for sender in senders:
		sender.join()
	queue.put_nowait(None)
	listener.join()

if __name__ == '__main__':
	main()


"""
Write a method anagrams_or_not that takes in two words and returns a boolean indicating whether or not the words are anagrams. Anagrams are words that contain the same characters but not necessarily in the same order. Solve this without using .sort
"""

def anagrams_or_not(word1, word2):
    arr1 = []
    arr2 = []
    for ele1 in word1:
        arr1.append(ele1)

    x = arr1

    for ele2 in word2:
        arr2.append(ele2)

    y = arr2

    x.sort()
    y.sort()

    if x == y:
        return True
    else:
        return False

print(anagrams_or_not("cat", "act"))  # => true
print(anagrams_or_not("restful", "fluster"))  # => true
print(anagrams_or_not("cat", "dog"))  # => false
print(anagrams_or_not("boot", "bootcamp"))  # => false


import sys
import heapq
input = sys.stdin.readline

heap = []
N = int(input())
for i in range(N):
    value = int(input())
    if value == 0:
        if len(heap) != 0:
            print(heapq.heappop(heap))
        else:
            print(0)
    else:
        heapq.heappush(heap, value)


import numpy as np

arr = np.array([1, 2, 3])

for x in arr:
  print(x)

# Iterating 2-D Arrays
arr = np.array([[1, 2, 3], [4, 5, 6]])

for x in arr:
  print(x)

for x in arr:
  for y in x:
      print(y)

# Iterating 3-D Arrays
print("Iterating 3-D Arrays")
arr = np.array([[[1, 2, 3], [4, 5, 6]], [[7, 8, 9], [10, 11, 12]]])

for x in arr:
  print(x)

for x in arr:
    for y in x:
      for z in y:
          print(z)

# Iterating Arrays Using nditer()
print("Iterating Arrays Using nditer()")
arr = np.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]])

for x in np.nditer(arr):
  print(x)

# Iterating With Different Step Size
print("Iterating With Different Step Size")
arr = np.array([[1, 2, 3, 4], [5, 6, 7, 8]])

for x in np.nditer(arr[:, ::2]):
  print(x)

#Enumerated Iteration Using ndenumerate()
print("Enumerated Iteration Using ndenumerate()")
arr = np.array([[1, 2, 3, 4], [5, 6, 7, 8]])

for idx, x in np.ndenumerate(arr):
  print(idx, x)


from hashlib import sha256
import json
import time

from flask import Flask, request

class Block:
    def __init__(self, index, transactions, timestamp, previous_hash):
        self.index = index
        self.transactions = transactions
        self.timestamp = timestamp
        self.previous_hash = previous_hash
        self.nonce = 0

    def compute_hash(self):
        """
        """
        block_string = json.dumps(self.__dict__, sort_keys=True)
        return sha256(block_string.encode()).hexdigest()

class Blockchain:
    difficulty = 0

    def __init__(self):
        self.unconfirmed_transactions = []
        self.chain = []
        self.create_genesis_block()

    def create_genesis_block(self):
        """
        """
        genesis_block = Block(0, [], time.time(), "0")
        genesis_block.hash = genesis_block.compute_hash()
        print(genesis_block.hash)
        self.chain.append(genesis_block)

    @property
    def last_block(self):
        return self.chain[-1]

    def add_block(self, block, proof):
        """
        """
        previous_hash = self.last_block.hash

        if previous_hash != block.previous_hash:
            return False

        if not Blockchain.is_valid_proof(block, proof):
            return False

        block.hash = proof
        self.chain.append(block)

        return True

    def proof_of_work(self, block):
        block.nonce = 0

        computed_hash = block.compute_hash()
        while not computed_hash.startswith('0' * Blockchain.difficulty):
            block.nonce += 1
            computed_hash = block.compute_hash()
            print("hash > " + computed_hash)
        return computed_hash

    def add_new_transaction(self, transaction):
        self.unconfirmed_transactions.append(transaction)

    @classmethod
    def is_valid_proof(cls, block, block_hash):
        """

        """
        return (block_hash.startswith('0' * Blockchain.difficulty) and
                block_hash == block.compute_hash())

    @classmethod
    def check_chain_validity(cls, chain):
        result = True
        previous_hash = "0"

        for block in chain:
            block_hash = block.hash
            delattr(block, "hash")

            if not cls.is_valid_proof(block, block.hash) or previous_hash != block.previous_hash :
                result = False
                break

            block.hash, previous_hash = block_hash, block_hash

        return result

    def mine(self):
        """

        """
        if not self.unconfirmed_transactions:
            return False

        last_block = self.last_block

        new_block = Block(
            index=last_block.index + 1,
            transactions=self.unconfirmed_transactions,
            timestamp=time.time(),
            previous_hash=last_block.hash
        )

        proof = self.proof_of_work(new_block)
        self.add_block(new_block, proof)

        self.unconfirmed_transactions = []
        # announce it to the network
        announce_new_block(new_block)
        return new_block.index

app = Flask(__name__)

blockchain = Blockchain()

peers = set()

@app.route('/new_transaction', methods=['POST'])
def new_transaction():
    tx_data = request.get_json()
    required_fields = ["author", "content"]

    for field in required_fields:
        if not tx_data[field]:
            return "Invalid transaction data", 404

    tx_data["timestamp"] = time.time()

    blockchain.add_new_transaction(tx_data)

    return "Success", 201

@app.route('/chain', methods=['GET'])
def get_chain():
    consensus()
    chain_data = []
    for block in blockchain.chain:
        chain_data.append(block.__dict__)
    return json.dumps({"length" : len(chain_data), "chain": chain_data})

@app.route('/pending_tx')
def get_pending_tx():
    return json.dumps(blockchain.unconfirmed_transactions)

@app.route('/mine', methods=['GET'])
def mine_unconfirmed_transactions():
    result = blockchain.mine()
    if not result:
        return "No transactions to mine"
    return "Block #{} is mined".format(result)

@app.route('/add_block', methods=['POST'])
def validate_and_and_block():
    block_data = request.get_json()
    block = Block(
        block_data['index'],
        block_data['transactions'],
        block_data['timestamp'],
        block_data['previous_hash']
    )

    proof = block_data['hash']
    added = blockchain.add_block(block, proof)

    if not added:
        return "The block was discarded by the node", 400

    return "Block added to the chain", 201

@app.route('/add_nodes', methods=['POST'])
def register_new_peers():
    nodes = request.get_json()
    if not nodes:
        return "Invalid data", 400

    for node in nodes:
        peers.add(node)

    return "Success", 201

def consensus():
    global blockchain

    longest_chain = None
    current_len = len(blockchain.chain)

    for node in peers:
        response = request.get('http://{}/chain'.format(node))
        length = response.json()['length']
        chain = response.json()['chain']

        if length > current_len and blockchain.check_chain_validity(chain):
            current_len = length
            longest_chain = chain

        if longest_chain:
            blockchain = longest_chain
            return True

        return False

def announce_new_block(block):
        for peer in peers:
            url = "http://{}/add_block".format(peer)
            print('announce_new_block >> ' + url)
            print('json data >> ' + json.dumps(block.__dict__, sort_keys=True))
            request.post(url, data=json.dumps(block.__dict__, sort_keys=True))

app.run(debug=True, port=8000)


# 스택

import sys

N = int(sys.stdin.readline())
stk = []

for _ in range(N):
    temp = sys.stdin.readline().split()
    if temp[0] == 'push':
        stk.append(int(temp[1]))
    elif temp[0] == 'pop':
        if len(stk) == 0:
            print(-1)
        else:
            print(stk.pop())
    elif temp[0] == 'size':
        print(len(stk))
    elif temp[0] == 'empty':
        if len(stk) == 0:
            print(1)
        else:
            print(0)
    elif temp[0] == 'top':
        if len(stk) == 0:
            print(-1)
        else:
            print(stk[-1])


"""Create data plots for ERP-SCANR project - plots for group analysis."""

import os
import matplotlib.pyplot as plt

from erpsc.core.db import check_db

#########################################################################################
#########################################################################################
#########################################################################################

def plot_time_assocs(dat, save_fig=False):
    """

    Parameters
    ----------
    dat : list of list of [str, str, int]
        ERP data - [association, P or N, latency]
    """

    # Plot params
    offsets = {'P': 50, 'N': -50}
    rotations = {'P': 45, 'N': -45}

    # Initialize Plot
    fig = plt.figure(figsize=(12, 5))
    fig.suptitle('ERP Correlates Across Time', fontsize=24, fontweight='bold')
    ax = fig.add_subplot(111)

    # Set plot limits
    ax.set_xlim([50, 600])
    ax.set_ylim([-100, 100])

    # Add x-ticks
    plt.xticks([250, 500], ['250 ms', '500 ms'])
    ax.set_yticks([])

    # Set ticks and plot lines
    ax.spines['right'].set_color('none')
    ax.spines['bottom'].set_position('center')
    ax.spines['top'].set_color('none')
    ax.xaxis.set_ticks_position('none')
    ax.yaxis.set_ticks_position('none')
    ax.spines['left'].set_linewidth(2)
    ax.spines['bottom'].set_linewidth(2)

    # Add data to plot from
    for d in dat:

        # Text takes: [X-pos, Y-pos, word, rotation]
        #  Where X-pos is latency, y-pos & rotation are defaults given +/-
        ax.text(d[2], offsets[d[1]], d[0], rotation=rotations[d[1]], fontsize=20)

    # Save out - if requested
    if save_fig:

        db = check_db(db)
        s_file = os.path.join(db.figs_path, 'LatencyAssociations' + '.svg')

        plt.savefig(s_file, transparent=True)


import bohrium
import util
import functools
import operator
from bohrium_api import _info

class test_set_bool_mask_scalar:
    def init(self):
        for dtype in _info.numpy_types():
            dtype = "np.%s"%dtype.name
            for cmd, shape in util.gen_random_arrays("R", 2, min_ndim=1, samples_in_each_ndim=1,
                                                     dtype=dtype, no_views=True):
                cmd = "R = bh.random.RandomState(42); res=%s; " \
                      "m = R.random_integers(0, 1, res.shape, bohrium=BH).astype(np.bool); " % cmd
                yield (cmd, dtype)

    @util.add_bh107_cmd
    def test_set_scalar(self, arg):
        (cmd, dtype) = arg
        cmd += "res[m] = %s(42)" % dtype
        return cmd

    @util.add_bh107_cmd
    def test_set_scalar_nan(self, arg):
        (cmd, dtype) = arg
        if dtype in util.TYPES.FLOAT:
            cmd += "res[m] = np.nan"
        else:
            cmd = "res = 0"
        return cmd

class test_set_bool_mask:
    def init(self):
        for cmd, shape in util.gen_random_arrays("R", 3, max_dim=50, dtype="np.float64"):
            nelem = functools.reduce(operator.mul, shape)
            if nelem == 0:
                continue
            cmd = "R = bh.random.RandomState(42); res=%s; " \
                  "m = R.random_integers(0, 1, res.shape, bohrium=BH).astype(np.bool); " % cmd
            yield (cmd)

    @util.add_bh107_cmd
    def test_set(self, cmd):
        cmd += "res[m] = res.copy()[m] * 42"
        return cmd

class test_get_bool_mask:
    def init(self):
        for ary, shape in util.gen_random_arrays("R", 3, max_dim=50, dtype="np.float64"):
            nelem = functools.reduce(operator.mul, shape)
            if nelem == 0:
                continue
            cmd = "R = bh.random.RandomState(42); a=%s; " % ary
            cmd += "m = R.random_integers(0, 1, a.shape, bohrium=BH).astype(np.bool); "
            yield (cmd)

    @util.add_bh107_cmd
    def test_get(self, cmd):
        cmd += "res = a[m]"
        return cmd

class test_where:
    def init(self):
        for dtype in _info.numpy_types():
            dtype = "np.%s" % dtype.name
            for cmd, shape in util.gen_random_arrays("R", 2, min_ndim=1, samples_in_each_ndim=1,
                                                     dtype=dtype, no_views=True):
                nelem = functools.reduce(operator.mul, shape)
                # NumPy returns some fucked up shapes when working on 1-sized arrays
                # thus we will ignore them
                if nelem > 1:
                    cmd = "R = bh.random.RandomState(42); a = %s; " % cmd
                    cmd += "b = R.random(%s, dtype=%s, bohrium=BH); " % (shape, dtype)
                    cmd += "m = R.random_integers(0, 1, size=a.shape, dtype=np.bool, bohrium=BH); "
                    yield (cmd, dtype)

    def test_scalar_condition(self, arg):
        (cmd, dtype) = arg
        cmd += "res = M.where(%s(True), a, b)" % dtype
        return cmd

    def test_scalar1(self, arg):
        (cmd, dtype) = arg
        cmd += "res = M.where(m, %s(42), a)" % dtype
        return cmd

    def test_scalar2(self, arg):
        (cmd, dtype) = arg
        cmd += "res = M.where(m, a, %s(42))" % dtype
        return cmd

    def test_array(self, arg):
        (cmd, dtype) = arg
        cmd += "res = M.where(m, a, b)"
        return cmd

    def test_nan_array(self, arg):
        (cmd, dtype) = arg
        if dtype in util.TYPES.FLOAT:
            cmd += "a[0] = M.nan;"
            cmd += "res = M.where(M.isnan(a), M.ones_like(a), a)"
            return cmd
        else:
            return "res = 0"

    def test_nan_scalar1(self, arg):
        (cmd, dtype) = arg
        cmd += "res = M.where(m, M.nan, a)"
        return cmd

    def test_nan_scalar2(self, arg):
        (cmd, dtype) = arg
        cmd += "res = M.where(m, a, M.nan)"
        return cmd


from numpy.random import seed, laplace, choice
from pandas import DataFrame, merge
from itertools import product

from generative_models.data_synthesiser_utils.datatypes.FloatAttribute import FloatAttribute
from generative_models.data_synthesiser_utils.datatypes.IntegerAttribute import IntegerAttribute
from generative_models.data_synthesiser_utils.datatypes.StringAttribute import StringAttribute
from generative_models.data_synthesiser_utils.utils import bayes_worker, normalize_given_distribution, exponential_mechanism

from generative_models.generative_model import GenerativeModel

from utils.constants import *
from utils.logging import LOGGER

class IndependentHistogram(GenerativeModel):

    def __init__(self, metadata, histogram_bins=10, infer_ranges=False, multiprocess=True):
        self.metadata = self._read_meta(metadata)
        self.histogram_bins = histogram_bins

        self.datatype = DataFrame
        self.multiprocess = bool(multiprocess)
        self.infer_ranges = bool(infer_ranges)

        self.DataDescriber = None

        self.trained = False

        self.__name__ = 'IndependentHistogram'

    def fit(self, data):
        assert isinstance(data, self.datatype), f'{self.__class__.__name__} expects {self.datatype} as input data but got {type(data)}'
        LOGGER.debug(f'Start fitting IndependentHistogram model to data of shape {data.shape}...')
        if self.trained:
            self.trained = False
            self.DataDescriber = None

        self.DataDescriber = DataDescriber(self.metadata, self.histogram_bins, self.infer_ranges)
        self.DataDescriber.describe(data)
        LOGGER.debug(f'Finished fitting IndependentHistogram')
        self.trained = True

    def generate_samples(self, nsamples):
        assert self.trained, "Model must be fitted to some data first"

        LOGGER.debug(f'Generate synthetic dataset of size {nsamples}')
        synthetic_dataset = DataFrame(columns=self.DataDescriber.attr_names)
        for attr_name, Attr in self.DataDescriber.attr_dict.items():
            binning_indices = Attr.sample_binning_indices_in_independent_attribute_mode(nsamples)
            synthetic_dataset[attr_name] = Attr.sample_values_from_binning_indices(binning_indices)

        LOGGER.debug(f'Generated synthetic dataset of size {nsamples}')
        return synthetic_dataset

    def _read_meta(self, metadata):
        """ Read metadata from metadata file."""
        metadict = {}

        for cdict in metadata['columns']:
            col = cdict['name']
            coltype = cdict['type']

            if coltype == FLOAT or coltype == INTEGER:
                metadict[col] = {
                    'type': coltype,
                    'min': cdict['min'],
                    'max': cdict['max']
                }

            elif coltype == CATEGORICAL or coltype == ORDINAL:
                metadict[col] = {
                    'type': coltype,
                    'categories': cdict['i2s'],
                    'size': len(cdict['i2s'])
                }

            else:
                raise ValueError(f'Unknown data type {coltype} for attribute {col}')

        return metadict

class BayesianNet(GenerativeModel):
    """
    A BayesianNet model using non-private GreedyBayes to learn conditional probabilities
    """
    def __init__(self, metadata, histogram_bins=10, degree=1, infer_ranges=False, multiprocess=True, seed=None):
        self.metadata = self._read_meta(metadata)
        self.histogram_bins = histogram_bins
        self.degree = degree
        self.num_attributes = len(metadata['columns'])

        self.multiprocess = bool(multiprocess)
        self.infer_ranges = bool(infer_ranges)
        self.seed = seed
        self.datatype = DataFrame

        self.bayesian_network = None
        self.conditional_probabilities = None
        self.DataDescriber = None
        self.trained = False

        self.__name__ = 'BayesianNet'

    def fit(self, data):
        assert isinstance(data, self.datatype), f'{self.__class__.__name__} expects {self.datatype} as input data but got {type(data)}'
        assert len(list(data)) >= 2, "BayesianNet requires at least 2 attributes(i.e., columns) in dataset."
        LOGGER.debug(f'Start training BayesianNet on data of shape {data.shape}...')
        if self.trained:
            self.trained = False
            self.DataDescriber = None
            self.bayesian_network = None
            self.conditional_probabilities = None

        self.DataDescriber = DataDescriber(self.metadata, self.histogram_bins, self.infer_ranges)
        self.DataDescriber.describe(data)

        encoded_df = DataFrame(columns=self.DataDescriber.attr_names)
        for attr_name, column in self.DataDescriber.attr_dict.items():
            encoded_df[attr_name] = column.encode_values_into_bin_idx()

        self.bayesian_network = self._greedy_bayes_linear(encoded_df, self.degree)

        self.conditional_probabilities = self._construct_conditional_probabilities(self.bayesian_network, encoded_df)

        LOGGER.debug(f'Finished training Bayesian net')
        self.trained = True

    def generate_samples(self, nsamples):
        LOGGER.debug(f'Generate synthetic dataset of size {nsamples}')
        assert self.trained, "Model must be fitted to some real data first"
        synthetic_data = DataFrame(columns=self.DataDescriber.attr_names)

        # Get samples for attributes modelled in Bayesian net
        encoded_dataset = self._generate_encoded_dataset(nsamples)

        for attr in self.DataDescriber.attr_names:
            column = self.DataDescriber.attr_dict[attr]
            if attr in encoded_dataset:
                synthetic_data[attr] = column.sample_values_from_binning_indices(encoded_dataset[attr])
            else:
                # For attributes not in BN use independent attribute mode
                binning_indices = column.sample_binning_indices_in_independent_attribute_mode(nsamples)
                synthetic_data[attr] = column.sample_values_from_binning_indices(binning_indices)

        return synthetic_data

    def _generate_encoded_dataset(self, nsamples):
        encoded_df = DataFrame(columns=self._get_sampling_order(self.bayesian_network))

        bn_root_attr = self.bayesian_network[0][1][0]
        root_attr_dist = self.conditional_probabilities[bn_root_attr]
        encoded_df[bn_root_attr] = choice(len(root_attr_dist), size=nsamples, p=root_attr_dist)

        for child, parents in self.bayesian_network:
            child_conditional_distributions = self.conditional_probabilities[child]

            for parents_instance in child_conditional_distributions.keys():
                dist = child_conditional_distributions[parents_instance]
                parents_instance = list(eval(parents_instance))

                filter_condition = ''
                for parent, value in zip(parents, parents_instance):
                    filter_condition += f"(encoded_df['{parent}']=={value})&"

                filter_condition = eval(filter_condition[:-1])
                size = encoded_df[filter_condition].shape[0]
                if size:
                    encoded_df.loc[filter_condition, child] = choice(len(dist), size=size, p=dist)

            # Fill any nan values by sampling from marginal child distribution
            marginal_dist = self.DataDescriber.attr_dict[child].distribution_probabilities
            null_idx = encoded_df[child].isnull()
            encoded_df.loc[null_idx, child] = choice(len(marginal_dist), size=null_idx.sum(), p=marginal_dist)

        encoded_df[encoded_df.columns] = encoded_df[encoded_df.columns].astype(int)

        return encoded_df

    def _get_sampling_order(self, bayesian_net):
        order = [bayesian_net[0][1][0]]
        for child, _ in bayesian_net:
            order.append(child)
        return order

    def _greedy_bayes_linear(self, encoded_df, k=1):
        """Construct a Bayesian Network (BN) using greedy algorithm."""
        dataset = encoded_df.astype(str, copy=False)

        # Optional: Fix sed for reproducibility
        if self.seed is not None:
            seed(self.seed)

        root_attribute = choice(dataset.columns)
        V = [root_attribute]
        rest_attributes = set(dataset.columns)
        rest_attributes.remove(root_attribute)
        bayesian_net = []
        while rest_attributes:
            parents_pair_list = []
            mutual_info_list = []

            num_parents = min(len(V), k)
            for child, split in product(rest_attributes, range(len(V) - num_parents + 1)):
                task = (child, V, num_parents, split, dataset)
                res = bayes_worker(task)
                parents_pair_list += res[0]
                mutual_info_list += res[1]

            idx = mutual_info_list.index(max(mutual_info_list))

            bayesian_net.append(parents_pair_list[idx])
            adding_attribute = parents_pair_list[idx][0]
            V.append(adding_attribute)
            rest_attributes.remove(adding_attribute)

        return bayesian_net

    def _construct_conditional_probabilities(self, bayesian_network, encoded_dataset):
        k = len(bayesian_network[-1][1])
        conditional_distributions = {}

        # first k+1 attributes
        root = bayesian_network[0][1][0]
        kplus1_attributes = [root]
        for child, _ in bayesian_network[:k]:
            kplus1_attributes.append(child)

        freqs_of_kplus1_attributes = self._get_attribute_frequency_counts(kplus1_attributes, encoded_dataset)

        # get distribution of root attribute
        root_marginal_freqs = freqs_of_kplus1_attributes.loc[:, [root, 'count']].groupby(root).sum()['count']
        conditional_distributions[root] = normalize_given_distribution(root_marginal_freqs).tolist()

        for idx, (child, parents) in enumerate(bayesian_network):
            conditional_distributions[child] = {}

            if idx < k:
                stats = freqs_of_kplus1_attributes.copy().loc[:, parents + [child, 'count']]
            else:
                stats = self._get_attribute_frequency_counts(parents + [child], encoded_dataset)

            stats = DataFrame(stats.loc[:, parents + [child, 'count']].groupby(parents + [child]).sum())

            if len(parents) == 1:
                for parent_instance in stats.index.levels[0]:
                    dist = normalize_given_distribution(stats.loc[parent_instance]['count']).tolist()
                    conditional_distributions[child][str([parent_instance])] = dist
            else:
                for parents_instance in product(*stats.index.levels[:-1]):
                    dist = normalize_given_distribution(stats.loc[parents_instance]['count']).tolist()
                    conditional_distributions[child][str(list(parents_instance))] = dist

        return conditional_distributions

    def _get_attribute_frequency_counts(self, attributes, encoded_dataset):
        # Get attribute counts for category combinations present in data
        counts = encoded_dataset.groupby(attributes).size()
        counts.name = 'count'
        counts = counts.reset_index()

        # Get all possible attribute combinations
        attr_combs = [range(self.DataDescriber.attr_dict[attr].domain_size) for attr in attributes]
        full_space = DataFrame(columns=attributes, data=list(product(*attr_combs)))
        # stats.reset_index(inplace=True)
        full_counts = merge(full_space, counts, how='left')
        full_counts.fillna(0, inplace=True)

        return full_counts

    def _read_meta(self, metadata):
        """ Read metadata from metadata file."""
        metadict = {}

        for cdict in metadata['columns']:
            col = cdict['name']
            coltype = cdict['type']

            if coltype == FLOAT or coltype == INTEGER:
                metadict[col] = {
                    'type': coltype,
                    'min': cdict['min'],
                    'max': cdict['max']
                }

            elif coltype == CATEGORICAL or coltype == ORDINAL:
                metadict[col] = {
                    'type': coltype,
                    'categories': cdict['i2s'],
                    'size': len(cdict['i2s'])
                }

            else:
                raise ValueError(f'Unknown data type {coltype} for attribute {col}')

        return metadict

class PrivBayes(BayesianNet):
    """"
    A differentially private BayesianNet model using GreedyBayes
    """
    def __init__(self, metadata, histogram_bins=10, degree=1, epsilon=.1, infer_ranges=False, multiprocess=True, seed=None):
        super().__init__(metadata=metadata, histogram_bins=histogram_bins, degree=degree, infer_ranges=infer_ranges, multiprocess=multiprocess, seed=seed)

        self.epsilon = float(epsilon)

        self.__name__ = f'PrivBayesEps{self.epsilon}'

    @property
    def laplace_noise_scale(self):
        return 2 * (self.num_attributes - self.degree) / (self.epsilon / 2)

    def _greedy_bayes_linear(self, encoded_df, k=1):
        """Construct a Bayesian Network (BN) using greedy algorithm."""
        dataset = encoded_df.astype(str, copy=False)
        num_tuples, num_attributes = dataset.shape

        # Optional: Fix seed for reproducibility
        if self.seed is not None:
            seed(self.seed)

        attr_to_is_binary = {attr: dataset[attr].unique().size <= 2 for attr in dataset}

        root_attribute = choice(dataset.columns)
        V = [root_attribute]
        rest_attributes = set(dataset.columns)
        rest_attributes.remove(root_attribute)
        bayesian_net = []
        while rest_attributes:
            parents_pair_list = []
            mutual_info_list = []

            num_parents = min(len(V), k)
            for child, split in product(rest_attributes, range(len(V) - num_parents + 1)):
                task = (child, V, num_parents, split, dataset)
                res = bayes_worker(task)
                parents_pair_list += res[0]
                mutual_info_list += res[1]

            sampling_distribution = exponential_mechanism(self.epsilon/2, mutual_info_list, parents_pair_list, attr_to_is_binary,
                                                          num_tuples, num_attributes)
            idx = choice(list(range(len(mutual_info_list))), p=sampling_distribution)

            bayesian_net.append(parents_pair_list[idx])
            adding_attribute = parents_pair_list[idx][0]
            V.append(adding_attribute)
            rest_attributes.remove(adding_attribute)

        return bayesian_net

    def _get_attribute_frequency_counts(self, attributes, encoded_dataset):
        """ Differentially private mechanism to get attribute frequency counts"""
        # Get attribute counts for category combinations present in data
        counts = encoded_dataset.groupby(attributes).size()
        counts.name = 'count'
        counts = counts.reset_index()

        # Get all possible attribute combinations
        attr_combs = [range(self.DataDescriber.attr_dict[attr].domain_size) for attr in attributes]
        full_space = DataFrame(columns=attributes, data=list(product(*attr_combs)))
        full_counts = merge(full_space, counts, how='left')
        full_counts.fillna(0, inplace=True)

        # Get Laplace noise sample
        noise_sample = laplace(0, scale=self.laplace_noise_scale, size=full_counts.index.size)
        full_counts['count'] += noise_sample
        full_counts.loc[full_counts['count'] < 0, 'count'] = 0

        return full_counts

class DataDescriber(object):
    def __init__(self, metadata, histogram_bins, infer_ranges=False):
        self.metadata = metadata
        self.histogram_bins = histogram_bins
        self.infer_ranges = infer_ranges

        self.attr_dict = None
        self.attr_names = None

    def describe(self, df):
        self.attr_names = self._get_attr_names()
        self.attr_dict = self._represent_input_dataset_by_columns(df)

        for col, Attribute in self.attr_dict.items():
            Attribute.infer_distribution()

    def _get_attr_names(self):
        return [c for c in self.metadata.keys()]

    def _represent_input_dataset_by_columns(self, df):
        attr_dict = {}

        for col, cdict in self.metadata.items():
            coltype = cdict['type']

            paras = (col, df[col], self.histogram_bins)
            if coltype in NUMERICAL:
                if coltype == FLOAT:
                    Attribute = FloatAttribute(*paras)
                else:
                    Attribute = IntegerAttribute(*paras)

                if self.infer_ranges:
                    cmin, cmax = min(df[col]), max(df[col])
                else:
                    cmin, cmax = cdict['min'], cdict['max']

                Attribute.set_domain(domain=(cmin, cmax))

            elif coltype in STRINGS:
                Attribute = StringAttribute(*paras)
                if self.infer_ranges:
                    ccats = list(df[col].unique())
                else:
                    ccats = cdict['categories']

                Attribute.set_domain(domain=ccats)

            else:
                raise Exception(f'The DataType of {col} is unknown.')

            attr_dict[col] = Attribute

        return attr_dict


import boto3
from os import getenv
from dotenv import load_dotenv
import logging
from botocore.exceptions import ClientError
from boto3.s3.transfer import TransferConfig
import argparse
import json
import magic
import os
import logging
import errno
import sys
import os
import threading
import ntpath
from pathlib import Path
from hurry.filesize import size, si
from urllib.request import urlopen, Request
from random import choice
parser = argparse.ArgumentParser()
parser.add_argument('--bucket_name', "-bn", type=str, help='Name of S3 bucket')
parser.add_argument('--url', type=str, help='link to download file')
parser.add_argument('--file_name', "-fn", type=str, help='uploaded file name')
parser.add_argument('--tool', '-t', type=str, help='choose function')
parser.add_argument('--filepath', "-fp", type=str, help='file path for upload')
parser.add_argument('--multipart_threshold', "-mth", type=int, default=5 * 1024 * 1024 * 1024, help='Multipart threshold in bytes (default: 5GB)')
parser.add_argument('--days', '-d', type=int, help='number of days when object will be deleted')
parser.add_argument('--memetype', '-mt', type=str, help='memetype which is allowed to upload')
parser.add_argument('-del', dest='delete', action='store_true', help='Delete the file')
parser.add_argument('-vers', dest='versioning', action='store_true', help='check versioning')
parser.add_argument('-verslist', dest='versionlist', action='store_true', help='version list')
parser.add_argument('-prevers', dest='previous_version', action='store_true', help='roll back to previous version')
parser.add_argument('-orgobj', dest='organize_objects', action='store_true', help='put files in folder according extention type')
parser.add_argument('--inspire', nargs='?', const='true', help='Quote from api' )
parser.add_argument('-save', action='store_true', help='upload quote.json to s3 bucket' )
args = parser.parse_args()
s3 = boto3.client('s3')
from hashlib import md5
from time import localtime
load_dotenv()
s3 = boto3.client('s3')
def init_client():
    try:
        client = boto3.client("s3",
                              aws_access_key_id=getenv("aws_access_key_id"),
                              aws_secret_access_key=getenv(
                                  "aws_secret_access_key"),
                              aws_session_token=getenv("aws_session_token"),
                              region_name=getenv("aws_region_name")
                              #  config=botocore.client.Config(
                              #      connect_timeout=conf.remote_cfg["remote_timeout"],
                              #      read_timeout=conf.remote_cfg["remote_timeout"],
                              #      region_name=conf.remote_cfg["aws_default_region"],
                              #      retries={
                              #          "max_attempts": conf.remote_cfg["remote_retries"]}
                              )
        # check if credentials are correct
        return client
    except ClientError as e:
        logging.error(e)
    except:
        logging.error("Unexpected error")
def list_buckets(s3_client):
    try:
        # https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListBuckets.html
        # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/client/list_buckets.html
        return s3_client.list_buckets()
    except ClientError as e:
        logging.error(e)
        return False
def create_bucket(s3_client, bucket_name, region=getenv("aws_region_name")):
    # Create bucket
    try:
        location = {'LocationConstraint': region}
        # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/client/create_bucket.html
        response = s3_client.create_bucket(
            Bucket=args.bucket_name,
            CreateBucketConfiguration=location
        )
    except ClientError as e:
        logging.error(e)
        return False
    status_code = response["ResponseMetadata"]["HTTPStatusCode"]
    if status_code == 200:
        return True
    return False
def delete_bucket(s3_client, bucket_name):
    # Delete bucket
    try:
        # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3/client/delete_bucket.html
        response = s3_client.delete_bucket(Bucket=args.bucket_name)
    except ClientError as e:
        logging.error(e)
        return False
    status_code = response["ResponseMetadata"]["HTTPStatusCode"]
    if status_code == 200:
        return True
    return False
def bucket_exists(s3_client, bucket_name):
    try:
        response = s3_client.head_bucket(Bucket=args.bucket_name)
    except ClientError as e:
        logging.error(e)
        return False
    status_code = response["ResponseMetadata"]["HTTPStatusCode"]
    if status_code == 200:
        return True
    return False
def download_file_and_upload_to_s3(s3_client, bucket_name, url, file_name, keep_local=False):
    from urllib.request import urlopen
    import io
    import urllib
    url = args.url
    format=urllib.request.urlopen(url).info()['content-type']
    format=format.split('/')
    formatlist = ["bmp", "jpg", "jpeg", "png", "webp", "mp4"]
    if format[1] in formatlist:
        with urlopen(args.url) as response:
            content = response.read()
            try:
                s3_client.upload_fileobj(
                    Fileobj=io.BytesIO(content),
                    Bucket=args.bucket_name,
                    ExtraArgs={'ContentType': 'image/jpg'},
                    Key=args.file_name
                )
                print("ფოტო აიტვირთა")
            except Exception as e:
                logging.error(e)

        if keep_local:
            with open(file_name, mode='wb') as jpg_file:
                jpg_file.write(content)
    else:
        print("ამ გაფართოების მქონე ფაილის ატვირთვა აკრძალულია")
    # public URL
    return "https://s3-{0}.amazonaws.com/{1}/{2}".format(
        'us-west-2',
        bucket_name,
        file_name
    )
def set_object_access_policy(s3_client, bucket_name, file_name):
    try:
        response = s3_client.put_object_acl(
            ACL="public-read",
            Bucket=args.bucket_name,
            Key=args.file_name
        )
    except ClientError as e:
        logging.error(e)
        return False
    status_code = response["ResponseMetadata"]["HTTPStatusCode"]
    if status_code == 200:
        return True
    return False
def generate_public_read_policy(bucket_name):
    import json
    policy = {
        "Version": "2012-10-17",
        "Statement": [
            {
                "Sid": "PublicReadGetObject",
                "Effect": "Allow",
                "Principal": "*",
                "Action": "s3:GetObject",
                "Resource": f"arn:aws:s3:::{args.bucket_name}/*",
            }
        ],
    }

    return json.dumps(policy)
def create_bucket_policy(s3_client, bucket_name):
    s3_client.put_bucket_policy(
        Bucket=bucket_name, Policy=generate_public_read_policy(args.bucket_name)
    )
    print("Bucket policy შეიქმნა წარმატებით")
def read_bucket_policy(s3_client, bucket_name):
    try:
        policy = s3_client.get_bucket_policy(Bucket=args.bucket_name)
        policy_str = policy["Policy"]
        print(policy_str)
    except ClientError as e:
        logging.error(e)
        return False
def upload_file(s3_client, bucket_name, file_name, filepath):
    with open(args.filepath, 'rb') as f:
        s3_client.upload_fileobj(f, args.bucket_name, args.file_name)
    print(f'{args.file_name} ფაილი აიტვირთა წარმატებით')
def lifecycle(s3_client, bucket_name , days):
    lifecycle_config = {
        'Rules': [
            {
                'ID': 'Delete after {} days'.format(args.days),
                'Status': 'Enabled',
                'Prefix': '',
                'Expiration': {
                    'Days': args.days
                }
            }
        ]
    }
    s3_client.put_bucket_lifecycle_configuration(Bucket=args.bucket_name, LifecycleConfiguration=lifecycle_config)
    print(f'ბაკეტი {args.bucket_name} წაიშლება {args.days} დღეში ')
def big_file_upload(s3_client, bucketname, file_name, filepath, multipart_threshold):
    config = TransferConfig(multipart_threshold=args.multipart_threshold)
    with open(args.filepath, 'rb') as f:
        object_key = args.file_name or args.filepath.split('/')[-1]
        s3_client.upload_fileobj(f, args.bucket_name, object_key, Config=config)
    print(f'{args.file_name} აიტვირთა წარმატებით')
def list_objects(s3_client, bucket_name):
    response = s3_client.list_objects(Bucket=args.bucket_name)
    for obj in response['Contents']:
        print(obj['Key'])
def previous_version(s3_client, bucket_name, file_name):
    try:
        response = s3_client.get_object(Bucket=args.bucket_name, Key=args.file_name)
        latest_version_id = response['VersionId']
        response = s3_client.list_object_versions(Bucket=args.bucket_name, Prefix=args.file_name)
        previous_version_id = response['Versions'][1]['VersionId']
        response = s3_client.copy_object(
            Bucket=args.bucket_name,
            Key=args.file_name,
            CopySource={'Bucket': args.bucket_name, 'Key': args.file_name, 'VersionId': previous_version_id})
        print(f'{args.file_name} ფაილი დაუბრუნდა წინა ვერსიას')
    except:
        print(f'{args.file_name} ფაილის წინა ვერსიაზე ვერ დაბრუნდა')
def version_list(s3_client, bucket_name, file_name):
    response = s3_client.list_object_versions(Bucket=args.bucket_name, Prefix=args.file_name)
    num_versions = len(response['Versions'])
    dates = [v['LastModified'] for v in response['Versions']]
    print(f'Bucket name: {args.bucket_name}')
    print(f'File name: {args.file_name}')
    print(f'Number of versions: {num_versions}')
    print('Creation dates of versions:')
    for date in dates:
        print(date)
def versioning(s3_client, bucket_name):
    output = s3_client.get_bucket_versioning(Bucket=args.bucket_name,)
    try:
        status = output['Status']
        print(f'ბაკეტზე {args.bucket_name} ვერსიონირება ჩართლია')
    except:
        print(f'ბაკეტზე {args.bucket_name} ვერსიონირება არ არის ჩართული')
def delete_file(s3_client, bucket_name, filename):
    response = s3_client.delete_object(Bucket=args.bucket_name, Key=args.file_name)
    print(f'{args.file_name} ფაილი წაიშალა')
def organize_objects(s3_client, bucket_name):
    counter = {}
    response = s3_client.list_objects_v2(Bucket=args.bucket_name)
    if response is not None:
        contents = response.get('Contents', [])
        for obj in contents:
            key = obj['Key']
            if '.' in key:
                extension_name = key.split(".")[-1]
                if not counter.get(extension_name):
                    counter[extension_name] = 1
                    destination_key = extension_name + '/' + key
                    s3_client.put_object(Bucket=args.bucket_name, Key=extension_name+'/')
                    s3_client.copy_object(Bucket=args.bucket_name, CopySource={'Bucket': args.bucket_name, 'Key': key}, Key=destination_key)
                    s3_client.delete_object(Bucket=args.bucket_name, Key=key)
                else:
                    counter[extension_name] += 1
                    destination_key = extension_name + '/' + key
                    s3_client.copy_object(Bucket=args.bucket_name, CopySource={'Bucket': args.bucket_name, 'Key': key}, Key=destination_key)
                    s3_client.delete_object(Bucket=args.bucket_name, Key=key)

    else:
        print('Response is None')
    print(counter)
def upload_with_magic(s3_client, bucket_name, file_name, filepath):
    ext = magic.from_file(args.filepath, mime=True).split('/')[-1] + '/' + args.file_name
    with open(args.filepath, 'rb') as f:
        s3_client.upload_fileobj(f, args.bucket_name, ext)
        print(f'{args.file_name} წარმატებით აიტვირთა')
def delete_old_versions(s3_client, bucket_name, file_name, days):
    from datetime import date
    from datetime import datetime
    response = s3_client.list_object_versions(Bucket=args.bucket_name, Prefix=args.file_name)
    dates = [v['LastModified'] for v in response['Versions']]
    today = str(date.today())
    for version in response['Versions']:
        id = version['VersionId']
        responses = s3_client.head_object(Bucket=args.bucket_name, Key=args.file_name, VersionId=id)
        last_modified = responses['LastModified']
        date = str(last_modified)
        date = date.split(' ', 1)[0]
        d2 = datetime.strptime(date, "%Y-%m-%d")
        d1 = datetime.strptime(today, "%Y-%m-%d")
        delta = d1 - d2
        if delta.days > args.days:
            s3_client.delete_object(Bucket=args.bucket_name, Key=args.file_name, VersionId=id)
            print(f'ვერსია {id} წაიშალა რადგან {args.days} დღეზე მეტი ხნის წინ შეიქმნა')
def upload_file_multipart(s3_client, filepath, bucket_name, file_name, metadata=None):
    MP_THRESHOLD = 1
    MP_CONCURRENCY = 5
    MAX_RETRY_COUNT = 3
    log = logging.getLogger('s3_uploader')
    log.setLevel(logging.INFO)
    format = logging.Formatter("%(asctime)s: - %(levelname)s: %(message)s", "%H:%M:%S")
    stream_handler = logging.StreamHandler(sys.stdout)
    stream_handler.setFormatter(format)
    log.addHandler(stream_handler)
    log.info("Uploading [" + args.filepath + "] to [" + args.bucket_name + "] bucket ...")
    log.info("S3 path: [ s3://" + args.bucket_name + "/" + args.file_name + " ]")
    # Multipart transfers occur when the file size exceeds the value of the multipart_threshold attribute
    if not Path(args.filepath).is_file:
        log.error("File [" + file + "] does not exist!")
        raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), file)

    if args.file_name is None:
        log.error("object_path is null!")
        raise ValueError("S3 object must be set!")

    GB = 1024 ** 3
    mp_threshold = MP_THRESHOLD*GB
    concurrency = MP_CONCURRENCY
    transfer_config = TransferConfig(multipart_threshold=mp_threshold, use_threads=True, max_concurrency=concurrency)

    login_attempt = False
    retry = MAX_RETRY_COUNT

    while retry > 0:
        try:
            s3_client.upload_file(args.filepath, args.bucket_name, args.file_name, Config=transfer_config, ExtraArgs=metadata)
            sys.stdout.write('\n')
            log.info("File [" + args.file_name + "] uploaded successfully")
            log.info("Object name: [" + args.file_name + "]")
            retry = 0

        except ClientError as e:
            log.error("Failed to upload object!")
            log.exception(e)
            if e.response['Error']['Code'] == 'ExpiredToken':
                log.warning('Login token expired')
                retry -= 1
                log.debug("retry = " + str(retry))
                login_attempt = True
                login()
            else:
                log.error("Unhandled error code:")
                log.debug(e.response['Error']['Code'])
                raise

        except boto3.exceptions.S3UploadFailedError as e:
            log.error("Failed to upload object!")
            log.exception(e)
            if 'ExpiredToken' in str(e):
                log.warning('Login token expired')
                log.info("Handling...")
                retry -= 1
                log.debug("retry = " + str(retry))
                login_attempt = True
                login()
            else:
                log.error("Unknown error!")
                raise

    if login_attempt:
        raise Exception("Tried to login " + str(MAX_RETRY_COUNT) + " times, but failed to upload!")
def static_website(s3_cient, bucket_name):
    with open(args.filepath, 'rb') as f:
        s3_cient.upload_fileobj(f, args.bucket_name, args.file_name, ExtraArgs={'ContentType': 'text/html'})
    website_configuration = {
    'ErrorDocument': {'Key': 'error.html'},
    'IndexDocument': {'Suffix': 'index.html'},
    }
    s3_client.put_bucket_website(Bucket=args.bucket_name ,WebsiteConfiguration=website_configuration)
    print("სტატიკური ვებსაიტი დაიჰოსტა წარმატებით")
def upload_and_host(s3_client, bucket_name, filepath):
    create_bucket_policy(s3_client, args.bucket_name)
    local_folder_path = args.filepath
    for root, dirs, files in os.walk(local_folder_path):
        for file in files:
            local_file_path = os.path.join(root, file)
            # Use relative path for S3 key, e.g. folder/subfolder/file.txt
            s3_key = os.path.relpath(local_file_path, local_folder_path).replace('\\', '/')
            s3_client.upload_file(local_file_path, args.bucket_name, s3_key, ExtraArgs={'ContentType': 'text/html'})

    website_configuration = {
        'ErrorDocument': {'Key': 'error.html'},
        'IndexDocument': {'Suffix': 'index.html'},
        }
    s3_client.put_bucket_website(Bucket=args.bucket_name, WebsiteConfiguration=website_configuration)
    response = s3_client.get_bucket_location(Bucket=args.bucket_name)

    region = response['LocationConstraint']

    if not region:
        region = 'us-east-1'
    print(f'http://{args.bucket_name}.s3-website-{region}.amazonaws.com')
def analytics(data):
  stats = {"quotes": 0}
  for index, each in enumerate(data):

    if not stats.get(each["author"]):
      stats[each["author"]] = {"quote_index": [index], "quotes_avaliable": 1}
    else:
      stats[each["author"]]['quote_index'].append(index)
      stats[each["author"]]['quotes_avaliable'] += 1
    stats['quotes'] += 1

  return stats
def main():
  headers = {
  'user-agent':
  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36',
   }
  json_result = []
  with urlopen(
      Request("https://type.fit/api/quotes", data=None,
              headers=headers)) as response:
    json_result = json.loads(response.read().decode())
    if args.inspire == "true":
        print(json.dumps(choice(json_result)["text"], indent=4))
    else:
        json_string = json.dumps(json_result)
        list_of_quotes = json.loads(json_string)
        for quote in list_of_quotes:
            if quote["author"] == args.inspire:
                print(quote["text"])
                if args.save == True:
                    with open("quotes.json", "w") as f:
                        json.dump(quote, f)
                    with open("quotes.json", "rb") as f:
                        s3_client.upload_fileobj(f, args.bucket_name, "quotes.json")
                        print(f'quotes.json აიტვირთა {args.bucket_name} ბაკეტში')
                break

if __name__ == "__main__":
    s3_client = init_client()

if args.tool == 'init_client' or args.tool == 'ic':
    init_client()
if args.tool == 'list_bucket' or args.tool == 'lb':
    buckets = list_buckets(s3_client)
    if buckets:
        for bucket in buckets['Buckets']:
            print(f'  {bucket["Name"]}')
if args.tool == 'create_bucket' or args.tool == 'cb':
    response = s3_client.list_buckets()
    for bucket in response['Buckets']:
        if bucket['Name'] == args.bucket_name:
            print(f'The bucket {args.bucket_name} უკვე არსებობს.')
            break
    else:
        s3_client.create_bucket(Bucket=args.bucket_name)
        print(f'bucket {args.bucket_name} შეიქმნა.')
if args.tool == 'delete_bucket' or args.tool == 'db':
    response = s3_client.list_buckets()
    for bucket in response['Buckets']:
        if bucket['Name'] == args.bucket_name:
            response = s3_client.delete_bucket(Bucket=args.bucket_name)
            print(f'bucket {args.bucket_name} წაიშალა')
            break
    else:
      print(f'bucket {args.bucket_name} არ არსებობს.')
if args.tool == 'bucket_exists' or args.tool == 'be':
    try:
        bucket_exists(s3_client, args.bucket_name)
        print(f'The bucket {args.bucket_name} არსებობს.')
    except:
        print(f'The bucket {args.bucket_name} არ არსებობს')
if args.tool == 'set_object_access_policy' or args.tool == 'soap':
    try:
        set_object_access_policy(s3_client, args.bucket_name, args.file_name)
        print("object access policy დაემატა წარმატებით")
    except:
        print("object access policy ვერ დაემატა")
if args.tool == 'generate_public_read_policy' or args.tool == 'gprp':
    try:
        generate_public_read_policy(args.bucket_name)
        print("public read policy დაგენერირდა")
    except:
        print("public read policy ვერ დაგენერირდა")
if args.tool == 'read_bucket_policy' or args.tool == 'rbp':
    read_bucket_policy(s3_client, args.bucket_name)
if args.tool == 'create_bucket_policy' or args.tool == 'cbp':
    create_bucket_policy(s3_client, args.bucket_name)
if args.tool == "download_file_and_upload_to_s3" or args.tool == 'du':
    download_file_and_upload_to_s3(s3_client, args.bucket_name, args.url, args.file_name, keep_local=False)
if args.tool == "upload" or args.tool == 'u':
    meme=args.filepath.split('.')[-1]
    if args.memetype == meme:
        upload_file(s3_client, args.bucket_name, args.file_name, args.filepath)
    else:
        print(f' {meme} გაფართების მქონე ფაილის ატვირთვა არ არის ნებადართული')
if args.tool == 'lifecycle' or args.tool == 'lc':
    lifecycle(s3_client , args.bucket_name, args.days)
if args.tool == 'big_file_upload' or args.tool == 'bfu':
    meme=args.filepath.split('.')[-1]
    if args.memetype == meme:
        big_file_upload(s3_client, args.bucket_name, args.file_name, args.filepath, args.multipart_threshold)
    else:
        print(f'{meme} გაფართების მქონე ფაილის ატვირთვა არ არის ნებადართული')
if args.tool == list_objects or args.tool =='lo':
    list_objects(s3_client, args.bucket_name)
if args.delete == True:
    delete_file(s3_client, args.bucket_name, args.file_name)
if args.versioning == True:
    versioning(s3_client, args.bucket_name)
if args.versionlist == True:
    version_list(s3_client, args.bucket_name, args.file_name)
if args.previous_version == True:
    previous_version(s3_client, args.bucket_name, args.file_name)
if args.organize_objects == True:
    organize_objects(s3_client, args.bucket_name)
if args.tool == "upload_with_magic" or args.tool == "uwm":
    upload_with_magic(s3_client, args.bucket_name, args.file_name, args.filepath)
if args.tool == "delete_old_versions" or args.tool == "dov":
    delete_old_versions(s3_client, args.bucket_name, args.file_name, args.days)
if args.tool == "upload_file_multipart" or args.tool == "ufm":
    name = args.filepath.split('/')[-1]
    type = args.filepath.split('.')[-1]
    size = os.path.getsize(args.filepath)/(1024 ** 3)
    metadata = {}
    metadata['Metadata'] = {'name': name, 'type': type,'size': f'{size}GB'}
    upload_file_multipart(s3_client, args.filepath, args.bucket_name, args.file_name, metadata=metadata)
if args.tool == "static_website" or args.tool == "sw":
    static_website(s3_client, args.bucket_name)
if args.tool == upload_and_host or args.tool == "uh":
    upload_and_host(s3_client, args.bucket_name, args.filepath)
if args.inspire is not None:
    main()


import csv

def write_appointments_to_csv(given_file, appointment_list, date_format, date_format_str):
    """
    A function that writes a list of appointments to a given file
    :param given_file: The file to which the appointments will be written
    :param appointment_list: The list of appointments to write to file
    :param date_format: The format in which dates should be written to file
    :param date_format_str: The string representation of the date format e.g dd-mm-yyyy
    """

    fieldnames = ['id', 'patient_id', 'title', 'notes', 'owner', 'appointment_date ({0})'.format(date_format_str),
                  'start_time', 'end_time', 'created_by', 'date_created ({0})'.format(date_format_str), 'modified_by',
                  'date_last_modified_by ({0})'.format(date_format_str)]
    writer = csv.DictWriter(given_file, fieldnames=fieldnames, delimiter=';')
    writer.writeheader()

    for appointment in appointment_list:
        patient = appointment.patient
        owner = appointment.owner

        # only export appointments that have been assigned to a patient
        if patient:
            writer.writerow({fieldnames[0]:appointment.id, fieldnames[1]:patient.identifier,
                             fieldnames[2]:appointment.title, fieldnames[3]:appointment.notes, fieldnames[4]:owner,
                             fieldnames[5]:appointment.appointment_date.strftime(date_format),
                             fieldnames[6]:appointment.start_time, fieldnames[7]:appointment.end_time,
                             fieldnames[8]:appointment.created_by,
                             fieldnames[9]:appointment.date_created.strftime(date_format),
                             fieldnames[10]:appointment.last_modified_by,
                             fieldnames[11]:appointment.date_last_modified.strftime(date_format)})


import torch
from torch import nn

from nets.backbone import get_feature_extractor_classifier
from nets.region_proposal_network import RegionProposalNetwork
from nets.roi_pooling import VGG16RoIHead
from targets.proposal_target_creator import ProposalTargetCreator
from utils.to_tensor import cvt_tensor

class FasterRCNN(nn.Module):

    def __init__(self,
                 feat_stride=16,
                 n_fg_class=20,
                 loc_normalize_mean=(0., 0., 0., 0.),
                 loc_normalize_std=(0.1, 0.1, 0.2, 0.2),
                 pre_train=False):
        super(FasterRCNN, self).__init__()

        self.loc_normalize_mean = loc_normalize_mean
        self.loc_normalize_std = loc_normalize_std
        self.pre_train = pre_train

        extractor, classifier = get_feature_extractor_classifier(pre_train=self.pre_train)

        self.feature_extractor = extractor
        self.rpn = RegionProposalNetwork(512,
                                         512,
                                         feat_stride=feat_stride,
                                         pre_train=self.pre_train)

        self.head = VGG16RoIHead(n_class=n_fg_class + 1,
                                 roi_size=7,
                                 spatial_scale=(1. / feat_stride),
                                 classifier=classifier,
                                 pre_train=self.pre_train)

        self.proposal_target_creator = ProposalTargetCreator(pos_ratio=0.4)

    def forward(self, x, labels, bbox, anchor, scale=1.):
        img_size = x.shape[2:]
        batch = x.shape[0]

        feature = self.feature_extractor(x)

        pred_scores, pred_locs, pred_rois, pred_roi_indices = self.rpn(feature, img_size, anchor, scale)
        sample_rois = []
        sample_roi_indices = []
        gt_roi_loc_array = []
        gt_roi_label_array = []
        for b in range(batch):
            bbox_count = len(torch.where(labels[b, ..., -1] >= 0)[0])
            roi_indices = torch.where(pred_roi_indices == b)[0]
            sample_roi, gt_roi_loc, gt_roi_label = self.proposal_target_creator(
                pred_rois[roi_indices],
                bbox[b, 0:bbox_count, ...],
                labels[b, 0:bbox_count, ...],
                self.loc_normalize_mean,
                self.loc_normalize_std)
            sample_rois.append(sample_roi)
            i = torch.full((sample_roi.shape[0], 1), b)
            sample_roi_indices.append(cvt_tensor(i.long()))
            gt_roi_loc_array.append(gt_roi_loc)
            gt_roi_label_array.append(gt_roi_label)

        sample_roi_t = torch.concat(sample_rois, dim=0)
        sample_roi_indices_t = torch.concat(sample_roi_indices, dim=0)
        sample_roi_indices_t = sample_roi_indices_t.view(sample_roi_t.shape[0])
        roi_cls_locs, roi_scores = self.head(feature, sample_roi_t, sample_roi_indices_t)
        gt_roi_locs = torch.concat(gt_roi_loc_array, dim=0)
        gt_roi_labels = torch.concat(gt_roi_label_array, dim=0)
        return pred_scores, pred_locs, roi_cls_locs, roi_scores, gt_roi_locs, gt_roi_labels

    def predict(self, x, anchor, scale=1.):
        img_size = x.shape[2:]
        feature = self.feature_extractor(x)
        pred_scores, pred_locs, pred_rois, pred_roi_indices = self.rpn(feature, img_size, anchor, scale)
        roi_cls_locs, roi_scores = self.head(feature, pred_rois, pred_roi_indices)
        return roi_cls_locs, roi_scores, pred_rois, pred_roi_indices

if __name__ == "__main__":
    f_rcnn = FasterRCNN()

    inputImage = torch.Tensor(1, 3, 800, 800).float()
    print(f_rcnn)


from setuptools import setup, find_packages

VERSION = '0.4.1'
DESCRIPTION = "detectda - detecting features in videos using TDA"
LONG_DESCRIPTION = "detectda - a cubical persistent homology package for the detection and hypothesis testing of features in greyscale videos"

setup(
	name="detectda",
	version = VERSION,
	author = "Andrew M. Thomas",
	author_email = "<me@andrewmthomas.com>",
	description = DESCRIPTION,
	long_description = LONG_DESCRIPTION,
	package=find_packages(),
	install_requires=[
		'gudhi',
		'shapely >= 2.0.1',
		'joblib',
		'scikit-image',
		'scikit-learn >= 0.23.1',
		'numpy',
		'tqdm',
        'pandas',
		'matplotlib',
		'opencv-contrib-python',
        'scipy'
	],
	keywords = ['tda', 'cubical', 'image processing'],
	classifiers = [
		'Operating System :: MacOS',
		'Operating System :: Microsoft :: Windows',
		'Programming Language :: Python'
	],
	entry_points={
		'gui_scripts': [
			'identify_polygon = detectda.idpo:identify_polygon'
		]
	}
)


import requests
from bs4 import BeautifulSoup
import re
import json

URL = 'http://testing-ground.scraping.pro/blocks'
page = requests.get(URL)

soup = BeautifulSoup(page.content, 'html.parser')

data = []
for result in soup.find_all('div', class_=re.compile('prod')):
# result = soup.find('div', class_=re.compile('prod'))
    item={}

    item["name"] = result.div.text
    print("name: " + item["name"])

    item["description"] = result.span.text
    item["description"] = item["description"].split(',')[1::]
    print("description: " + ', '.join(map(str, item["description"])))

    item["best_price"] = result.find_all('span')

    a_str = "is_best_price: "
    a_bool = True
    b_bool = False

    if item["best_price"][1].has_attr("class"):
        str_best_price = a_str + str(a_bool)
        print (str_best_price)
    else:
        str_best_price = a_str + str(b_bool)
        print (str_best_price)

    item["price"] = result.find_all('span')
    print("price: " + item["price"][1].text)

    for i in result.find_all('span'):
        if soup.find('div', class_='disc'):
            print("hello")
        else:
            print("NONONO")

    # disc = result.find_all('span')
    #
    # discount = result.disc[1].div.text
    # print("discount is here")

    print()

    data.append(item)
    # print(data)

    # with open("textbooks.json", "w") as writeJSON:
    #     json.dump(item, writeJSON, ensure_ascii=False)

# for elem in elems:
#     name = elem.find('div', class_='name')
#     # description = elem.find('div', class_='company')
#     price = elem.find('span', class_='best')
#     if None in (name, price):
#         continue
#     print(name.text.strip())
#     # print(descriptoin.text.strip())
#     print(price.text.strip())
#     print()


#pip install --upgrade google-cloud-vision

# Linux --------------------------------------------------------------------------------

# export GOOGLE_APPLICATION_CREDENTIALS="KEY_PATH"
# KEY_PATH를 서비스 계정 키가 포함된 JSON 파일의 경로로 바꿉니다.
# 예를 들면 다음과 같습니다.
# export GOOGLE_APPLICATION_CREDENTIALS="/home/user/Downloads/service-account-file.json"

# Windows --------------------------------------------------------------------------------

# PowerShell:
# $env:GOOGLE_APPLICATION_CREDENTIALS="KEY_PATH"
# KEY_PATH를 서비스 계정 키가 포함된 JSON 파일의 경로로 바꿉니다.
# 예를 들면 다음과 같습니다.
# $env:GOOGLE_APPLICATION_CREDENTIALS="C:\Users\username\Downloads\service-account-file.json"

# 명령 프롬프트:
# set GOOGLE_APPLICATION_CREDENTIALS=KEY_PATH

# ----------------------------------------------------------------------------------------
# reference : https://konlpy.org/ko/latest/install/

# Linux Mecab 설치 코드
# !sudo apt-get install g++ openjdk-8-jdk python3-dev python3-pip curl
# !python3 -m pip install --upgrade pip
# !python3 -m pip install konlpy
# !sudo apt-get install curl git
# !bash <(curl -s https://raw.githubusercontent.com/konlpy/konlpy/master/scripts/mecab.sh)

# ----------------------------------------------------------------------------------------

import io
import os
import cv2
import numpy as np
# file_path = './receipt_image.jpg'
file_path = "down/result.jpg"
# save_path = "./save_text.txt"
save_path = 'receipt_text/GCVtext_new_14.txt'

# ----------------------------------------------------------------------------------------
# ETRI 정보가 들어온 경우 이를 형태소 분류기로 분류해 해당 단어를 형태소로 나누고 이 내용이
Input_ETRI = ['블랙아메리카노','칸타타헤이즐넛',"빅컵얼음"]

def GCV_model(file_path, save_path):

    # Set environment variable
    os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "ocr1-346505-37a54a8fe96a.json"

    # Imports the Google Cloud client library
    from google.cloud import vision

    # Instantiates a client
    client = vision.ImageAnnotatorClient()

    # The name of the image file to annotate
    file_name = os.path.abspath(file_path)
    # img_array = np.fromfile(file_name, np.uint8)
    # img = cv2.imdecode(img_array, cv2.IMREAD_COLOR)
    # crop_img = cv2.resize(img, dsize=(1024, 768), fx=5, fy=5, interpolation=cv2.INTER_LINEAR)
    # Loads the image into memory
    with io.open(file_name, 'rb') as image_file:
        content = image_file.read()

    image = vision.Image(content=content)

    # Performs text detection on the image file
    response = client.text_detection(image=image)
    texts = response.text_annotations

    print('Texts:')
    print(type(texts))
    print(texts)

    # 파일 내용을 지우기 위한 용도
    with open(save_path, "w", encoding="UTF-8") as file:
        pass

    for text in texts:
        with open(save_path, "a",encoding="UTF-8") as file:
            file.write(text.description + '\n')
            # vertices = (['({},{})'.format(vertex.x, vertex.y)
            #              for vertex in text.bounding_poly.vertices])
            # file.write('bounds: {}'.format(','.join(vertices)) + '\n')
            file.close()

    if response.error.message:
        raise Exception(
            '{}\nFor more info on error messages, check: '
            'https://cloud.google.com/apis/design/errors'.format(
                response.error.message))

GCV_model(file_path,save_path)

# 이 밑 코드부터는 ETRI API로 생성된 text를 받았다고 가정하고 구성할 것이다.
# 예를 들어 CU,세븐일레븐,GS25,이마트24 편의점이 있다고 하면

with open(save_path, "r+",encoding="UTF-8") as file:
    strings = file.readlines()

def CU(Input_ETRI,strings):
    # for voice_text in Input_ETRI:
    del_index = []
    # enable = False
    for i, string in enumerate(strings):
        if 'POS' in string.strip(): #and not enable:
            del_index.append(i)
            # enable = True

        if "과세" in string.strip().strip(' ') or '부가' in string.strip().strip(' '):
            del_index.append(i)

    strings = strings[del_index[0] + 1:del_index[1]]

    print(strings)

    # length = 0
    # enable = False
    # for i, string in enumerate(strings):
    #     if string.strip().isdigit() and not enable:
    #         length = i
    #         enable = True

    digit_list1 = []
    digit_list2 = []
    text_list = []

    match_digit = [] # Input으로 들어온 ETRI정보와 일치하는 물품의 가격
    match_text = [] # Input으로 들어온 ETRI정보와 일치하는 물품의 이름

    for string in strings:
        new_string = string.strip().replace(',','').replace(' ','').replace('.','').replace('-','')
        if new_string.isdigit() or new_string == '증정' or new_string == '증점':
            if new_string == '증정' or new_string == '증점':
                digit_list2.append('증정')
            elif len(new_string) <= 2:
                digit_list1.append(int(new_string))
            else:
                digit_list2.append(int(new_string))
        else:
            text_list.append(new_string)
    print(digit_list1) # 숫자 중 개수 정보 리스트
    print(digit_list2) # 숫자 중 가격 정보 리스트
    print(text_list) # 감지한 전체 상품 리스트
    for voice_text in Input_ETRI:
        for j, text in enumerate(text_list):
            if voice_text in text:
                print('물품 :',text_list[j])
                match_digit.append(text_list[j])
                # print('개수 :',digit_list1[j])
                print('가격 :',digit_list2[j])
                match_text.append(digit_list2[j])

CU(Input_ETRI,strings)