import numpy as np
import arff
import sklearn.preprocessing

class MultiTargetDataset:

  """
  Initialize the dataset.
  """
  def __init__(self):
    self.instance_names = None
    self.features = None
    self.instances_per_file = []
    self.labels = []
    self.feature_names = []
    self.feature_types = []
    self.target_names = []
    self.target_types = []
    self.num_classes = []
    self.target2index = {}
    self.datafile_scales = []
    self.global_scale = None
    self.label_scales = []

  """
  Initializes the dataset from a spec file in CSV format,
  which contains the names of ARFF files and the # of target attributes per file.
  """
  def init_from_spec(self, spec_file):
    fh = open(spec_file)
    for line in fh:
      if line.startswith("#") or line == "":
        continue
      els = line.split(",")
      if len(els) != 2:
        raise ValueError("Invalid line in spec file: must contain two comma-separated values")
      arff_file, num_targets = els[0], int(els[1])
      self.add_arff(arff_file, num_targets)
    fh.close()

  """
  Add data from ARFF file. Supports multi-label data, missing labels,
  and completely unlabeled data (no target attributes).
  """
  def add_arff(self, arff_file, num_targets=1):
    fh = open(arff_file)
    arff_data = arff.load(fh, encode_nominal=True)
    fh.close()
    arff_values = np.array(arff_data['data'])

    # determine attributes and targets
    feature_indices = []
    target_indices = []
    feature_names = []
    target_names = []
    feature_types = []
    target_types = []
    has_instance_name = False
    for attr_index, (attr_name, attr_type) in enumerate(arff_data['attributes']):
      if attr_type == 'STRING':
        # assume first string attribute is instance name
        if attr_index == 0:
          has_instance_name = True
        # ignore all string attributes
        continue
      if attr_index >= len(arff_data['attributes']) - num_targets:
        if attr_name in target_names:
          raise ValueError("Target %s is duplicated in arff file %s" % (attr_name, arff_file))
        target_indices.append(attr_index)
        target_names.append(attr_name)
        target_types.append(attr_type)
      else:
        feature_indices.append(attr_index)
        feature_names.append(attr_name)
        feature_types.append(attr_type)

    # check feature names and types
    if self.feature_names:
      if feature_names != self.feature_names:
        raise ValueError("Feature names in arff file %s are not compatible with other features: %s != %s" % (arff_file, feature_names, self.feature_names))
    else:
      self.feature_names = feature_names
    if self.feature_types:
      if feature_types != self.feature_types:
        raise ValueError("Feature types in arff file %s are not compatible with other features: %s != %s" % (arff_file, feature_types, self.feature_types))
    else:
      self.feature_types = feature_types

    # append instance names
    if has_instance_name:
      instance_names = arff_values[:,0].astype(np.str)
    else:
      instance_names = np.array([arff_data['relation'] + '_instance_' + str(i+1) \
                                for i in range(arff_values.shape[0])])
    if self.instance_names is None:
      self.instance_names = instance_names
    else:
      self.instance_names = np.concatenate((self.instance_names, instance_names))

    # get features
    new_data = arff_values[:,feature_indices].astype(np.float64)

    # append features
    if self.features is None:
      prev_num_instances = 0
      self.features = new_data
    else:
      prev_num_instances = self.features.shape[0]
      self.features = np.vstack((self.features, new_data))
    num_instances = self.features.shape[0]
    self.instances_per_file.append(range(prev_num_instances, num_instances))

    # generate multi-task labels (one column per task)
    for target_index, target_name, target_type in zip(target_indices, target_names, target_types):
      new_labels = arff_values[:,target_index]
      # existing task --> append new labels to existing label column
      if target_name in self.target_names:
        idx = self.target2index[target_name]
        if self.target_types[idx] != target_type:
          raise ValueError("Target type mismatch in arff file %s for target %s: %s vs. %s" \
            % (arff_file, target_name, target_type, self.target_types[idx]))
        self.labels[idx] = np.concatenate((self.labels[idx], new_labels))
      # new task --> create new label column
      else:
        self.target_names.append(target_name)
        if prev_num_instances > 0:
          self.labels.append(np.concatenate(([None] * prev_num_instances, new_labels)))
        else:
          self.labels.append(new_labels)
        self.target2index[target_name] = len(self.labels) - 1
        self.target_types.append(target_type)
        self.num_classes.append(len(target_type) if isinstance(target_type, list) else 0)
    # pad label columns to the same length
    for target_index in range(len(self.labels)):
      num_labels = self.labels[target_index].shape[0]
      if num_labels != num_instances:
        self.labels[target_index] = np.concatenate((self.labels[target_index], [None] * (num_instances - num_labels)))

  """
  Write multi-target dataset to ARFF file. Supports missing labels.
  """
  def write_arff(self, arff_file, relation='multi_target'):
    # get string representation of numeric labels (class index or regression target)
    def encode_labels(labels, label_type):
      if str(label_type).upper() == 'NUMERIC':
        return labels
      elif isinstance(label_type, list): # nominal
        labels_numeric = np.copy(labels) # copy
        labels_numeric[labels == None] = -1
        label_type_with_missing = label_type + ['?']
        # note: can't index by object type array
        return np.take(label_type_with_missing, labels_numeric.astype(np.int64))
      else:
        raise ValueError("label_type = '%s' not allowed" % label_type)

    fh = open(arff_file, "w")
    arff_data = {
      'data': np.hstack((np.expand_dims(self.instance_names, axis=-1),
                         self.features,
                         np.hstack([np.expand_dims(encode_labels(self.labels[idx], self.target_types[idx]), axis=-1) \
                           for idx in range(len(self.labels))])
                       )),
      'attributes': [('instance_name', 'STRING')] \
        + list(zip(self.feature_names, self.feature_types)) \
        + list(zip(self.target_names, self.target_types)),
      'relation': relation,
      'description': 'multi-target dataset generated by openXData 0.1'
    }
    arff.dump(arff_data, fh)
    fh.close()

  """
  Determine if the task with the given index is a classification task.
  """
  def is_classification(self, task_index):
    return str(self.target_types[task_index]).upper() != 'NUMERIC'

  """
  Process the features of the dataset, applying standardization per data file.
  """
  def standardize_features_per_datafile(self):
    num_datafiles = len(self.instances_per_file)
    self.datafile_scales = []
    for i in range(num_datafiles):
      scaler = sklearn.preprocessing.StandardScaler()
      self.features[self.instances_per_file[i],:] = \
        scaler.fit_transform(self.features[self.instances_per_file[i],:])
      self.datafile_scales.append(scaler)

  """
  Process the features of a test dataset, applying standardization from the training set per data file.
  """
  def standardize_features_per_datafile_test(self, test_dataset):
    num_datafiles = len(self.instances_per_file)
    num_datafiles_other = len(test_dataset.instances_per_file)
    if num_datafiles != num_datafiles_other:
      raise ValueError("Number of data files does not match: %d vs. %d" % \
        (num_datafiles, num_datafiles_other))
    if len(self.datafile_scales) != num_datafiles_other:
      raise ValueError("Number of scales files does not match number of data files: %d vs. %d" % \
        (len(self.datafile_scales), num_datafiles_other))
    for i in range(num_datafiles):
      test_dataset.features[test_dataset.instances_per_file[i],:] = \
        self.datafile_scales[i].transform(test_dataset.features[test_dataset.instances_per_file[i],:])

  """
  Process the features of the dataset, applying global standardization.
  """
  def standardize_features_global(self):
    scaler = sklearn.preprocessing.StandardScaler()
    self.features = scaler.fit_transform(self.features)
    self.global_scale = scaler

  """
  Process the features of a test dataset, applying global standardization from the training set.
  """
  def standardize_features_global_test(self, test_dataset):
    if self.global_scale is None:
      raise ValueError("Global scale has not been computed!")
    test_dataset.features = self.global_scale.transform(test_dataset.features)

  """
  Standardize the regression labels for each task.
  """
  def standardize_labels(self):
    num_tasks = len(self.labels)
    self.label_scales = []
    for task_index in range(num_tasks):
      if self.is_classification(task_index):
        scaler = None
      else:
        defined_indices = self.labels[task_index] != None
        scaler = sklearn.preprocessing.StandardScaler()
        std_labels = scaler.fit_transform(self.labels[task_index][defined_indices].reshape(-1, 1).astype(np.float64))
        #print("DEBUG STD:", np.std(std_labels[:,0]))
        self.labels[task_index][defined_indices] = std_labels[:,0]
      self.label_scales.append(scaler)

  """
  Process the labels of a test dataset, applying standardization from the current dataset.
  """
  def standardize_labels_test(self, test_dataset):
    num_tasks = len(self.labels)
    if len(test_dataset.labels) != num_tasks:
      raise ValueError("Datasets must have the same # of tasks")
    for task_index in range(num_tasks):
      if not self.is_classification(task_index):
        defined_indices = test_dataset.labels[task_index] != None
        std_labels = self.label_scales[task_index].transform(test_dataset.labels[task_index][defined_indices].reshape(-1, 1).astype(np.float64))
        test_dataset.labels[task_index][defined_indices] = std_labels[:,0]

  """
  Invert the standardization for the regression labels of each task.
  """
  def unstandardize_labels(self):
    num_tasks = len(self.labels)
    if len(self.label_scales) != num_tasks:
      raise ValueError("Unexpected number of label scales, expected the number of tasks (%d)" % num_tasks)
    for task_index in range(num_tasks):
      if self.is_classification(task_index):
        continue
      else:
        defined_indices = self.labels[task_index] != None
        scaler = self.label_scales[task_index]
        unstd_labels = scaler.inverse_transform(self.labels[task_index][defined_indices].reshape(-1, 1).astype(np.float64))
        self.labels[task_index][defined_indices] = unstd_labels[:,0]

  """
  Return a list of pairs (task_name, num_classes) for each task in the dataset.
  """
  def get_tasks(self):
    return list(zip(self.target_names, self.num_classes))

  """
  Return the number of features in the dataset.
  """
  def get_num_features(self):
    return self.features.shape[1]

  """
  Return a list of numpy matrices, one for each task, where the first column is
  the numeric label and the second column is the label presence flag (0 = missing label).
  """
  def get_labels_and_flags(self):
    ret = []
    num_tasks = len(self.labels)
    for task_index in range(num_tasks):
      label_type = np.int64 if self.is_classification(task_index) else np.float64
      labels = np.copy(self.labels[task_index])
      labels[labels == None] = 0  # dummy label
      flags = (self.labels[task_index] != None).astype(label_type)
      labels_flags = np.hstack((
        np.expand_dims(labels.astype(label_type), axis=-1),
        np.expand_dims(flags, axis=-1)
      ))
      ret.append(labels_flags)
    return ret

  """
  Update the missing labels of the dataset w/ the most certain predictions for each task.
  """
  def update_labels_with_predictions(self, predictions, uncertainty, num_instances_to_label, standardize_predictions=False):

    num_tasks = len(self.labels)
    assert len(predictions) == num_tasks, "mismatched number of tasks in predictions"
    assert len(uncertainty) == num_tasks, "mismatched number of tasks in predictions"
    #assert percentage > 0 and percentage <= 1, "percentage must be in (0,1]"
    assert num_instances_to_label > 0
    total_unlabeled_instances = 0

    for task_index in range(num_tasks):

      num_instances = predictions[task_index].shape[0]
      assert predictions[task_index].shape[0] == self.labels[task_index].shape[0], "mismatch between number of predictions and labels"

      if standardize_predictions and not self.is_classification(task_index):
        scaler = sklearn.preprocessing.StandardScaler()
        ptask = scaler.fit_transform(predictions[task_index].reshape(-1, 1))[:,0]
      else:
        ptask = predictions[task_index]

      unlabeled_instances = list(filter(lambda i: self.labels[task_index][i] is None, range(num_instances)))
      # most certain instances out of the unlabeled ones
      num_selected_instances = min(num_instances_to_label, len(unlabeled_instances))
      print("Added labels for", num_selected_instances, "/", len(unlabeled_instances), "unlabeled instances for task", self.target_names[task_index])
      selected_instances = sorted(unlabeled_instances, \
        key=lambda i: uncertainty[task_index][i])[:num_selected_instances]
      predicted_labels = ptask[selected_instances]
      self.labels[task_index][selected_instances] = predicted_labels
      #print("Selected instances:", selected_instances, \
      #  "with uncertainties =", uncertainty[task_index][selected_instances])
      total_unlabeled_instances += (len(unlabeled_instances) - num_selected_instances)

    return total_unlabeled_instances


# -*- coding:utf-8 -*-
import numpy as np
import collections
from utils.vocab import Vocabulary
from utils.misc_utils import *

class DialogBatchInput(
    collections.namedtuple("DialogBatchInput",
                           ("source",
                            "source_length",
                            "source_topic",
                            "target_input",
                            "target_output",
                            "target_length"))):
    pass

class DialogBucket(object):
    def __init__(self, bucket_size, batch_size):
        self._bucket_size = bucket_size
        self._batch_size = batch_size
        self._bucket_data = []

    @property
    def bucket_size(self):
        return self._bucket_size

    @property
    def batch_size(self):
        return self._batch_size

    @property
    def num_samples(self):
        return len(self._bucket_data)

    def add_sample(self, sample):
        self._bucket_data.append(sample)

    def generate_batches(self, shuffle=False):
        idx = list(range(self.num_samples))
        if shuffle:
            np.random.shuffle(idx)
        batches = []
        start_idx = 0
        while start_idx < self.num_samples:
            end_idx = start_idx + self.batch_size
            if end_idx > self.num_samples:
                end_idx = self.num_samples
            batch_idx = idx[start_idx:end_idx]
            batch_data = [self._bucket_data[i] for i in batch_idx]
            batches.append(batch_data)
            start_idx += self.batch_size
        return batches

class DialogIterator(object):
    def __init__(self, dialog_data,dialog_topic,
                 batch_size,
                 sos_idx,
                 eos_idx,
                 pad_idx=0,
                 max_len=None,
                 max_turn=None,
                 infer=False,
                 shuffle=False,
                 bucket_config=None):

        self._dialog_data = dialog_data
        self._dialog_topic = dialog_topic
        self._batch_size = batch_size
        self._sos_idx = sos_idx
        self._eos_idx = eos_idx
        self._pad_idx = pad_idx
        self._max_len = max_len
        self._max_turn = max_turn
        self._infer = infer
        self._bucket_config = bucket_config
        self._shuffle = shuffle

        if not self._infer:
            # generate source response pairs
            ctx_resp_pairs = self.generate_context_response_pairs(self._dialog_data,self._dialog_topic)
            self._num_samples = len(ctx_resp_pairs)
            self._dialog_buckets = self.make_bucket_data(ctx_resp_pairs)
        else:
            self._num_samples = len(dialog_data)

    @property
    def num_samples(self):
        return self._num_samples

    def generate_context_response_pairs(self, dialog_data,dialog_topic):
        ctx_resp_pairs = []
        ignore_samples = 0
        for dialog,topic in zip(dialog_data,dialog_topic):
            turns = len(dialog)
            if turns < 2:
                ignore_samples += 1
                continue
            for i in range(turns - 1):
                reps_idx = i + 1
                ii = 0 if reps_idx <= self._max_turn else reps_idx - self._max_turn
                ctx = dialog[ii:reps_idx]
                tp = topic[ii:reps_idx]
                resp = dialog[reps_idx]
                if len(ctx) > self._max_turn:
                    raise ValueError("Max Turn Error")
                ctx_resp_pairs.append((ctx,tp, resp))

        print("generate source-response paris over; dialog samples:%d; pairs num:%d; ignored samples:%d" %
              (len(self._dialog_data), len(ctx_resp_pairs), ignore_samples))
        return ctx_resp_pairs

    def make_bucket_data(self, ctx_resp_pairs):
        # init dialog buckets from config
        dialog_buckets = []
        assert self._bucket_config is not None
        for (bucket_size, batch_size) in self._bucket_config:
            dialog_bucket = DialogBucket(bucket_size, batch_size)
            dialog_buckets.append(dialog_bucket)

        for (ctx,tp, resp) in ctx_resp_pairs:
            turn_size = len(ctx)
            for dialog_bucket in dialog_buckets:
                if turn_size > dialog_bucket.bucket_size:
                    continue
                dialog_bucket.add_sample((ctx,tp, resp))
                break
            else:
                if self._max_turn is not None:
                    assert len(ctx) <= self._max_turn
                if len(ctx) <= dialog_buckets[0].bucket_size:
                    dialog_buckets[0].add_sample((ctx,tp, resp))
                else:
                    dialog_buckets[-1].add_sample((ctx,tp, resp))
        bucket_samples = [dialog_bucket.num_samples for dialog_bucket in dialog_buckets]
        print('total bucket samples:\t', sum(bucket_samples))
        return dialog_buckets

    def _dynamic_padding(self, batch_data):
        lengths = [len(x) for x in batch_data]
        max_len = max(lengths)
        if self._max_len is not None and max_len > self._max_len:
            max_len = self._max_len

        pad_data = []
        pad_lengths = []
        for data, length in zip(batch_data, lengths):
            if length > max_len:
                pad_data += [data[:max_len]]
                pad_lengths += [max_len]
            else:
                pad_data += [data + [self._pad_idx] * (max_len - length)]
                pad_lengths += [length]

        pad_data = np.asarray(pad_data, dtype='int32')
        pad_lengths = np.asarray(pad_lengths, dtype='int32')
        return pad_data, pad_lengths

    def _dynamic_padding_multi_turn(self, batch_data,batch_topic):
        turns = [len(d) for d in batch_data]
        sent_lengths = [[len(sent) for sent in d] for d in batch_data]
        max_turn = max(turns)
        max_length = max(max(sent_lengths))

        if self._max_turn is not None and self._max_turn < max_turn:
            max_turn = self._max_turn
        if self._max_len is not None and self._max_len < max_length:
            max_length = self._max_len

        pad_dialog_data, pad_dialog_lengths,pad_dialog_topic = [], [],[]
        for turn, turn_data,turn_topic in zip(turns, batch_data,batch_topic):
            if turn > max_turn:
                turn_data = turn_data[-max_turn:]
            pad_sents = []
            pad_sent_lengths = []
            pad_sent_topic = []
            pad_sent_max_length = max_length
            for sent,tp in zip(turn_data,turn_topic):
                if len(sent) > max_length:
                    sent = sent[:max_length]

                pad_sents.append(sent + [self._pad_idx] * (pad_sent_max_length - len(sent)))
                pad_sent_lengths.append(len(sent))
                pad_sent_topic.append(tp)

            while len(pad_sents) < max_turn:
                pad_sents += [[self._pad_idx] * pad_sent_max_length]
                pad_sent_lengths += [0]
                pad_sent_topic+=[0]

            pad_dialog_data += [pad_sents]
            pad_dialog_lengths += [pad_sent_lengths]
            pad_dialog_topic+=[pad_sent_topic]

        pad_dialog_data = np.asarray(pad_dialog_data, dtype='int32')
        pad_sent_lengths = np.asarray(pad_dialog_lengths, dtype='int32')

        return pad_dialog_data,pad_dialog_topic, pad_sent_lengths

    def next_batch(self):
        batch_samples = []
        if not self._infer:
            for dialog_bucket in self._dialog_buckets:
                batch_samples += dialog_bucket.generate_batches(self._shuffle)

            # shuffle by batches
            if self._shuffle:
                np.random.shuffle(batch_samples)

            for batch_data in batch_samples:
                ctx_data = []
                ctx_topic = []
                resp_input_data = []
                resp_output_data = []
                for (ctx,tp, resp) in batch_data:
                    ctx_data.append(ctx)
                    ctx_topic.append(tp)
                    resp_input_data += [[self._sos_idx] + resp]
                    resp_output_data += [resp + [self._eos_idx]]

                pad_ctx,pad_ctx_topic, pad_ctx_length = self._dynamic_padding_multi_turn(ctx_data,ctx_topic)
                pad_resp_input, resp_length = self._dynamic_padding(resp_input_data)
                pad_resp_output, _ = self._dynamic_padding(resp_output_data)
                yield DialogBatchInput(source=pad_ctx,
                                       source_length=pad_ctx_length,
                                       source_topic = pad_ctx_topic,
                                       target_input=pad_resp_input,
                                       target_output=pad_resp_output,
                                       target_length=resp_length)

        else:
            # only use context when infer
            batch_ctx_data = []
            batch_ctx_topic = []
            for ctx_data,tp in zip(self._dialog_data,self._dialog_topic):
                batch_ctx_data += [ctx_data]
                batch_ctx_topic +=[tp]
                if len(batch_ctx_data) == self._batch_size:
                    pad_ctx,pad_ctx_topic, pad_ctx_length = self._dynamic_padding_multi_turn(batch_ctx_data,batch_ctx_topic)
                    yield DialogBatchInput(source=pad_ctx,
                                           source_length=pad_ctx_length,
                                           source_topic=pad_ctx_topic,
                                           target_input=None,
                                           target_output=None,
                                           target_length=None)
                    batch_ctx_data = []
                    batch_ctx_topic = []
            if len(batch_ctx_data) > 0:
                pad_ctx,pad_ctx_topic, pad_ctx_length = self._dynamic_padding_multi_turn(batch_ctx_data,batch_ctx_topic)
                yield DialogBatchInput(source=pad_ctx,
                                       source_length=pad_ctx_length,
                                       source_topic=pad_ctx_topic,
                                       target_input=None,
                                       target_output=None,
                                       target_length=None)

def get_dialog_data_iter(vocab: Vocabulary,
                         dialog_file,
                         batch_size,
                         max_len=None,
                         max_turn=None,
                         model="HRED",
                         infer=False,
                         shuffle=False,
                         bucket_config=None):
    dialog_data = load_dialog_data(dialog_file, '</d>')
    dialog_idx = [[vocab.convert2idx(sent[1:]) for sent in dialog] for dialog in dialog_data]
    #dialog_topic = []
    #for dialog in dialog_data:
    #    print(dialog)
    #    dialog_topic.append([sent[0] for sent in dialog])
    #print(dialog_data)
    dialog_topic = [[sent[0] for sent in dialog] for dialog in dialog_data]
    if model == 'HRED':
        data_iter = DialogIterator(dialog_data=dialog_idx,
                                   dialog_topic=dialog_topic,
                                   batch_size=batch_size,
                                   sos_idx=vocab.sos_idx,
                                   eos_idx=vocab.eos_idx,
                                   pad_idx=vocab.pad_idx,
                                   max_len=max_len,
                                   max_turn=max_turn,
                                   infer=infer,
                                   shuffle=shuffle,
                                   bucket_config=bucket_config)
        return data_iter
    else:
        raise NotImplementedError("Not Implemented Data Iterator")

def get_train_iter(data_dir, vocab, config):
    train_file = os.path.join(data_dir, 'train.%s.txt' % config.prefix)
    train_iter = get_dialog_data_iter(vocab=vocab,
                                      dialog_file=train_file,
                                      batch_size=config.batch_size,
                                      max_len=config.max_len,
                                      max_turn=config.max_turn,
                                      model=config.model,
                                      bucket_config=config.buckets,
                                      infer=False,
                                      shuffle=True)

    valid_file = os.path.join(data_dir, 'valid.%s.txt' % config.prefix)
    valid_iter = get_dialog_data_iter(vocab=vocab,
                                      dialog_file=valid_file,
                                      batch_size=config.batch_size,
                                      max_len=config.max_len,
                                      max_turn=config.max_turn,
                                      model=config.model,
                                      bucket_config=config.buckets,
                                      infer=False,
                                      shuffle=False)

    test_file = os.path.join(data_dir, 'test.%s.txt' % config.prefix)
    test_iter = get_dialog_data_iter(vocab=vocab,
                                     dialog_file=test_file,
                                     batch_size=config.batch_size,
                                     max_len=config.max_len,
                                     max_turn=config.max_turn,
                                     model=config.model,
                                     bucket_config=config.buckets,
                                     infer=False,
                                     shuffle=False)

    return train_iter, valid_iter, test_iter

def get_infer_iter(context_file, vocab, config):
    infer_iter = get_dialog_data_iter(vocab=vocab,
                                      dialog_file=context_file,
                                      batch_size=config.infer_batch_size,
                                      max_len=config.max_len,
                                      max_turn=config.max_turn,
                                      model=config.model,
                                      infer=True,
                                      shuffle=False)
    return infer_iter


# -*- coding: utf-8 -*-
"""
Created on Mon Jan  4 12:17:27 2021

@author: AYSE
"""
import time

def merge(dizi):
   # print ("parçalanan değerler " + str(dizi))
    if len(dizi) > 1:
        orta = len(dizi) // 2
        solDizi = dizi[:orta]
        sagDizi = dizi[orta:]

        merge(solDizi)
        merge(sagDizi)
        mergeSort(dizi, solDizi, sagDizi)

def mergeSort(dizi, solDizi, sagDizi):
    i = 0
    j = 0
    k = 0

    while i < len(solDizi) and j < len(sagDizi):

        if solDizi[i] < sagDizi[j]:
            dizi[k] = solDizi[i]
            i = i + 1
        else:
            dizi[k] = sagDizi[j]
            j = j + 1
        k = k + 1

    while i < len(solDizi):
        dizi[k] = solDizi[i]
        i = i + 1
        k = k + 1

    while j < len(sagDizi):
        dizi[k] = sagDizi[j]
        j = j + 1
        k = k + 1
   # print ("Birleştirilmiş hali " + str(dizi))

numbers = open("sortdata.txt").read().split()
numbers = [int(number) for number in numbers]

start_time = time.time()

print (("sırasız dizi: "), numbers)
merge(numbers)
stop_time = time.time()
print (("sıralı dizi: "), numbers)
print("geçen zaman", stop_time - start_time)


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import streamlit as st
import sys
import pandas as pd
import os

# In[2]:
st.set_option('deprecation.showPyplotGlobalUse', False)
st.enableXsrfProtection = True

import tweepy

st.markdown(
    """
    <style>
      .main{
      background-color:#E1E8ED;
      }
      </style>

    """,
    unsafe_allow_html=True
)

st.header('WELCOME TO MY PROJECT SENTIMENTAL ANALYSIS')
st.write("-----------BY NANDAN TIWARI--------------")
import streamlit as st
from datetime import date

today = date.today()
today=today.strftime("%d/%m/%Y")
st.write("DATE:-",today)

st.image("twitter.png",use_column_width=True)

# Add a selectbox to the sidebar:
add_selectbox = st.sidebar.selectbox('Select Project Running Environment',["Home",'Real time sentimental analysis',"Analysis sentiment Training data"])

def load_homepage() -> None:
    for i in range(2):
        st.write(" ")
    st.header("ðŸŽ² Dashbord for Sentiment Analysis of Twitter Data")
    st.write("Sentiment Analysis is the process of extracting the opinion or emotion from text data.   "
             "So Sometimes its called opinion mining or emotion AI. "
             "Moreover, it felt like a nice opportunity to see how much information can be "
             "extracted from relatively simple data.")

if add_selectbox=='Home':
    load_homepage()

elif (add_selectbox=='Real time sentimental analysis'):
    import os
    command="streamlit run ./t_app.py"
    os.system(command)

elif (add_selectbox=='Analysis sentiment Training data'):
    import os
    command="streamlit run ./tweet_app.py"
    os.system(command)

# In[6]:


# -*- coding: utf-8 -*-
"""
Created on Sat Nov 03 11:50:21 2018

@author: Khizar Anjum
"""

import synapseclient
import numpy as np
import sys

syn = synapseclient.login(username, password)
import shutil
#%%
def update_progress(progress):
    barLength = 30 # Modify this to change the length of the progress bar
    status = ""
    if isinstance(progress, int):
        progress = float(progress)
    if not isinstance(progress, float):
        progress = 0
        status = "error: progress var must be float\r\n"
    if progress < 0:
        progress = 0
        status = "Halt...\r\n"
    if progress >= 1:
        progress = 1
        status = "Done...\r\n"
    block = int(round(barLength*progress))
    text = "\rPercent: [{0}] {1}% {2}".format( "#"*block + "-"*(barLength-block), progress*100, status)
    sys.stdout.write(text)
    sys.stdout.flush()
#%% This segment downloads the voice activity data!
row_ids = np.arange(72137,137159);
voice_table = 'syn5511444';
columns = ['audio_audio.m4a','audio_countdown.m4a'];
folder = ['\\Audio','\\Countdown'];

for i in range(2):
    for j in row_ids:
        filepath = syn.downloadTableFile(voice_table, columns[i], downloadLocation=\
                             'E:\LUMS\Fall 2018\mPower\Data\Voice Data' + folder[i],\
                             rowId=j, versionNumber=1, \
                             ifcollision='keep.both');
        update_progress(((j+1)*(i+1))/(2*len(row_ids)));

#%% FASTER DOWNLOAD METHOD!
results = syn.tableQuery('SELECT * FROM syn5511444 LIMIT 100 OFFSET 100')
file_map = syn.downloadTableColumns(results, columns)
for file_handle_id, path in file_map.items():
    shutil.move(path,'E:\\LUMS\\Fall 2018\\mPower\\Data\\Voice Data' + folder[0] + '\\'+\
            str(file_handle_id)+'-'+path.split('\\')[-1][:-4]+'.m4a')
#%%
    '''
for file_handle_id, path in file_map.items():
    shutil.move(path,'E:\\LUMS\\Fall 2018\\mPower\\Data\\Voice Data' + folder[0] + '\\'+\
            str(file_handle_id)+'-'+path.split('\\')[-1][:-4]+'.m4a')
    '''


"""
This is my final project program for my python class
This is a program that takes in strings and evaluates them. This program is capable of performing multiple tasks.
Firstly, it could be used to check if two words are anagrams. (Anagram: a word is an anagram of another if when
rearranged can be used to form the other word). E.g., crate and trace are anagrams.
Secondly, it would be able to identify palindromes. (Palindrome: a word is a palindrome if when reversed
would give the same word as when in place). E.g., Bob is a palindrome
And lastly, the program would be capable of checking for the longest common prefix among a list of words.
E.g., if the user inputs [flower, flow, flight], the program return "fl".
Inputs = The user enters his/her choice of operation using assigned numbers; the user also provides strings for
evaluation based on what operation the user is performing
Outputs = The program displays the evaluation results for every operation performed on the strings inputted by the
user
"""

def main():
    print("THIS A VERY COOL PROGRAM THAT CAN TELL ANAGRAMS, CHECK PALINDROME, AND CHECK THE LONGEST COMMON PREFIX")
    choice()

def choice():
    try:
        user = int(input("What operation would you like to perform?\n1 - Check if two words are anagrams of each other"
                         "\n2 - Check if a word is palindromic\n3 - Check the longest common prefix in a list"
                         "\n0 - To Exit\nEnter your choice: "))
        if user == 1:
            word1 = input("Enter the first word: ")
            word2 = input("Enter the second word: ")
            anagram(word1, word2)
            print("\n")
            choice()

        elif user == 2:
            word = input("Enter the word you want to check if it's a palindrome: ")
            palindrome(word)
            print("\n")
            choice()

        elif user == 3:
            no_of_words = int(input("How many words do you want to put in the list? "))
            lst = []
            for i in range(no_of_words):
                word = input("Enter word: ")
                lst.append(word)
            longest_common_prefix(lst)
            print("\n")
            choice()

        elif user == 0:
            end_program()

        else:
            print("That is not assigned\n")
            choice()

    except:
        print("That is not a correct input!\n")
        choice()

def anagram(word1, word2):
    word_1 = word1
    word_2 = word2
    word1 = list(word1)
    word2 = list(word2)
    if len(word1) != len(word2):
        return print("OOPS, %s IS NOT AN ANAGRAM OF %s" % (word_1.upper(), word_2.upper()))

    for i in range(len(word1)):
        if word1[i] in word2:
            word2.remove(word1[i])
            i += 1
        else:
            return print("OOPS, %s IS NOT AN ANAGRAM OF %s" % (word_1.upper(), word_2.upper()))
    return print("YES, %s IS AN ANAGRAM OF %s" % (word_1.upper(), word_2.upper()))

def palindrome(word):
    c = ""
    for i in word:
        if i.isalnum():
            c += i
        else:
            continue
    y = ("".join(reversed(c)))
    if y.lower() == c.lower():
        return print("YES, %s IS A PALINDROME!" % word.upper())

    else:
        return print("OOPS, %s IS NOT A PALINDROME" % word.upper())

def longest_common_prefix(lst):
    shortest = min(lst, key=len)
    res = ""
    for i in range(len(shortest)):
        for j in range(len(lst)):
            p = lst[j]
            if shortest[i] == p[i]:
                j += 1
            else:
                if not res:
                    return print("THERE IS NO COMMON PREFIX")
                else:
                    return print("THE LONGEST COMMON PREFIX IS", res)
        res += shortest[i]
        i += 1
    return print("THE LONGEST COMMON PREFIX IS", res)

def end_program():
    try:  # keeps the program running until user enters an integer
        k = int(input("\nAre you sure you want to Exit?\n1 - Continue using program\n0 - EXIT\nEnter your choice: "))
        if k == 0:
            print("Goodbye!!!")
        elif k == 1:
            print("\n")
            main()
        else:
            print("That's not assigned")
            end_program()

    except:  # keeps the program running until user enters an integer
        print("Please enter an integer")
        end_program()

main()


from sys import getsizeof

import numpy as np
import networkx as nx
import matplotlib.pyplot as plt
import time
from memory_profiler import profile

#
# def permute(a, l, r):
#     if l == r:
#         print(a)
#     else:
#         for i in range(l, r + 1):
#             a[l], a[i] = a[i], a[l]
#             permute(a, l + 1, r)
#             a[l], a[i] = a[i], a[l]  # backtrack

class ScheduleSolver():
    def __init__(self):
        self.memory_history = []
        self.subject_domain_variables = {"ma": "Math", "ch": "Chemistry", "ph": "philosophy", "hi": "History"}  # D-all
        self.time_domain_variables = { "1": "1", "2": "2", "3": "3","4": "4", "5": "5", "6": "6", "7": "7","8": "8",}  # D-all
        self.constraints = {}
        self.total_counter = 0
        self.iterations = 0

    def permutation(self, lst,place):

        if len(lst) == 0:
            return []

        if len(lst) == 1:
            return [lst]

        l = []
        self.iterations += 1
        for i in range(len(lst)):
            m = lst[i]

            remLst = lst[:i] + lst[i + 1:]
            self.memory_history.append((remLst))

            for p in self.permutation(remLst,place-1):
                l.append([m] + p)
        return l

    def check_constraints(self,perm):
        is_correct = True
        self.total_counter += 1
        # [['8', 'hi'], ['7', 'ph'], ['6', 'ch']]
        for el in perm:
            if el[1] == 'ch':
                if int(el[0]) % 2 == 0:
                    is_correct = False
            if el[1] == 'hi':
                if int(el[0]) not in [2,4]:
                    is_correct = False
            if el[1] == 'ph':
                if int(el[0]) not in [5,7]:
                    is_correct = False

        for i in range(1,len(perm)):
            if abs(int(perm[i][0]) - int(perm[i-1][0])) == 1:
                if perm[i][1] in ['ch', 'ma'] and perm[i-1][1] in ['hi','ph']:
                    return False
                if perm[i-1][1] in ['ch', 'ma'] and perm[i][1] in ['hi','ph']:
                    return False
        if int(perm[0][0]) == 1:
            return False

        sum = 0
        for el in perm:
            if int(el[0]) > 4:
                sum += 1
            else:
                sum -= 1
        if sum not in [-3,3]:
            return False

        return is_correct

    def run_simple_backtrack(self):

        a = list(self.subject_domain_variables.keys())
        b = list(self.time_domain_variables.keys())
        a_combos = self.permutation(a,0)
        a_combos = [x[:3] for x in a_combos]
        a_final = []
        for el in a_combos:
            if el not in a_final:
                a_final.append(el)
        # print(len(a_final))

        b_combos = self.permutation(b,0)
        b_combos = [x[:3] for x in b_combos]
        b_final = []
        for el in b_combos:
            if el not in b_final:
                b_final.append(el)
        # print(len(b_final))

        results = []
        for el1 in b_final:
            for el2 in a_final:
                choice = [[el1[i],el2[i]] for i in range(3)]
                choice.sort(key=lambda x: int(x[0]), reverse=False)
                if choice not in results:
                    results.append(choice)

        # print(results)

        solutions = []
        for p in results:
            if self.check_constraints(p):
                print("_" * 6, p)
                solutions.append(p)
        print("We got total steps: ", self.total_counter)
        print("We got total choices: ", len(results))
        print("We got solutions: ", len(solutions))

@profile
def main():
    time.sleep(0.2)
    times = []
    runtimes = 1
    for i in range(runtimes):

        start_time = time.time()
        solver = ScheduleSolver()
        solver.run_simple_backtrack()

        end_time = time.time()
        print("Time that it took to run everythink: ", end_time - start_time)
        times.append(end_time - start_time)
        time.sleep(1)
        print("Iterations:", solver.iterations, " | Constraints: ", solver.total_counter)

        print("We got size consumption for the algorithm of: ", getsizeof(solver.memory_history))
    print("We got an average runtime of:", sum(times) / runtimes, " with a max of: ", max(times), " and a min of: ",
          min(times))

main()


from sys import maxsize

class Naive:
    def __init__(self, size, matrix):
        self.n = size
        self.graph = matrix
        self.s = 0
        self.min_path = maxsize

    def travellingSalesmanProblem(self):
        vertex = []
        for i in range(self.n):
            if i != self.s:
                vertex.append(i)
        self.min_path = maxsize
        while True:
            current_pathweight = 0
            k = self.s
            for i in range(len(vertex)):
                current_pathweight += self.graph[k][vertex[i]]
                k = vertex[i]
            current_pathweight += self.graph[k][self.s]
            self.min_path = min(self.min_path, current_pathweight)
            if not self.next_permutation(vertex):
                break

    def next_permutation(self, L):
        n = len(L)
        i = n - 2
        while i >= 0 and L[i] >= L[i + 1]:
            i -= 1
        if i == -1:
            return False
        j = i + 1
        while j < n and L[j] > L[i]:
            j += 1
        j -= 1
        L[i], L[j] = L[j], L[i]
        left = i + 1
        right = n - 1
        while left < right:
            L[left], L[right] = L[right], L[left]
            left += 1
            right -= 1
        return True


try:

    import zmq
    import time
    import sys

except Exception as e:

    print(e)

class forwarder(object):
    '''
    Forwarder object acts as a stable point between critical objects
    If any of the critical objects fail, this aviods an entire system shut-down
    '''

    def __init__(self,socket_number):

        self.context = zmq.Context(1)
        self.frontend_socket_number = socket_number
        self.backend_socket_number = socket_number+1

        print('Forwarder is ready')

        # Socket facing clients
        frontend = self.context.socket(zmq.SUB)
        frontend.bind("tcp://127.0.0.1:{}".format(self.frontend_socket_number))

        frontend.setsockopt_string(zmq.SUBSCRIBE, "")

        # Socket facing services
        backend = self.context.socket(zmq.PUB)
        backend.bind("tcp://127.0.0.1:{}".format(self.backend_socket_number))

        zmq.device(zmq.FORWARDER, frontend, backend)

        time.sleep(5) # Since binding takes time, sleep for a few seconds before running

if __name__ == "__main__":

    try:

        socket_number = int(sys.argv[1])

        f1 = forwarder(socket_number)

    except:

        print( 'Error in forwarder object' )

'''
About ZMQ:
https://www.randonomicon.com/zmq/2018/09/19/zmq-bind-vs-connect.html

Use BIND for stable tings & Use CONNECT for volatile things
Use BIND when listening & Use CONNECT when broadcasting
Use BIND for incoming & Use CONNECT for outgoing
Use BIND for long-lived process & Use CONNECT for short-lived process

'''


import boto3
import pyzmail
import urllib.parse
import json
import os
import time
from bs4 import BeautifulSoup
from logging import getLogger, INFO

logger = getLogger(__name__)
logger.setLevel(INFO)

s3 = boto3.resource('s3')

print('Loading function')
### lambda #############################################
def lambda_handler(event, context):

    ### initializer ####################################
    key_output = os.environ.get("KEY_TEXT_OUTPUT")

    print("============ logger.info ============")
    logger.info(json.dumps(event))

    # Get the object
    bucket = event['Records'][0]['s3']['bucket']['name']
    key = urllib.parse.unquote_plus(event['Records'][0]['s3']['object']['key'], encoding='utf-8')
    messageid = key.split("/")[1] # key sample [ses-output/********]

    print("bucket: ", bucket)
    print("key :",key)
    print("messageid :",messageid)

    try:
        response = s3.meta.client.get_object(Bucket=bucket, Key=key)
        email_body = response['Body'].read()
        email_message = pyzmail.PyzMessage.factory(email_body)
        print("email_message :",email_message)

        print("============ parse ============")
        # Extract information from the email
        email = email_message
        from_addr = email.get_address('from')
        to_addr = email.get_address('to')
        subject = email.get_subject()
        text = email.text_part.get_payload().decode(email.text_part.charset)

        print('From:', from_addr)
        print('To:', to_addr)
        print('Subject:', subject)
        print('Body:', text)

        # create put_message
        plain = extract_plain_text(text)
        print("plain :",plain)
        link_list = find_link(text)
        print("link_list :",link_list)

        attachments = [
            {
                "color": "#37438f",
                "title": subject,
                "text": plain,
                **({"fields": [
                    {
                        "title": "link " + str(index + 1),
                        "value": link,
                        "short": True
                    } for index, link in enumerate(link_list)]
                } if link_list else {}),
                "footer": "send from " + ",".join(from_addr),
                "ts": int(time.time())
            }
        ]

        attachments_json = json.dumps(attachments).encode('utf-8')

        # push bucket
        bucket_source = s3.Bucket(bucket)
        bucket_source.put_object(ACL='private',
                                 Body=attachments_json,
                                 Key=key_output + "/" + messageid + ".json",
                                 ContentType='application/json'
                                 )
        return 'end'

    except Exception as e:
        print(e)
        print('Error getting object {} from bucket {}.'.format(key, bucket))
        raise e

def extract_plain_text(html):
    after = html.replace("<br>", "\n")
    print("replaced raw: ",after)

    soup = BeautifulSoup(after, 'html.parser')

    # タグを取り除いたプレーンテキストを抽出
    plain_text = soup.get_text()

    return plain_text

def find_link(html):
    soup = BeautifulSoup(html, 'html.parser')

    link_list = []

    # リンク
    links = soup.find_all('a')
    for link in links:
        link_list.append(link.get('href'))

    return link_list


from __future__ import unicode_literals
from django.utils.encoding import python_2_unicode_compatible
import uuid
from django.db import models
from django.conf import settings
from django.utils.translation import ugettext_lazy as _
from django_countries.fields import CountryField
from organisations.models import Organisation

class InterestArea(models.Model):
    interestArea = models.TextField()

    def __str__(self):
        return f'{self.interestArea}'

class BaseProfile(models.Model):
    user = models.OneToOneField(
        settings.AUTH_USER_MODEL, on_delete=models.CASCADE, primary_key=True
    )
    surname = models.CharField(
            _("Surname"),
            max_length=200,
            blank=True,
            null=True)
    slug = models.UUIDField(
            default=uuid.uuid4,
            blank=True,
            editable=False)
    # Add more user profile fields here. Make sure they are nullable
    # or with default values
    picture = models.ImageField(
        _("Profile picture"),
        upload_to="profile_pics/%Y-%m-%d/",
        null=True,
        blank=True
    )
    title = models.CharField(_("Title"), max_length=200, blank=True, null=True)
    bio = models.TextField(
            _("Short Bio and disciplinary background"),
            max_length=2000,
            blank=True,
            null=True)
    interestAreas = models.ManyToManyField(InterestArea, blank=True)
    orcid = models.CharField(_(
        "ORCID (If you have registered on the ORCID platform for researchers and have"
        "a persistent digital identifier (your ORCID iD) you can add it here to link this"
        "profile with your professional information such as affiliations, grants, publications,"
        "peer review, and more.)"),
        max_length=50,
        blank=True,
        null=True)
    organisation = models.ManyToManyField(Organisation, blank=True)
    email_verified = models.BooleanField(_("Email verified"), default=False)
    country = CountryField(
            blank_label='(select country)',
            null=True,
            blank=True)

    # Privacy and subscriptions
    profileVisible = models.BooleanField(
            default=False)
    contentVisible = models.BooleanField(
            default=True)
    digest = models.BooleanField(
            default=True)

    # Permission to manage projects from a country
    manageProjectsFromCountry = CountryField(null=True, blank=True)

    class Meta:
        abstract = True

@python_2_unicode_compatible
class Profile(BaseProfile):
    def __str__(self):
        return "{}'s profile".format(self.user)


from policyengine_us.model_api import *

class ar_personal_credits_base(Variable):
    value_type = float
    entity = TaxUnit
    label = "Arkansas base personal credits"
    unit = USD
    definition_period = YEAR
    reference = (
        "https://www.dfa.arkansas.gov/images/uploads/incomeTaxOffice/2021_AR1000F_FullYearResidentIndividualIncomeTaxReturn.pdf"
        "https://www.dfa.arkansas.gov/images/uploads/incomeTaxOffice/2022_AR1000F_FullYearResidentIndividualIncomeTaxReturn.pdf#page=1"
        "https://www.dfa.arkansas.gov/images/uploads/incomeTaxOffice/2022_AR1000F_and_AR1000NR_Instructions.pdf#page=12"
    )
    defined_for = StateCode.AR

    def formula(tax_unit, period, parameters):
        person = tax_unit.members
        # Only head and spouse are eligible for the personal credit amounts
        head_or_spouse = person("is_tax_unit_head_or_spouse", period)
        p = parameters(period).gov.states.ar.tax.income.credits.personal
        aged = person("age", period) >= p.age_threshold
        # Arkansas provides an additional "aged special" credit for people who
        # do not receive retirement or disability benefit exemptions.
        receives_retirement_or_disability_exemption = (
            person(
                "ar_retirement_or_disability_benefits_exemptions_indv", period
            )
            > 0
        )
        aged_special = aged & ~receives_retirement_or_disability_exemption
        # Blind filers get an additional personal tax credit amount
        blind = person("is_blind", period)
        # Deaf filers get an additional personal tax credit amount
        deaf = person("is_deaf", period)
        # Widowed and head of household filers receive an additional credit amount
        filing_status = tax_unit("filing_status", period)
        statuses = filing_status.possible_values
        widow = filing_status == statuses.WIDOW
        hoh = filing_status == statuses.HEAD_OF_HOUSEHOLD
        filing_status_eligible = widow | hoh

        personal_credit_count = tax_unit.sum(
            head_or_spouse
            * (
                1
                + aged.astype(int)
                + blind.astype(int)
                + deaf.astype(int)
                + aged_special.astype(int)
            )
        ) + filing_status_eligible.astype(int)
        return personal_credit_count * p.amount.base


import re

from model.ScopusQueries import ScopusQueries

class QueryConverter:

    @property
    def query(self):
        return self._query

    @property
    def scopus_queries(self):
        if self._query is not None:
            self.calculate_scopus_queries()
        return self._scopus_queries

    @property
    def mycore_queries(self):
        return self._mycore_queries

    def __init__(self, query):
        self._query = query
        self._scopus_queries = ScopusQueries
        self._mycore_queries = None

    def calculate_scopus_queries(self):
        scopus_queries = ScopusQueries()
        overall_filter_strings = get_filter_strings(self._query.query_definitions.query_filters)
        for query_definition in self._query.query_definitions.query_definition:
            queries = []
            for overall_filter_string in overall_filter_strings:
                overall_query_string = overall_filter_string
                query_length = len(overall_filter_string)
                query_string = overall_filter_string
                for query_line in query_definition.query_lines:
                    query_term = ''
                    if query_line.query_line is not '':

                        # generate query term for individual query line
                        query_term = query_line.field + '(' + clean_up_line(query_line.query_line) + ')'

                        # add query term to overall query string. if it is the first entry, connect by AND and open bracket,
                        # else add it by OR
                        if overall_query_string is overall_filter_string:
                            overall_query_string = ' AND (' + query_term
                        else:
                            overall_query_string += ' OR ' + query_term

                        # add the query term to the query string. these are designed to be below the limit of 2800
                        # for the scopus api. If the addition of the query term would exceed this limit, the line is closed
                        # and added to the scopus search strings, otherwise it is added to the current search line.
                        if (query_length + len(query_term)) < 2800:
                            # add query term to query string. if it is the first entry, connect by AND and open bracket,
                            # else add it by OR
                            if query_string is overall_filter_string:
                                query_string += ' AND (' + query_term
                            else:
                                query_string += ' OR ' + query_term
                        else:
                            queries.append(query_string + ')')
                            query_string = overall_filter_string + '(' + query_term
                    if query_string is not '':
                        queries.append(query_string + ')')
                        query_string = overall_filter_string + '(' + query_term

            scopus_queries.add_search_string(queries)
            scopus_queries.add_search_id(str(self._query.identifier) + '.' + str(query_definition.identifier))
        self._scopus_queries = scopus_queries
        return self._scopus_queries

def clean_up_line(line):
    new_line = re.sub(' +', ' ', line).replace('\n', '').replace('\t', '').replace('\u201c', '"') \
        .replace('\u201d', '"').replace('\u2018', '"').replace('\u2019', '"').strip()
    return new_line

def get_filter_strings(query_filters):
    list_of_filter_lists = []
    timerange_filter = get_timerange_filter_string(query_filters.timerange)
    if timerange_filter != '':
        list_of_filter_lists.append([timerange_filter])
    for query_filter in query_filters.query_filters:
        if query_filter.filter_type == 'subjectarea':
            subjectearea_filter = get_subjectarea_filter(query_filter)
            if subjectearea_filter != '':
                list_of_filter_lists.append([subjectearea_filter])
        else:
            list_of_filter_lists.append(get_filter_list(query_filter))
    return get_filter_lists(list_of_filter_lists)

def get_timerange_filter_string(timerange):
    string = ''
    if timerange is not None:
        if timerange.field is not '':
            string = '(' + timerange.field + ' AFT ' + timerange.start
            if timerange.end is not '':
                string = string + ' AND ' + timerange.field + ' BEF ' + timerange.end
            string = string + ')'
    return clean_up_line(string)

def get_subjectarea_filter(query_filter):
    string = '('
    for i, subjects in enumerate(query_filter.filter_term.split(' OR ')):
        if i > 0:
            string += ' OR '
        string += ' LIMIT-TO(SUBJAREA, \'' + query_filter.filter_term + '\')'
    string += ')'
    return clean_up_line(string)

def get_filter_list(query_filter):
    filter_list = []
    for filter_term in query_filter.filter_term.split('|'):
        string = '({}({}))'.format(query_filter.filter_field, filter_term)
        string = clean_up_line(string.replace(' OR ', ') OR ' + query_filter.filter_field + '('))
        filter_list.append(string)
    return filter_list

def get_filter_lists(list_of_filter_lists):
    list = []
    for filter_list in list_of_filter_lists:
        if len(list) == 0:
            list = filter_list
        else:
            list = [' and '.join((i, j)) for i in list for j in filter_list]
    return list


import os
import subprocess

__docformat__ = "restructuredtext"

def __get_command_output(command_string, cwd=None):
    """ Execute arbitrary commands.

    Parameters
    ----------
    command_list : strings
        the command and its arguments

    cwd : string
        the directory where the command should be executed

    Returns
    -------
    output : string
        the raw output of the command executed
    """

    command_list = command_string.split(' ')
    p = subprocess.Popen(command_list, stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE, cwd=cwd)
    p.wait()
    return p.returncode, p.stdout.read(), p.stderr.read()

class __GitException(Exception):
    pass

def __get_git_output(command, directory):
    try:
        ret_code, output, error = __get_command_output('git %s' % command,
                                                       cwd=directory)
    except OSError as e:
        # if git is not installed, an OSError is raised
        raise __GitException(e)
    if ret_code != 0:
        raise __GitException(error)
    else:
        return output.strip()

def __is_git_repo(directory):
    try:
        __get_git_output('rev-parse', directory)
    except __GitException:
        return False
    else:
        return True

def __git_describe(directory):
    return __get_git_output('describe --always --dirty', directory)

def version():
    pisak_dir = os.path.dirname(__file__)
    if __is_git_repo(pisak_dir):
        return __git_describe(pisak_dir)
    else:
        return ''


import sys

n,m=map(int,sys.stdin.readline().split())
parent=[i for i in range(n+1)]
num=[1]*(n+1)

def find(x):
    if parent[x]!=x:
        parent[x]=find(parent[x])
    return parent[x]

def unite(a,b):
    a=find(a)
    b=find(b)

    if b>a:
        parent[b]=a
        num[a]+=num[b]
        num[b]=0
    elif a>b:
        parent[a]=b
        num[b]+=num[a]
        num[a]=0

for i in range(m):
    a,b=map(int,sys.stdin.readline().split())
    unite(a,b)

ctp,hansol,k=map(int,sys.stdin.readline().split())

ctp_num=find(ctp)
hansol_num=find(hansol)

v=[]
for i in range(1,n+1):
    if parent[i]!=i:
        continue
    x=find(i)

    if x==ctp_num or x==hansol_num:
        continue

    v.append(num[x])

v.sort(reverse=True)

for i in range(k):
    num[ctp_num]+=v[i]

print(num[ctp_num])


import pytest

from api.neo4j import get_driver
from api.dao.ratings import RatingDAO

movie = '769'
user = '1185150b-9e81-46a2-a1d3-eb649544b9c4'
email = 'graphacademy.reviewer@neo4j.com'
rating = 5

@pytest.fixture(autouse=True)
def before_all(app):
    with app.app_context():
        driver = get_driver()

        def merge_data(tx):
            return tx.run("""
            MERGE (u:User {userId: $user})
            SET u.email = $email
            MERGE (m:Movie {tmdbId: $movie})
            """, user=user, movie=movie, email=email).consume()

        with driver.session() as session:
            session.execute_write(merge_data)
            session.close()

def test_store_integer(app):
    with app.app_context():
        driver = get_driver()

        dao = RatingDAO(driver)

        output = dao.add(user, movie, rating)

        assert output["tmdbId"] == movie
        assert output["rating"] is rating


import torch
import torch.nn as nn

class Classifier(nn.Module):
    def __init__(self):
        super(Classifier, self).__init__()
        self.encoder = nn.Sequential(
            nn.Conv2d(1, 32, kernel_size=3, stride=2, padding=1),
            nn.LeakyReLU(),
            nn.BatchNorm2d(32),
            nn.Conv2d(32, 64, kernel_size=3, stride=2, padding=1),
            nn.LeakyReLU(),
            nn.BatchNorm2d(64),
            nn.Conv2d(64, 128, kernel_size=3, stride=2, padding=1),
            nn.LeakyReLU(),
            nn.BatchNorm2d(128),
            nn.Flatten(),
            nn.Linear(128 * 32 * 32, 256),
            nn.LeakyReLU(),
        )
        self.model = nn.Sequential(
            self.encoder,
            nn.Linear(256, 100),
            nn.LeakyReLU(),
            nn.Linear(100, 2),
            nn.LeakyReLU(),
        )

    def forward(self, x):
        x = self.model(x)
        x = nn.functional.softmax(x, dim=1)
        x = torch.argmax(x, dim=1)
        return x


import sklearn
import pandas as pd
import numpy as np
import catboost
import random
from sklearn.model_selection import train_test_split
from sklearn.metrics import *
def get_n_best(arg_sphere, arg_numbest):
  data_bez_con=pd.read_csv('model_dataset.csv')
  data_bez_con.drop(['Unnamed: 0'], axis=1, inplace=True)
  data_bez_con.columns=list(range(0,10))
  cb=catboost.CatBoostClassifier(cat_features=[0,1,7,10])
  cb.load_model('modelcb.cbm')
  ans_d=data_bez_con.copy()
  ans_d[10]=[arg_sphere for i in range(ans_d.shape[0])]
  a=pd.read_csv('data_with_concur.csv')
  a.drop(['Unnamed: 0'], axis=1, inplace=True)
  preds=cb.predict_proba(ans_d)[:,1]
  ress=preds.argsort()[-1*arg_numbest:][::-1]
  answer=[]
  for i in ress:
    answer.append({'lat':sum(list(map(float,data_bez_con.iloc[i][0][1:-1].split(', '))))/2,
     'lng':sum(list(map(float,data_bez_con.iloc[i][1][1:-1].split(', '))))/2,
     'info':f'Оценка: {100*preds[i]}, Количество конкурентов рядом: {a.iloc[i][arg_sphere]}, Среднее количество людей в час: {round(data_bez_con.iloc[i][2],1)}, Средняя ценя аренды за год за кв. м. {data_bez_con.iloc[i][9]}'})
  return answer


from typing import List, Dict, Optional, Iterable, Tuple
from itertools import chain, cycle
from functools import partial
from multiprocessing import Manager, Pool, Queue, cpu_count

import numpy as np
import simpy
from . import simulation as cs
from .cache import get_from_cache, save_to_cache
from .early_stop import EarlyStopError, process_early_stop
from .lab import laboratory
from .parameters import Parameters
from .population import Population
from .progress import ProgressBar
from .random import RandomParametersState, RandomParameter
from .stats import Stats
from .simulation_environment import SimulationEnvironment
from .metrics import METRICS

SIMULATION_ENGINE_VERSION = '0.0.3'
MAX_WAIT_UNTIL_D0 = 90

def get_stats_matrix(populations: Dict, duration):
    num_populations = len(populations)
    num_metrics = len(cs.MEASUREMENTS)
    num_ages = len(cs.age_str)
    stats = np.zeros([num_populations, num_metrics, num_ages, duration])
    return stats

def track_population(senv: SimulationEnvironment):
    while True:
        yield senv.env.timeout(1.0)
        if senv.d0 is None:
            if senv.sim_params.d0_infections * senv.scaling < np.array([p.infected for p in senv.people]).sum():
                senv.d0 = int(senv.env.now + 0.01)
            else:
                continue
        if int(senv.env.now + 0.01) - senv.d0 >= senv.duration:
            return
        cs.log_stats(senv)
        try:
            if senv.simulation_queue:
                senv.simulation_queue.put(1)
        except TypeError:  # Communication error with progress bar
            pass

def get_house_size(house_sizes):  # Number of people living in the same house
    return cs.p_choice(house_sizes)

def get_age_group(age_probabilities, age_risk):
    return age_risk[cs.p_choice(age_probabilities)]

def set_initial_infection(sim_params: Parameters, people: Iterable[cs.Person]):
    success = False
    while not success:
        someone = cs.choice(people, 1)[0]
        if someone.age_group.index < sim_params.min_age_group_initially_infected:
            continue
        success = someone.expose_to_virus()

def get_population(senv: SimulationEnvironment, population_params: Population) -> \
        List:
    people = []
    n = int(population_params.inhabitants * senv.scaling)
    initially_infected = population_params.seed_infections
    while len(people) < n:
        people.extend(generate_people_in_new_house(senv, population_params))
    for _ in range(initially_infected):
        set_initial_infection(senv.sim_params, people)
    try:
        if senv.creation_queue:
            senv.creation_queue.put(len(people))
    except TypeError:  # Communication error with progress bar
        pass
    return people

def generate_people_in_new_house(senv: SimulationEnvironment, population_params: Population):
    house_size = get_house_size(population_params.home_size_probabilities)
    house = cs.Home(population_params.geosocial_displacement)
    age_probabilities = population_params.age_probabilities
    age_groups = population_params.age_groups
    age_group_house = get_age_group(age_probabilities, age_groups)
    home_age_cofactor = senv.sim_params.home_age_cofactor
    for _ in range(house_size):
        age_group = (age_group_house
                     if np.random.random() < home_age_cofactor
                     else get_age_group(age_probabilities, age_groups)
                     )
        yield cs.Person(senv, age_group, house)

def add_randomness_to_age_group(senv: SimulationEnvironment, age_group, population_params: Population, i):
    severity = np.array(age_group.severity)
    age_bias = senv.sim_params.severity_bias * (i - 4)
    new_odds = np.exp(np.log(severity / (1.0 - severity)
                             ) + senv.sim_params.severity_deviation + age_bias)
    age_group.severity = new_odds / (1.0 + new_odds)
    if isinstance(population_params.isolation_propensity_increase, RandomParameter):
        population_params.isolation_propensity_increase = population_params.isolation_propensity_increase
    age_group.isolation_adherence += population_params.isolation_propensity_increase

def create_populations(senv: SimulationEnvironment) -> Dict[str, List[cs.Person]]:
    populations = {}
    for population_params in senv.sim_params.population_segments:
        for i, age_group in enumerate(population_params.age_groups):
            add_randomness_to_age_group(senv, age_group, population_params, i)
        populations[population_params.name] = get_population(senv, population_params)
    return populations

def simulate(
        sim_number,
        sim_params,
        simulation_size,
        duration,
        simulate_capacity,
        use_cache,
        creation_queue: Optional[Queue] = None,
        simulation_queue: Optional[Queue] = None,
) -> (np.ndarray, RandomParametersState):
    if use_cache:
        args = (
            sim_number, sim_params, simulation_size, duration, simulate_capacity, SIMULATION_ENGINE_VERSION)
        results = get_from_cache(args)
        if results:
            try:
                if creation_queue:
                    creation_queue.put(simulation_size)
                if simulation_queue:
                    simulation_queue.put(duration)
            except TypeError:  # Communication error with progress bar
                pass
            return results[1]

    cs.seed(sim_number)
    np.random.seed(sim_number)
    scaling = simulation_size / sim_params.total_inhabitants
    env = simpy.Environment()
    sim_params = sim_params.clone()
    sim_params.random_parameters_state.materialize_object(sim_params)
    senv = SimulationEnvironment(
        env=env,
        sim_params=sim_params,
        duration=duration,
        sim_number=sim_number,
        scaling=scaling,
        simulate_capacity=simulate_capacity,
        isolation_factor=0.0,
        attention=simpy.resources.resource.PriorityResource(env,
                                                            capacity=int(sim_params.capacity_hospital_max * scaling)),
        hospital_bed=simpy.resources.resource.PriorityResource(env,
                                                               capacity=int(
                                                                   sim_params.capacity_hospital_beds * scaling)),
        ventilator=simpy.resources.resource.PriorityResource(env,
                                                             capacity=int(sim_params.capacity_ventilators * scaling)),
        icu=simpy.resources.resource.PriorityResource(env, capacity=int(sim_params.capacity_icu * scaling)),
        stats=get_stats_matrix(sim_params.population_segments, duration),
        street_expositions_interval=sim_params.street_transmission_scale_days,
        social_group_expositions_interval=(
                sim_params.street_transmission_scale_days
                + sim_params.social_group_transmission_scale_difference
        ),
        creation_queue=creation_queue,
        simulation_queue=simulation_queue,
        lab=laboratory(env, scaling),
    )
    senv.populations = create_populations(senv)
    senv.people = list(chain.from_iterable(senv.populations.values()))
    env.process(track_population(senv))
    for intervention in sim_params.interventions:
        intervention.setup(senv)
    for early_stop in sim_params.early_stops or []:
        env.process(process_early_stop(senv, early_stop))
    while not senv.d0:
        if env.now < MAX_WAIT_UNTIL_D0:
            env.run(until=env.now + 1)
        else:
            senv.d0 = MAX_WAIT_UNTIL_D0
    try:
        env.run(until=duration + senv.d0 + 0.011)
    except EarlyStopError:
        pass
    stats = senv.stats / senv.scaling
    if use_cache:
        save_to_cache(args, (stats, sim_params.random_parameters_state))
    return stats, sim_params.random_parameters_state

def simulate_wrapped(i_params, **kwargs):
    return simulate(*i_params, **kwargs)

def get_sim_params_list(sim_params: Parameters, random_states: List[RandomParametersState], n: int) -> List[
    Tuple[int, Parameters]]:
    random_states_iter = cycle(random_states or [RandomParametersState()])
    sim_params_list = []
    for random_state, i in zip(random_states_iter, range(n)):
        sim_params_with_state = sim_params.clone()
        sim_params_with_state.random_parameters_state = random_state
        sim_number = int(i / len(random_states)) if random_states else i
        sim_params_list.append((sim_number, sim_params_with_state))
    return sim_params_list

def run_simulations(
        sim_params: Parameters,
        simulate_capacity=False,
        duration: int = 80,
        number_of_simulations: int = 4,  # For final presentation purposes, a value greater than 10 is recommended
        simulation_size: int = 100000,  # For final presentation purposes, a value greater than 500000 is recommended
        random_states: Optional[List[RandomParametersState]] = None,
        fpath=None,
        use_cache=True,
        tqdm=None,  # Optional tqdm function to display progress
):
    if tqdm:
        manager = Manager()
        creation_queue = manager.Queue()
        simulation_queue = manager.Queue()

    sim_params_list = get_sim_params_list(sim_params, random_states, number_of_simulations)

    simulate_with_params = partial(simulate_wrapped,
                                   simulation_size=simulation_size,
                                   duration=duration,
                                   simulate_capacity=simulate_capacity,
                                   use_cache=use_cache,
                                   creation_queue=creation_queue if tqdm else None,
                                   simulation_queue=simulation_queue if tqdm else None,
                                   )
    try:
        pool = Pool(min(cpu_count(), number_of_simulations))
        all_stats = pool.imap(simulate_with_params, sim_params_list)
        if tqdm:
            creation_bar, simulation_bar = show_progress(tqdm, creation_queue, simulation_queue, simulation_size,
                                                         number_of_simulations, duration)
            creation_bar.start()
            simulation_bar.start()
        all_stats = list(all_stats)
    finally:
        pool.close()
        pool.join()
        if tqdm:
            creation_bar.stop()
            creation_bar.join()
            simulation_bar.stop()
            simulation_bar.join()
    stats = combine_stats(all_stats, sim_params)
    if fpath:
        stats.save(fpath)
    return stats

def combine_stats(all_stats: List[Tuple[np.ndarray, RandomParametersState]], sim_params: Parameters):
    mstats = np.stack([stats[0] for stats in all_stats])
    random_states = [stats[1] for stats in all_stats]
    population_names = tuple(p.name for p in sim_params.population_segments)
    return Stats(mstats, random_states, cs.MEASUREMENTS, METRICS, population_names, cs.age_str,
                 start_date=sim_params.start_date)

def show_progress(tqdm, creation_queue: Queue, simulation_queue: Queue, simulation_size: int,
                  number_of_simulations: int, duration: int):
    creation_bar = ProgressBar(tqdm, creation_queue, simulation_size * number_of_simulations, 0, 'Population')
    simulation_bar = ProgressBar(tqdm, simulation_queue, duration * number_of_simulations, 1, 'Simulation')
    return creation_bar, simulation_bar


from typing import Type

from fastapi_pagination import Page
from fastapi_pagination import Params as PaginationParams
from fastapi_pagination.ext.sqlalchemy import paginate
from sqlalchemy import and_, select
from sqlalchemy.ext.asyncio import AsyncSession

from src.config.database import Base
from src.config.exception import NotFoundException
from src.config.logger import logger

class BaseRepository:
    def __init__(self, session: AsyncSession = None, model: Type[Base] = None) -> None:
        self.__session = session
        self.__model = model

    def set_session(self, session: AsyncSession):
        self.__session = session

    async def get_all_paginate(self, params: PaginationParams, filters: list = None) -> Page[Base]:
        query = select(self.__model)
        if filters:
            query = query.where(and_(*filters))
        query = query.order_by(self.__model.id.desc())
        items = await paginate(self.__session, query, params)
        logger.info("Retrieve %s items from database", self.__model.__name__)
        return items

    async def add_one(self, model: Base) -> None:
        self.__session.add(model)
        await self.__session.flush()
        await self.__session.refresh(model)
        logger.info("%s saved in database", model.__class__.__name__)

    async def delete_one(self, filters: list) -> None:
        query = select(self.__model).where(and_(*filters))
        result = await self.__session.execute(query)
        to_delete = result.scalars().first()
        if not to_delete:
            raise NotFoundException()
        to_delete.deleted = True
        logger.info("%s marked as deleted in database, guid=%s", to_delete.__class__.__name__, to_delete.guid)


import xml.etree.ElementTree as ET

# 遍历节点函数
def read():
    # 创建XMLXML文档节点树
    tree = ET.parse("deliveries.xml")
    # root是根节点
    root = tree.getroot()
    # 获取所有的article节点。此处要注意findall中要写XPath表达式
    article_node_list = root.findall("article")
    print(article_node_list)
    # 遍历所有的article节点
    for article_node in article_node_list:
        print("------记录------")
        id = article_node.attrib["id"]
        print("id : {}".format(id))
        #遍历article节点所有子节点
        for node in article_node:
            # 判断是否是price节点
            if node.tag == "price":
                # 取出unitprice属性
                unitprice = node.attrib["unitprice"]
                print("unitprice : {}".format(unitprice))
            print("{} : {}".format(node.tag, node.text))

# 插入节点
def insert():
    # 创建XML文档节点树
    tree = ET.parse("deliveries.xml")
    # root是根节点
    root = tree.getroot()
    # 创建一个新的article节点对象
    new_article_node = ET.Element("article")
    new_article_node.attrib["id"] = "123456"
    # 创建一个新的name节点对象
    new_name_node = ET.SubElement(new_article_node, "name")
    new_name_node.text = "pears"
    # 创建一个新的price节点对象
    new_price_node = ET.SubElement(new_article_node, "price")
    new_price_node.text = "20"
    new_price_node.attrib["unitprice"] = "true"
    # 创建一个新的supplier节点对象
    new_supplier_node = ET.SubElement(new_article_node, "supplier")
    new_supplier_node.text = "Pear Inc."
    # 将article节点添加到根节点deliveries中
    root.append(new_article_node)
    # 保存到XML文件中
    tree.write("deliveries_2.xml", "utf-8")

# 删除节点
def remove():
    # 创建XML文档节点树
    tree = ET.parse("deliveries.xml")
    # root是根节点
    root = tree.getroot()
    # XPath表达式
    expression = ".//price[@unitprice='true']/.."
    # # 找到复合表达式的第一个
    # article_node = root.find(expression)
    # # 删除节点
    # root.remove(article_node)
    # # 保存到XML文件中国
    # tree.write("./deliveries_3.xml", "utf-8")
    article_nodes = root.findall(expression)
    for article_node in article_nodes:
        root.remove(article_node)
    tree.write("./deliveries_4.xml", "utf-8")

# 主函数
def main():
    read()
    # insert()
    remove()

if __name__ == '__main__':
    main()


# To train and test non-isotropic randomized smoothing.

from mnist_train import Net
from smoothing import Smooth
from datasets import get_dataset, imagenet_trainset, get_input_dim, get_num_classes, KittiDataset
from architectures import get_architecture

import argparse
import matplotlib.pyplot as plt
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torchvision import datasets, transforms, models
from torchvision.utils import save_image
from torch.optim.lr_scheduler import StepLR
from torch.utils.tensorboard import SummaryWriter

# CUSTOM: Add an option for your model
def get_dataset_name(model):
    """Converts model name to corresponding dataset name.
    Args:
        model (str): Name of model.
    Returns:
        (str) Name of dataset.
    """
    if model == "mnist":
        return "mnist"
    elif model == "fashion_mnist":
        return "fashion_mnist"
    elif model == "cifar10" or model == "cifar10_robust":
        return "cifar10"
    elif model == "imagenet" or model == "imagenet_robust":
        return "imagenet"
    elif model == "kitti":
        return "kitti"

# CUSTOM: Add an option for your model
def load_dataset(dataset, batch_size, test_batch_size, use_cuda):
    """Loads the train and test loader for a dataset.
    Args:
        dataset (str): Name of dataset.
        batch_size (int): Batch size of train set.
        test_batch_size (int): Batch size of test set.
        use_cuda (bool): Use cuda if true.
    Returns:
        (torch.utils.data.DataLoader, torch.utils.data.DataLoader)
        Train dataset dataloader, test dataset dataloader.
    """
    if dataset == "mnist":
        kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}
        train_loader = torch.utils.data.DataLoader(
            datasets.MNIST('../data', train=True, download=True,
                        transform=transforms.Compose([
                            transforms.ToTensor(),
                            transforms.Normalize((0.1307,), (0.3081,))
                        ])),
            batch_size=batch_size, shuffle=True, **kwargs)
        test_loader = torch.utils.data.DataLoader(
            datasets.MNIST('../data', train=False, transform=transforms.Compose([
                            transforms.ToTensor(),
                            transforms.Normalize((0.1307,), (0.3081,))
                        ])),
            batch_size=test_batch_size, shuffle=True, **kwargs)
    elif dataset == "fashion_mnist":
        kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}
        train_loader = torch.utils.data.DataLoader(
            datasets.FashionMNIST('datasets', train=True, download=True,
                        transform=transforms.Compose([
                            transforms.ToTensor(),
                            transforms.Normalize((0.1307,), (0.3081,))
                        ])),
            batch_size=batch_size, shuffle=True, **kwargs)
        test_loader = torch.utils.data.DataLoader(
            datasets.FashionMNIST('datasets', train=False, transform=transforms.Compose([
                            transforms.ToTensor(),
                            transforms.Normalize((0.1307,), (0.3081,))
                        ])),
            batch_size=test_batch_size, shuffle=True, **kwargs)
    elif dataset == "cifar10":
        kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}
        train_loader = torch.utils.data.DataLoader(get_dataset("cifar10", "train"), batch_size=batch_size, shuffle=True, **kwargs)
        test_loader = torch.utils.data.DataLoader(get_dataset("cifar10", "test"), batch_size=test_batch_size, shuffle=True, **kwargs)
    elif dataset == "imagenet":
        kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}
        train_set, test_set = imagenet_trainset()
        train_loader = torch.utils.data.DataLoader(train_set, batch_size=batch_size, shuffle=True, **kwargs)
        test_loader = torch.utils.data.DataLoader(test_set, batch_size=test_batch_size, shuffle=True, **kwargs)
    elif dataset == "kitti":
        kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}
        data_path = 'data/train/'
        dataset = KittiDataset('../datasets/kitti')
        # print(len(dataset))
        train_set, test_set = torch.utils.data.random_split(dataset, [6733, 748]) # 7481
        train_loader = torch.utils.data.DataLoader(train_set, batch_size=batch_size, shuffle=True, **kwargs)
        test_loader = torch.utils.data.DataLoader(test_set, batch_size=test_batch_size, shuffle=True, **kwargs)
    else:
        raise Exception("Must enter a valid dataset name")
    return train_loader, test_loader

# CUSTOM: Add an option for your model
def load_model(model_name, device):
    """Loads the model.
    Args:
        model_name (str): Name of model.
        device (torch.device): Device to load the model onto.
    Returns:
        (nn.Module) Loaded Pytorch model.
    """
    if model_name == "mnist":
        model = Net().to(device)
        model.load_state_dict(torch.load('models/mnist_cnn.pt'))
    elif model_name == "fashion_mnist":
        model = Net().to(device)
        model.load_state_dict(torch.load('models/fashion_mnist_cnn.pt'))
    elif model_name == "cifar10":
        checkpoint = torch.load("models/pretrained_models/cifar10/finetune_cifar_from_imagenetPGD2steps/PGD_10steps_30epochs_multinoise/2-multitrain/eps_64/cifar10/resnet110/noise_0.12/checkpoint.pth.tar")
        model = get_architecture(checkpoint["arch"], "cifar10")
        # model.load_state_dict(checkpoint['state_dict'])
        model.load_state_dict(torch.load('models/cifar10.pt'))
    elif model_name == "cifar10_robust":
        checkpoint = torch.load("models/pretrained_models/cifar10/finetune_cifar_from_imagenetPGD2steps/PGD_10steps_30epochs_multinoise/2-multitrain/eps_512/cifar10/resnet110/noise_0.12/checkpoint.pth.tar")
        model = get_architecture(checkpoint["arch"], "cifar10")
        model.load_state_dict(checkpoint['state_dict'])
    elif model_name == "imagenet":
        # checkpoint = torch.load("models/pretrained_models/imagenet/PGD_1step/imagenet/eps_127/resnet50/noise_0.25/checkpoint.pth.tar")
        # model = get_architecture(checkpoint["arch"], "imagenet")
        # model.load_state_dict(checkpoint['state_dict'])
        model = models.resnet50(pretrained=True).cuda()
    elif model_name == "imagenet_robust":
        checkpoint = torch.load("models/pretrained_models/imagenet/PGD_1step/imagenet/eps_1024/resnet50/noise_0.25/checkpoint.pth.tar")
        model = get_architecture(checkpoint["arch"], "imagenet")
        model.load_state_dict(checkpoint['state_dict'])
    elif model_name == "kitti":
        model = models.resnet50(pretrained=False).cuda()
        model.load_state_dict(torch.load('models/kitti.pt'))
    else:
        raise Exception("Must enter a valid model name")
    return model

def calculate_objective(sigma, icdf_pabar):
    """Calculates the certified area objective.
    Args:
        sigma (torch.tensor): Sigma vector.
        icdf_pabar (torch.tensor/np.array/float): Inverse Gaussian CDF of paBar.
    Returns:
        (torch.tensor) Certified area objective, single element tensor.
    """
    sigma = torch.abs(sigma)  # For log calculation. Negative or positive makes no difference in our formulation.
    eps = 0.000000001 # To prevent log from returning infinity.
    if isinstance(icdf_pabar, float):
        icdf_pabar = torch.tensor(icdf_pabar)
    elif not torch.is_tensor(icdf_pabar):
        icdf_pabar = torch.tensor(icdf_pabar.item())
    objective = torch.sum(torch.log(sigma+eps)) + torch.numel(sigma) * torch.log(icdf_pabar+eps)
    return objective

def train(args, smoothed_classifier, device, train_loader, optimizer, epoch, writer):
    """Optimizes sigma for the certified area objective, writes results to tensorboard.
    Args:
        args (argparse.ArgumentParser): Arguments containing N_train and batch_smooth.
        smoothed_classifier (smoothing.Smooth): Randomized smoother.
        device (torch.device): Device to load the data onto.
        train_loader (torch.utils.data.DataLoader): Train dataset loader.
        optimizer (torch.optim.Optimizer): Optimizer for smoother's sigma vector.
        epoch (int): Epoch of optimization process.
        writer (torch.utils.tensorboard.SummaryWriter): Writer for tensorboard data.
    """
    avg_objective = torch.tensor([0.0]).cuda()
    avg_accuracy = 0
    for batch_idx, (data, target) in enumerate(train_loader):
        data, target = data.to(device), target.to(device)
        optimizer.zero_grad()
        objective = torch.tensor([0.0]).cuda()
        accuracy = 0
        for i in range(data.shape[0]):
            prediction, icdf_pabar = smoothed_classifier.certify_training(data[i], args.N_train, args.batch_smooth, target[i])
            if prediction == target[i]:  # Add 0 to all if it predicts wrong.
                accuracy += 1
                objective += calculate_objective(smoothed_classifier.sigma, icdf_pabar)
        avg_objective += objective
        avg_accuracy += accuracy
        if accuracy != 0:
            objective /= accuracy  # Want to average objectives that are actually certified
            accuracy /= data.shape[0]
            loss = -objective
            loss.backward()
            optimizer.step()
        if batch_idx % args.log_interval == 0:
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tObjective: {:.6f}'.format(
                epoch, batch_idx * len(data), len(train_loader.dataset),
                100. * batch_idx / len(train_loader), objective.item()))
            print('Accuracy: {:.5f} \t sigma mean: {:.5f} \t sigma stddev: {:.5f}'.format(
                accuracy,
                torch.abs(smoothed_classifier.sigma).mean().item(),
                torch.abs(smoothed_classifier.sigma).std().item()))
    # Write avg objective and accuracy to tensorboard
    avg_objective /= avg_accuracy  # Divide by number of correctly classified points
    avg_accuracy /= len(train_loader.dataset)
    writer.add_scalar('objective/train', avg_objective, epoch-1)
    writer.add_scalar('accuracy/train', avg_accuracy, epoch-1)

def test(args, smoothed_classifier, device, test_loader, epoch, writer, comment):
    """Tests smoothed classifer for certified area objective, writes results to tensorboard.
    Args:
        args (argparse.ArgumentParser): Arguments containing N0, N, alpha, and batch_smooth.
        smoothed_classifier (smoothing.Smooth): Randomized smoother.
        device (torch.device): Device to load the data onto.
        test_loader (torch.utils.data.DataLoader): Test dataset loader.
        epoch (int): Epoch of optimization process.
        writer (torch.utils.tensorboard.SummaryWriter): Writer for tensorboard data.
        comment (str): String to append to use for saved sigma filenames.
    """
    objective = 0
    accuracy = 0
    with torch.no_grad():
        for data, target in test_loader:
            data, target = data.to(device), target.to(device)
            for i in range(data.shape[0]):
                prediction, icdf_pabar = smoothed_classifier.certify(data[i], args.N0, args.N, args.alpha, args.batch_smooth)
                if prediction == target[i]:  # Add 0 to all if it predicts wrong.
                    accuracy += 1
                    objective += calculate_objective(smoothed_classifier.sigma, icdf_pabar)
        objective /= accuracy  # Want to average objectives that are actually certified
        accuracy /= len(test_loader.dataset)
        print('\nAverage Test objective: {:.4f}'.format(objective))
        print('Percent correct: {:.4f}'.format(accuracy))
        print('Sigma avg: {:.4f}\n'.format(torch.abs(smoothed_classifier.sigma).mean()))
        writer.add_scalar('objective/test', objective, epoch-1)
        writer.add_scalar('accuracy/test', accuracy, epoch-1)
        writer.add_scalar('sigma/mean', torch.abs(smoothed_classifier.sigma).mean(), epoch-1)
        writer.add_scalar('sigma/stddev', torch.abs(smoothed_classifier.sigma).std(), epoch-1)
        sigma_img = torch.abs(smoothed_classifier.sigma)  # For image. Negative or positive makes no difference in our formulation.
        sigma_img = sigma_img - sigma_img.min()
        sigma_img = sigma_img / sigma_img.max()
        writer.add_image('sigma_linear_normalized', sigma_img, epoch-1)
        if args.save_sigma:
            torch.save(smoothed_classifier.sigma, 'models/sigmas/sigma' + comment + '_EPOCH_' + str(epoch) + '.pt')

def main():
    parser = argparse.ArgumentParser(description='Optimize certified area')
    parser.add_argument('--model', type=str,
                        help='filepath to saved model parameters')
    parser.add_argument('--lr', type=float, default=0.001,
                        help='learning rate')
    parser.add_argument('--gamma', type=float, default=0.8,
                        help='learning rate step gamma')
    parser.add_argument('--batch_size', type=int, default=2,
                        help='input batch size for training')
    parser.add_argument('--test_batch_size', type=int, default=2,
                        help='input batch size for testing')
    parser.add_argument("--sigma", type=float, default=0.01,
                        help="constant elements in sigma vector are initialized to")
    parser.add_argument("--sigma_add", type=float, default=0.01,
                        help="amount to add to sigma per sub-epoch if doing sigma modulation")
    parser.add_argument('--epochs', type=int, default=200,
                        help='number of epochs to train')
    parser.add_argument('--sub_epochs', type=int, default=10,
                        help='number of epochs to optimize for per initialization if doing sigma modulation')
    parser.add_argument('--comment_add', type=str, default="",
                        help='string to add to the end of name used for tensorboard and sigma filesaves')
    parser.add_argument('--sigma_mod', action='store_true', default=False,
                        help='modulate sigma, re-initialize after sub-epochs to train multiple times')
    parser.add_argument('--save_sigma', action='store_true', default=True,
                        help='save the sigma vector')
    parser.add_argument("--batch_smooth", type=int, default=32,
                        help="batch size for smoothed classifer when sampling random noise")
    parser.add_argument("--N0", type=int, default=32,
                        help='number of samples used in when estimating smoothed classifer prediction')
    parser.add_argument("--N", type=int, default=256,
                        help="number of samples used when estimating paBar")
    parser.add_argument("--N_train", type=int, default=32,
                        help="number of samples to use in training when the smoothed classifer samples noise")
    parser.add_argument("--alpha", type=float, default=0.001,
                        help="probability that paBar is not a true lower bound on pA")
    parser.add_argument('--no_cuda', action='store_true', default=False,
                        help='disables CUDA training')
    parser.add_argument('--seed', type=int, default=1,
                        help='random seed')
    parser.add_argument('--log-interval', type=int, default=10,
                        help='how many batches to wait before logging training status')
    args = parser.parse_args()

    comment = '_MODEL_' + args.model + '_LR_' + str(args.lr) + '_GAMMA_' + str(args.gamma)
    if args.sigma_mod:
        comment = comment + '_SIGMA_MOD'
    comment = comment + args.comment_add

    torch.manual_seed(args.seed)
    use_cuda = not args.no_cuda and torch.cuda.is_available()
    device = torch.device("cuda" if use_cuda else "cpu")
    train_loader, test_loader = load_dataset(get_dataset_name(args.model), args.batch_size, args.test_batch_size, use_cuda)
    # for batch_idx, (data, target) in enumerate(train_loader):
    #     if batch_idx < 10:
    #         print(batch_idx, data.size(), target)
    model = load_model(args.model, device)
    model.eval()
    writer = SummaryWriter(comment=comment)
    if args.sigma_mod:
        sigma = args.sigma
        for epoch in range(1, args.epochs + 1):
            smoother = Smooth(model, sigma=sigma,
                              num_classes=get_num_classes(get_dataset_name(args.model)),
                              data_shape=get_input_dim(get_dataset_name(args.model)))
            optimizer = optim.Adam([smoother.sigma], lr=args.lr)
            scheduler = StepLR(optimizer, step_size=1, gamma=args.gamma)
            for sub_epoch in range(1, args.sub_epochs + 1):
                effective_epoch = (epoch - 1) * args.sub_epochs + sub_epoch
                writer.add_scalar('sigma_mod/sigma_init', sigma, effective_epoch-1)
                train(args, smoother, device, train_loader, optimizer, effective_epoch, writer)
                test(args, smoother, device, test_loader, effective_epoch, writer, comment)
                scheduler.step()
            sigma += args.sigma_add
    else:
        smoother = Smooth(model, sigma=args.sigma,
                          num_classes=get_num_classes(get_dataset_name(args.model)),
                          data_shape=get_input_dim(get_dataset_name(args.model)))
        optimizer = optim.Adam([smoother.sigma], lr=args.lr)
        scheduler = StepLR(optimizer, step_size=1, gamma=args.gamma)
        for epoch in range(1, args.epochs + 1):
            train(args, smoother, device, train_loader, optimizer, epoch, writer)
            test(args, smoother, device, test_loader, epoch, writer, comment)
            scheduler.step()
    writer.close()

if __name__ == '__main__':
    main()


import matplotlib.pyplot as plt
import numpy as np

from step_1 import get_observations

n = 400
observations, increments = get_observations(0.6, 0.9, n, 1)

def get_fbm(increments):
    result = [0]
    value = 0
    for i in increments:
        value += i
        result.append(value)
    return result

fig, ax = plt.subplots()
x = list(np.linspace(0, 1, n))
ax.plot(x, observations, c="blue")
ax.plot(x, get_fbm(increments), c="green")
ax.set_xlabel("t")
ax.set_ylabel("Y")
plt.show()


from django.urls import path

from . import views

urlpatterns = [
    path('api/notes/<int:note_id>/', views.api_note),
    path('api/notes/', views.api_notes),
    path('details', views.details, name="details"),
    path('tags', views.tags, name="tags"),
    path('update', views.update, name="update"),
    path('add', views.add, name="add"),
    path('delete', views.delete, name="delete"),
    path('', views.index, name='index')
]