"""
Created on 03/03/23

@author: dimiboeckaerts

PhageDEPOdetection DATA PROCESSING
"""

# 0 - INPUTS
# --------------------------------------------------------------------------------
"""
Raw data:
    - protein sequences (FASTA) from MillardLab phage genomes that had a hit in the interpro domains
    - protein IDs & linked interpro domains (dict)
    - protein IDs & annotations (dict)
    - protein ESM2 embeddings for affinity propagation clustering
"""
# data paths
absolute_path = '/Users/dimi/Documents/GitHub/PhageDEPOdetection/'
protein_domains_path = absolute_path+'data/dbsuite_results.v2.json'
protein_df_path = absolute_path+'data/df_sequences.index.v2.csv'
annotations_path = absolute_path+'data/proteinID_annotation.v2.json'
protein_embeddings_path = absolute_path+'data/embeddings.proteins.v2.csv'

# domains and annotations to remove or keep
domains_to_remove = ['IPR000490', 'IPR000852', 'IPR001088', 'IPR001137',
                     'IPR001547', 'IPR004185', 'IPR004888', 'IPR006048',
                     'IPR006425', 'IPR008902', 'IPR011496']
domains_to_check = ['IPR000165', 'IPR000757', 'IPR000922', 'IPR001139',
                    'IPR001329', 'IPR001371', 'IPR001439', 'IPR001554',
                    'IPR005199', 'IPR006065', 'IPR007724', 'IPR007781',
                    'IPR008291', 'IPR008929', 'IPR010702', 'IPR010905',
                    'IPR011613']
annotations_to_remove = ['ribonucleoside', 'diphosphate', 'reductase', 'endolysin',
                         'RecA', 'UvsX-like', 'DNA helicase', 'Hoc-like head',
                         'epimerase', 'DNA repair', 'DNA annealing']

# 1 - LIBRARIES
# --------------------------------------------------------------------------------
import json
import numpy as np
import pandas as pd
from Bio import SeqIO
from sklearn.cluster import AffinityPropagation

# 2 - MAIN
# --------------------------------------------------------------------------------
"""
STEP 1: filter at the domain level
    - remove all interpro domains with not a single congruent phage depo annotation (manual)
    - add interpro domains to remove to a list, add questionable domains to a second list
    - remove the sequences linked to the interpro domains to remove
"""
print('Reading the data...')
# Read the data
with open(protein_domains_path) as f1:
    domains_dict = json.load(f1)
with open(annotations_path) as f2:
    annotations_dict = json.load(f2)
embeddings = pd.read_csv(protein_embeddings_path, header=None).iloc[:,:-1]
seq_df = pd.read_csv(protein_df_path, sep='\t', header=None)

# loop over the domains_dict and collect the protein IDs to remove or to check
proteins_to_remove = []
proteins_to_check = []
for proteinid in domains_dict.keys():
    domains_list = [domain.split('__')[0] for domain in domains_dict[proteinid]]
    if any([domain in domains_list for domain in domains_to_remove]):
        proteins_to_remove.append(proteinid)
    elif any([domain in domains_list for domain in domains_to_check]):
        proteins_to_check.append(proteinid)

"""
STEP 2: filter at the individual sequence level
    - doublecheck the sequences linked to the questionable domains
    - do affinity propagation clustering on the embeddings of all protein sequences
    - check the clusters in which questionable proteins are present
    - get all of the annotations of the proteins in those clusters
    - if any of the annotations of the proteins in the cluster are in the
        list of annotations to remove, than we remove the questionable protein
"""
print('Clustering with affinity propagation...')
# do affinity propagation with all of the sequences and their embeddings
X = embeddings.iloc[:, 1:]
af = AffinityPropagation(damping=0.90, preference=None, random_state=123, max_iter=1000,verbose=True).fit(X)
cluster_centers_indices = af.cluster_centers_indices_
cluster_ids = af.labels_

# loop over the proteins in the list of proteins to check
for protein in proteins_to_check:
    # retrieve the embeddings indices (column 0 in seq_df) of the proteins to check
    protein_id = int(seq_df.loc[seq_df.iloc[:,1]== protein, 0])

    # retrieve the cluster_ids using these embeddings indices
    embedding_index = list(embeddings.iloc[:,0]).index(protein_id)
    cluster_id = cluster_ids[embedding_index]

    # retrieve all of the protein_ids that are in that same cluster
    cluster_protein_ids = embeddings.loc[cluster_ids == cluster_id,0]

    # get the protein names for the proteins in the cluster
    protein_names = [list(seq_df.loc[seq_df.iloc[:,0]== this_id, 1]) for this_id in cluster_protein_ids]
    protein_names = [item for sublist in protein_names for item in sublist] # flatten the list

    # get the annotations for the proteins in the cluster
    annotations = [annotations_dict[protein] for protein in protein_names]

    # check if any of the annotations are in the list of annotations to remove
    # by checking the words of annotations_to_remove in each of the indiv annotations
    if any([annotation_rm in anno for anno in annotations for annotation_rm
            in annotations_to_remove]):
        proteins_to_remove.append(protein)
    # if only 'hypothetical' or 'unknown' is present as annotation in the cluster, remove the protein
    elif all([(anno == 'hypothetical protein' or anno == 'unknown function') for anno in annotations]):
        proteins_to_remove.append(protein)

"""
STEP 3: constructing the final database

IDs - Interpro entry - sequences - domains - annotations
"""
final_ids = []
final_IPRs = []
final_sequences = []
final_annotations = []
for proteinid in domains_dict.keys():
    if proteinid not in proteins_to_remove:
        final_ids.append(proteinid)
        final_IPRs.append(domains_dict[proteinid])
        final_sequence = seq_df.loc[seq_df.iloc[:,1]== proteinid, 2].values[0]
        final_sequences.append(final_sequence)
        final_annotations.append(annotations_dict[proteinid])

final_database = {'protein_ID': final_ids, 'IPRs': final_IPRs,
                  'sequence': final_sequences, 'annotation': final_annotations}

"""
STEP 4: Annotate domains and save dataframe for further use

Ideally, we do this with ResDom, but let's just keep it simple for now and follow the
simple rule that if a protein < 250 AAs, it is considered as a depo domain in its entirety,
and if it is larger than 250 AAs, the C-terminal part is considered as the depo domain.
"""
# identify domains with simple 200 AA cutoff
print('Annotating the domains...')
depo_domains = []
token_labels = []
count = 0
for sequence in final_database['sequence']:
    if len(sequence) < 250:
        depo_domains.append(sequence)
        token_labels.append([1]*len(sequence))
    else:
        depo_domains.append(sequence[200:])
        token_labels.append([0]*200 + [1]*(len(sequence)-200))

final_database['depo_domain'] = depo_domains
final_database['token_labels'] = token_labels
dbdf = pd.DataFrame(final_database)

# save the dataframe as json
print('Saving the final database...')
final_database_path = absolute_path+'data/final_database.json'
with open(final_database_path, 'w') as f:
    json.dump(final_database, f)

# save the database in format for finetuning
"""
https://huggingface.co/docs/datasets/loading#specify-features
{"version": "0.1.0",
 "train": [{"id": 1, "tokens":['M', 'L', 'P', ...], "labels": [0, 0, 1, 1, ...]},
          {"id": 322, "tokens": [...], "labels": [...]}]
 "test": ...
}
"""
max_length = 1024 # max length for ESM2
finetune_data = {'version': '0.1.0', 'train': [], 'test': []}
dbdf = dbdf.sample(frac=1).reset_index(drop=True) # shuffle
train_cutoff = int(dbdf.shape[0]*0.75)
for i, pid in enumerate(dbdf['protein_ID']):
    tokens = list(dbdf['sequence'][i])[:max_length]
    labels = dbdf['token_labels'][i][:max_length]
    this_data = {'id': pid, 'tokens': tokens, 'labels': labels}
    if i <= train_cutoff:
        finetune_data['train'].append(this_data)
    else:
        finetune_data['test'].append(this_data)

finetune_data_path = absolute_path+'data/finetune_data.json'
with open(finetune_data_path, 'w') as f:
    json.dump(finetune_data, f)

"""
Ideas & Remarks:
    - pre processing: get everything in one dataframe already so we only have
    one object to work with (i.e. annotation & linked domains)
    - better decision filter 2: keep the ambiguous individual sequences if they appear in a cluster in
    which approved interpro domains are present?!
    - for now, the reclustering is skipped. Looking at all the annotation within one cluster,
    it seems that the clusters are already quite specific, thus might be overkill to recluster?
    - due to the max_length cutoff, we're actually losing a part of the data.
    we could also add the remaining part as an extra data point to avoid throuwing away data
"""


import os
from os import listdir
from os.path import isfile, join
import tifffile as tiff
import numpy as np
from tqdm import tqdm
from skimage.color import rgb2lab
from skimage import exposure
import cv2
from PIL import Image

dataset = input('Potsdam (p) or Vaihingen (v) dataset? ')
while True:
    if dataset == 'p':
        path_img = '/home/mdias/datasets/potsdam/Images/'
        new_path_img = '/home/mdias/datasets/potsdam/Images_lab_hist/'
        break
    elif dataset == 'v':
        path_img = '/home/mdias/datasets/vaihingen/Images/'
        new_path_img = '/home/mdias/datasets/vaihingen/Images_l_eq_hist/'
        break
    else:
        dataset = input('p or v?')

if not os.path.exists(new_path_img): os.makedirs(new_path_img)
files = [f for f in listdir(path_img) if isfile(join(path_img, f))]
for f in tqdm(files):
    if '.tif' in f:
        img = tiff.imread(path_img+f)
        p2, p98 = np.percentile(img, (2, 98))
        new_img = exposure.rescale_intensity(img, in_range=(p2, p98))
        lab_img = rgb2lab(new_img)
        lab_img[:, :, 0] = lab_img[:, :, 0]/100
        lab_img[:, :, 1] = np.interp(lab_img[:, :, 1], (-128, 128), (0, 1))
        lab_img[:, :, 2] = np.interp(lab_img[:, :, 2], (-128, 128), (0, 1))
        #lab_img = np.expand_dims(lab_img[:, :, 0], axis=2)
        print(lab_img.shape)
        tiff.imsave(new_path_img+f, lab_img)


"""
  Implements a GP for neural networks. The kernels are taken from nn_comparators.
  -- kandasamy@cs.cmu.edu
"""

# pylint: disable=invalid-name
# pylint: disable=abstract-class-not-used
# pylint: disable=arguments-differ

import numpy as np
# Local
from gp import gp_core
from gp.gp_instances import basic_gp_args
from nn import nn_comparators
from utils.ancillary_utils import get_list_of_floats_as_str
from utils.reporters import get_reporter
from utils.option_handler import get_option_specs, load_options

_DFLT_KERNEL_TYPE = 'lpemd_sum'

nn_gp_specific_args = [
  get_option_specs('dist_type', False, 'lp-emd',
    'The type of distance. This should be lp, emd or lp-emd.'),
# Use given coeffcients by default
  get_option_specs('choose_mislabel_struct_coeffs', False, 'use_given',
    ('How to choose the mislabel and struct coefficients. Should be one of ' +
     'tune_coeffs or use_given. In the latter case, mislabel_coeffs and struct_coeffs ' +
     'should be non-empty.')),
  get_option_specs('compute_kernel_from_dists', False, True,
    'Should you compute the kernel from pairwise distances whenever possible.'),
  get_option_specs('mislabel_coeffs', False, '1.0-1.0-1.0-1.0',
    'The mislabel coefficients specified as a string. If -1, it means we will tune.'),
  get_option_specs('struct_coeffs', False, '0.1-0.25-0.61-1.5',
    'The struct coefficients specified as a string. If -1, it means we will tune.'),
  get_option_specs('lp_power', False, 1,
    'The powers to use in the LP distance for the kernel.'),
  get_option_specs('emd_power', False, 2,
    'The powers to use in the EMD distance for the kernel.'),
  get_option_specs('non_assignment_penalty', False, 1.0,
    'The non-assignment penalty.'),
  ]

nn_gp_args = gp_core.mandatory_gp_args + basic_gp_args + nn_gp_specific_args

# Main GP class for neural network architectures -----------------------------------------
class NNGP(gp_core.GP):
  """ A Gaussian process for Neural Networks. """

  def __init__(self, X, Y, kernel_type, nn_type,
               kernel_hyperparams, mean_func, noise_var,
               dist_type, tp_comp=None, list_of_dists=None, *args, **kwargs):
    # pylint: disable=too-many-arguments
    """ Constructor.
        kernel_type: Should be one of 'from_distance_computer', 'lp', 'emd', 'sum_lp-emd',
                     'product_lp-emd'
        kernel_hyperparams: is a python dictionary specifying the hyper-parameters for the
                            kernel. Will have parameters 'beta', 'scale', 'sum_scales',
                            'powers'.
        list_of_dists: Is a list of distances for the training set.
        tp_comp: a transport distance computer object.
    """
    kernel = self._get_kernel_from_type(kernel_type, nn_type, kernel_hyperparams,
                                        tp_comp, dist_type)
    self.nn_type = nn_type
    self.list_of_dists = list_of_dists
    # Call super constructor
    super(NNGP, self).__init__(X, Y, kernel, mean_func, noise_var,
                               handle_non_psd_kernels='project_first', *args, **kwargs)

  def build_posterior(self):
    """ Checks if the sizes of list of distances and the data are consistent before
        calling build_posterior from the super class.
    """
    if self.list_of_dists is not None:
      assert self.list_of_dists[0].shape == (len(self.X), len(self.Y))
    super(NNGP, self).build_posterior()

  def set_list_of_dists(self, list_of_dists):
    """ Updates the list of distances. """
    self.list_of_dists = list_of_dists

  def _get_training_kernel_matrix(self):
    """ Compute the kernel matrix from distances if they are provided. """
    if self.list_of_dists is not None:
      return self.kernel.evaluate_from_dists(self.list_of_dists)
    else:
      return self.kernel(self.X, self.X)

  def _child_str(self):
    """ Description of the child GP. """
    scales_str = self._get_scales_str()
    betas_str = 'betas=' + get_list_of_floats_as_str(self.kernel.hyperparams['betas'])
    return self.kernel.dist_type + ':: ' + scales_str + ' ' + betas_str

  def _get_scales_str(self):
    """ Returns a description of the alphas. """
    if 'alphas' in self.kernel.hyperparams.keys():
      return 'alphas=' + get_list_of_floats_as_str(self.kernel.hyperparams['alphas'])
    else:
      return 'scale=' + str(round(self.kernel.hyperparams['scale'], 4))

  @classmethod
  def _get_kernel_from_type(cls, kernel_type, nn_type, kernel_hyperparams,
                            tp_comp, dist_type):
    """ Returns the kernel. """
    if tp_comp is None:
      tp_comp = nn_comparators.get_otmann_distance_from_args(nn_type,
                  kernel_hyperparams['non_assignment_penalty'])
    # Now construct the kernel.
    if kernel_type in ['lpemd_prod', 'lp', 'emd']:
      return nn_comparators.DistProdNNKernel(tp_comp,
              kernel_hyperparams['mislabel_coeffs'],
              kernel_hyperparams['struct_coeffs'],
              kernel_hyperparams['betas'],
              kernel_hyperparams['scale'],
              kernel_hyperparams['powers'],
              dist_type)
    elif kernel_type in ['lpemd_sum']:
      return nn_comparators.DistSumNNKernel(tp_comp,
              kernel_hyperparams['mislabel_coeffs'],
              kernel_hyperparams['struct_coeffs'],
              kernel_hyperparams['alphas'],
              kernel_hyperparams['betas'],
              kernel_hyperparams['powers'],
              dist_type)
    else:
      raise ValueError('Unknown kernel_type %s.'%(kernel_type))

# GP Fitter for neural network architectures ---------------------------------------------
class NNGPFitter(gp_core.GPFitter):
  """ Fits a GP by tuning the kernel hyper-params. """
  # pylint: disable=attribute-defined-outside-init

  def __init__(self, X, Y, nn_type, tp_comp=None, list_of_dists=None,
               options=None, reporter=None, *args, **kwargs):
    """ Constructor. """
    self.X = X
    self.Y = Y
    self.nn_type = nn_type
    self.reporter = get_reporter(reporter)
    self.list_of_dists = list_of_dists
    self.num_data = len(X)
    self.tp_comp = tp_comp
    if options is None:
      options = load_options(nn_gp_args, 'GPFitter', reporter=reporter)
    super(NNGPFitter, self).__init__(options, *args, **kwargs)

  def _child_set_up(self):
    """ Sets up child-specific parameters for the GP Fitter. """
    # pylint: disable=too-many-branches
    # Check args
    if not self.options.dist_type in ['lp', 'emd', 'lp-emd']:
      raise ValueError('Unknown dist_type. Should be one of \'lp\', \'emd\', \'lp-emd\'.')
    if not self.options.kernel_type in ['lp', 'emd', 'lpemd_sum', 'lpemd_prod',
                                        'default']:
      raise ValueError('Unknown dist_type. Should be one of \'lp\', \'emd\', ' +
                       '\'lpemd_sum\', or \'lpemd_prod\'.')
    if not self.options.noise_var_type in ['tune', 'label', 'value']:
      raise ValueError('Unknown noise_var_type. Should be either tune, label or value.')
    if not self.options.mean_func_type in ['mean', 'median', 'const', 'zero']:
      raise ValueError('Unknown mean_func_type. Should be one of mean/median/const/zero.')
    if not self.options.choose_mislabel_struct_coeffs in ['use_given', 'tune_coeffs']:
      raise ValueError('Unknown value for choose_mislabel_struct_coeffs. Should be one ' +
                       'of use_given or tune_coeffs.')
    # Set kernel type
    if self.options.kernel_type == 'default':
      self.kernel_type = _DFLT_KERNEL_TYPE
    else:
      self.kernel_type = self.options.kernel_type
    # Check if dist_type and kernel_type are consistent
    if self.options.dist_type in ['lp', 'emd'] and \
       self.kernel_type != self.options.dist_type:
      raise ValueError('If dist_type is %s, then kernel_type should be %s.'%(
                       self.options.dist_type, self.options.dist_type))
    elif self.options.dist_type == 'lp-emd' and \
      self.kernel_type not in ['lpemd_prod', 'lpemd_sum']:
      raise ValueError('If dist_type is lp-emd, then kernel_type should be lpemd_sum or '
                       + 'lpemd_prod.')
    if self.options.choose_mislabel_struct_coeffs == 'use_given' and \
      (self.options.mislabel_coeffs == '' or self.options.struct_coeffs == ''):
      raise ValueError(('If choose_mislabel_struct_coeffs is use_given, then ' +
        'mislabel_coeffs and struct_coeffs cannot be empty. Given mislabel_coeffs=%s' +
        ' struct_coeffs=%s')%(self.options.mislabel_coeffs, self.options.struct_coeffs))
    self._preprocess_struct_mislabel_coeffs()
    # Check if we need to pre-compute distances
    if self.options.choose_mislabel_struct_coeffs == 'use_given' and \
       self.list_of_dists is None:
      if self.tp_comp is None:
        self.tp_comp = nn_comparators.get_otmann_distance_from_args(self.nn_type,
                                        self.options.non_assignment_penalty)
      self.list_of_dists = self.tp_comp(self.X, self.X, self.mislabel_coeffs,
                                        self.struct_coeffs, self.options.dist_type)
    # Some parameters we will be using often
    if len(self.Y) > 1:
      self.Y_var = self.Y.std() ** 2
    else:
      self.Y_var = 1.0
    # Compute the pairwise distances here
    # Set up based on whether we are doing maximum likelihood or post_sampling
    if self.options.hp_tune_criterion == 'ml':
      self.hp_bounds = []
      self._child_set_up_ml_tune()
    elif self.options.hp_tune_criterion == 'post_sampling':
      self.hp_priors = []
      self._child_set_up_post_sampling()

  def _preprocess_struct_mislabel_coeffs(self):
    """ Preprocesses the structural and mislabel coefficients. """
    if self.options.choose_mislabel_struct_coeffs == 'tune_coeffs':
      # If they are integers or floats
      self.num_mislabel_struct_coeffs = 1
      self.to_tune_mislabel_struct_coeffs = True
    elif (isinstance(self.options.mislabel_coeffs, list) and
          isinstance(self.options.struct_coeffs, list)) or \
         (isinstance(self.options.mislabel_coeffs, str) and
          isinstance(self.options.struct_coeffs, str)):
      # If they are lists or strings
      if isinstance(self.options.mislabel_coeffs, str):
        # If they are strings, obtain the lists
        self.mislabel_coeffs = [float(x) for x in self.options.mislabel_coeffs.split('-')]
        self.struct_coeffs = [float(x) for x in self.options.struct_coeffs.split('-')]
      else:
        self.mislabel_coeffs = self.options.mislabel_coeffs
        self.struct_coeffs = self.options.struct_coeffs
      # Store other parameters
      self.num_mislabel_struct_coeffs = len(self.mislabel_coeffs)
      self.to_tune_mislabel_struct_coeffs = False
      # Check if the lengths are the same
      if len(self.mislabel_coeffs) != len(self.struct_coeffs):
        raise ValueError('Length of mislabel and structural coefficients must be same.' +
                         'Given: mislabel: %s (%d), struct:%s (%d).'%(
                         str(self.options.mislabel_coeffs), len(self.mislabel_coeffs),
                         str(self.options.struct_coeffs), len(self.struct_coeffs)))
    else:
      raise ValueError('Bad format for mislabel and structural coefficients.' +
                       'Given: mislabel: %s, struct:%s.'%(
                      str(self.options.mislabel_coeffs), str(self.options.struct_coeffs)))

  def _child_set_up_ml_tune(self):
    """ Sets up tuning for Maximum likelihood. """
    # Order of hyper-parameters (this is critical): noies variance, scale/alphas, betas,
    # mislabel_coeff, struct_coeff
    # 1. Noise variance ---------------------------------------
    if self.options.noise_var_type == 'tune':
      self.noise_var_log_bounds = [np.log(0.005 * self.Y_var), np.log(0.2 * self.Y_var)]
      self.hp_bounds.append(self.noise_var_log_bounds)
    # 2. scale/alphas -----------------------------------------
    scale_log_bounds = [np.log(0.01 * self.Y_var), np.log(100 * self.Y_var)]
    if self.kernel_type in ['lp', 'emd', 'lpemd_prod']:
      self.hp_bounds.append(scale_log_bounds)
    elif self.kernel_type == 'lpemd_sum':
      self.hp_bounds.extend([scale_log_bounds] * 2)
    # 3. betas ------------------------------------------------
    lp_beta_log_bounds = [[np.log(1e-9), np.log(1e-3)]] * self.num_mislabel_struct_coeffs
    emd_beta_log_bounds = [[np.log(1e-1), np.log(1e2)]] * self.num_mislabel_struct_coeffs
    if self.options.dist_type == 'lp':
      all_beta_bounds = lp_beta_log_bounds
    elif self.options.dist_type == 'emd':
      all_beta_bounds = emd_beta_log_bounds
    elif self.options.dist_type == 'lp-emd':
      all_beta_bounds = [j for i in zip(lp_beta_log_bounds, emd_beta_log_bounds)
                         for j in i]
    self.hp_bounds.extend(all_beta_bounds) # extend with log_beta_bounds
    # 4 & 5. mislabel/struct_coeff ----------------------------
    if self.to_tune_mislabel_struct_coeffs:
      self.hp_bounds.append([0.001, 2.0]) # mislabel coefficient (not in log space)
      self.hp_bounds.append([0.001, 2.0]) # structural coefficient (not in log space)

  def _child_set_up_post_sampling(self):
    """ Sets up posterior sampling. """
    raise NotImplementedError('Not implemented post sampling yet.')

  def _child_build_gp(self, gp_hyperparams, build_posterior=True):
    """ Builds the GP from the hyper-parameters. """
    gp_hyperparams = gp_hyperparams[:] # create a copy of the list
    kernel_hyperparams = {}
    # mean function
    mean_func = gp_core.get_mean_func_from_options(self.options, self.Y) # Mean function
    # Extract GP hyper-parameters
    # 1. Noise variance ------------------------------------------------------
    noise_var = gp_core.get_noise_var_from_options_and_hyperparams(self.options,
                  gp_hyperparams, self.Y, 0)
    if self.options.noise_var_type == 'tune':
      gp_hyperparams = gp_hyperparams[1:]
    # 2. Scale and alphas
    if self.kernel_type in ['lp', 'emd', 'lpemd_prod']:
      kernel_hyperparams['scale'] = np.exp(gp_hyperparams[0])
      gp_hyperparams = gp_hyperparams[1:]
    elif self.kernel_type == 'lpemd_sum':
      kernel_hyperparams['alphas'] = np.exp(gp_hyperparams[:2])
      gp_hyperparams = gp_hyperparams[2:]
    # 3. betas
    if self.options.dist_type in ['lp', 'emd']:
      kernel_hyperparams['betas'] = np.exp(
                                    gp_hyperparams[:self.num_mislabel_struct_coeffs])
      gp_hyperparams = gp_hyperparams[self.num_mislabel_struct_coeffs:]
    elif self.options.dist_type == 'lp-emd':
      kernel_hyperparams['betas'] = np.exp(
                                    gp_hyperparams[:2 * self.num_mislabel_struct_coeffs])
      gp_hyperparams = gp_hyperparams[2 * self.num_mislabel_struct_coeffs:]
    # 4 & 5. mislabel and structural coefficents
    if self.to_tune_mislabel_struct_coeffs:
      kernel_hyperparams['mislabel_coeffs'] = gp_hyperparams[0]
      kernel_hyperparams['struct_coeffs'] = gp_hyperparams[1]
      gp_hyperparams = gp_hyperparams[2:]
    else:
      kernel_hyperparams['mislabel_coeffs'] = self.mislabel_coeffs
      kernel_hyperparams['struct_coeffs'] = self.struct_coeffs
    # Some checks
    assert len(gp_hyperparams) == 0
    # Build the remainder of the kernel ---------------------------------
    # - powers
    lp_powers = [self.options.lp_power] * self.num_mislabel_struct_coeffs
    emd_powers = [self.options.emd_power] * self.num_mislabel_struct_coeffs
    if self.options.dist_type == 'lp':
      kernel_hyperparams['powers'] = lp_powers
    elif self.options.dist_type == 'emd':
      kernel_hyperparams['powers'] = emd_powers
    elif self.options.dist_type == 'lp-emd':
      kernel_hyperparams['powers'] = [j for i in zip(lp_powers, emd_powers) for j in i]
    # - non_assignment_penalty
    kernel_hyperparams['non_assignment_penalty'] = self.options.non_assignment_penalty
    # Build the GP and return
    return NNGP(self.X, self.Y, self.kernel_type, self.nn_type,
                kernel_hyperparams, mean_func, noise_var,
                self.options.dist_type, tp_comp=self.tp_comp,
                list_of_dists=self.list_of_dists,
                build_posterior=build_posterior)


"""1.9 두 딕셔너리이 유사점 찾기
두 딕셔너리가 있고 여기서 유사점을 찾고 싶다 (동일한 키, 동일한 값)등"""

# example 1

a = {
    'x' : 1,
    'y' : 2,
    'z' : 3,
}

b = {
    'w' : 10,
    'x' : 11,
    'y' : 2
}

# 동일한 키 찾기
print(a.keys() & b.keys())

# a에만 있고 b에는 없는 키 찾기
print(a.keys() - b.keys())

# (키, 값)이 동일한 것 찾기
print(a.items() & b.items())

# 특정 키를 제거한 새로운 딕셔너리 만들기
c = {key:a[key] for key in a.keys() - {'z','w'}}
print(c) # {'x': 1, 'y': 2}

1.9 두 딕셔너리의 유사점 찾기.py


import utils as u

with open(__file__ + ".input.txt", "r+") as file:
    raw_input = file.read()

# Part 1

example_program = """inc a
jio a, +2
tpl a
inc a"""

def exec_program(program: str, starting_a=0):
    registries = {"a": starting_a, "b": 0}
    pointer = 0
    rows = program.splitlines()
    try:
        while True:
            row = rows[pointer]
            instruction = row[:3]
            if instruction == "hlf":  # hlf a
                target = row[4]
                registries[target] //= 2
                pointer += 1
            elif instruction == "tpl":
                target = row[4]
                registries[target] *= 3
                pointer += 1
            elif instruction == "inc":
                target = row[4]
                registries[target] += 1
                pointer += 1
            elif instruction == "jmp":
                offset = int(row[4:])
                pointer += offset
            elif instruction == "jie":
                target = row[4]
                if registries[target] % 2 == 0:
                    offset = int(row[6:])
                    pointer += offset
                else:
                    pointer += 1
            elif instruction == "jio":
                target = row[4]
                if registries[target] == 1:
                    offset = int(row[6:])
                    pointer += offset
                else:
                    pointer += 1
    except IndexError:
        return registries

example = exec_program(example_program)

u.assert_equals(example["a"], 2)

result = exec_program(raw_input)
u.assert_equals(result["b"], 170)
u.answer_part_1(result["b"])

result = exec_program(raw_input, starting_a=1)
u.answer_part_2(result["b"])


import unittest

class Point: # The object to hold the values of the points of the rectangles
    def __init__(self,x,y,):
        self.x = x
        self.y = y

def intersection(l1,r1,l2,r2):
    if(l1.x == r1.x or l1.y == r1.y or l2.x == r2.x or l2.y == r2.y): # Checking if the points are a line and not a rectangle
        return False
    if(l1.x >= r2.x or r2.y >= l1.y): # checking if the points are beside the first rectangle
        return False
    if(r1.y >= l2.y or r2.y >= l1.y): # checking if the points are above the first rectangle
        return False
    return True

def containment(l1,r1,l2,r2):
    if(l1.x == r1.x or l1.y == r1.y or l2.x == r2.x or l2.y == r2.y): # Checking if the points are a line and not a rectangle
        return False
    if(l1.y < l2.y or r1.x < r2.x): # Checking if the points are outside the first rectangle
        return False
    return True

def adjacency(l1,r1,l2,r2):
    if(l1.x == r1.x or l1.y == r1.y or l2.x == r2.x or l2.y == r2.y): # Checking if the points are a line and not a rectangle
        return False
    if(r1.x == l2.x or r2.y == l1.y): # Checking if one side is adjacent to the other
        return True

def main():# Main function

    topLeft1 = Point(int(input('Top left 1 X \n')),int(input('Top left 1 Y \n')))
    bottomRight1 = Point(int(input('bottom Right 1 X \n')),int(input('bottom Right 1 Y \n')))
    topLeft2 = Point(int(input('Top left 2 X \n')),int(input('Top left 2 Y \n')))
    bottomRight2 = Point(int(input('bottom Right 2 X \n')),int(input('bottom Right 2 Y \n')))

    if (containment(topLeft1,bottomRight1,topLeft2,bottomRight2)): # First check if the points of the second rectangle are within the first retangle
        print('Rectangle is Contained within the other')
        print('Not adjacent')
        print('They do not intersect')
    else:
        print('No Containment')
        if (adjacency(topLeft1,bottomRight1,topLeft2,bottomRight2)): # Next check if the rectangles are adjacent
            print('They are Adjacent')
            print('They do not intersect')
        else:
            print('Not adjacent')
            if (intersection(topLeft1,bottomRight1,topLeft2,bottomRight2)): # Finally check if they intersect
                print('Rectangles intersect')
            else:
                print('They do not intersect')

class TestProgram (unittest.TestCase):
        def test_Containment(self):
            topLeft1 = Point(0,10)
            bottomRight1 = Point(10,0)
            topLeft2 = Point(5,5)
            bottomRight2 = Point(6,1)
            self.assertEquals(containment(topLeft1,bottomRight1,topLeft2,bottomRight2),True)

        def test_adjacent(self):
            topLeft1 = Point(0,10)
            bottomRight1 = Point(10,0)
            topLeft2 = Point(10,5)
            bottomRight2 = Point(15,0)
            self.assertEquals(adjacency(topLeft1,bottomRight1,topLeft2,bottomRight2),True)

        def test_intersection(self):
            topLeft1 = Point(0,10)
            bottomRight1 = Point(10,0)
            topLeft2 = Point(10,5)
            bottomRight2 = Point(15,0)
            self.assertEquals(intersection(topLeft1,bottomRight1,topLeft2,bottomRight2),True)

if __name__ == '__main__':
    main()
    unittest.main()


#!/usr/bin/env python3.6
"""
Compare two manifests from a given product to see if there are any
potential commits in the older manifest which did not get included
into the newer manifest.  This will assist in determining if needed
fixes or changes have been overlooked being added to newer releases.
The form of the manifest filenames passed is a relative path based
on their location in the manifest repository (e.g. released/4.6.1.xml).
"""
import os
import argparse
import io
import logging
import os.path
import pathlib
import re
import subprocess
import sys
import contextlib
import traceback
from distutils.spawn import find_executable

import dulwich.patch
import dulwich.porcelain
import dulwich.repo
import lxml.etree

from manifest_tools.scripts.jira_util import connect_jira, get_tickets

@contextlib.contextmanager
def pushd(new_dir):
    old_dir = os.getcwd()
    os.chdir(new_dir)

    try:
        yield
    finally:
        os.chdir(old_dir)

DEVNULL = open(os.devnull, 'w')

class MissingCommits:
    """"""

    def __init__(self, logger, product_dir, old_manifest, new_manifest,
                 manifest_repo, reporef_dir, ignored_commits,
                 pre_merge, post_merge, merge_map):
        """
        Store key information into instance attributes and determine
        path of 'repo' program
        """

        self.missing_commits_found = False

        self.log = logger
        self.product_dir = product_dir
        self.old_manifest = old_manifest
        self.new_manifest = new_manifest
        self.manifest_repo = manifest_repo
        self.reporef_dir = reporef_dir
        self.ignored_commits = ignored_commits
        self.pre_merge = pre_merge
        self.post_merge = post_merge
        self.merge_map = merge_map

        # Projects we don't care about
        self.ignore_projects = ['testrunner', 'libcouchbase', 'product-texts', 'product-metadata']

        self.repo_bin = find_executable('repo')

        # We check jira and ignore tickets which are flagged "is a backport of" a ticket in the newer release
        try:
            self.jira = connect_jira()
        except Exception:
            self.log.critical("Jira connection failed")
            sys.exit(1)

    def backports_of(self, tickets):
        """For a list of tickets, gather any outward links flagged "is a backport of" in Jira
        and return a combined listing of the ticket references"""

        backports = []
        for ticket in tickets:
            try:
                jira_ticket = self.jira.issue(ticket)
                for issuelink in jira_ticket.raw["fields"]["issuelinks"]:
                    if issuelink["type"]["outward"] == "is a backport of":
                        # Ensure we're looking at the actual backport ticket,
                        # not a ticket that was itself backported
                        if "outwardIssue" in issuelink:
                            backports.append(issuelink["outwardIssue"]["key"])
            except Exception as exc:
                traceback.print_exc()
                self.log.error(f"Jira ticket retrieval failed for {ticket}")
                sys.exit(1)

        return backports

    def repo_sync(self):
        """
        Initialize and sync a repo checkout based on the target
        manifest; generate a new manifest with fixed SHAs in case
        the target contains branches (e.g. master) via the command
        'repo manifest -r' so 'git log' will work properly
        """

        # Create a 'product' directory to contain the repo checkout
        if self.product_dir.exists() and not self.product_dir.is_dir():
            self.log.warning(f'"{self.product_dir}" exists and is not a directory, '
                  f'removing...')
            try:
                self.product_dir.unlink()
            except OSError as exc:
                self.log.error(
                    f'Unable to delete "{self.product_dir}" file/link: '
                    f'{exc.message}'
                )
                sys.exit(1)

        if not self.product_dir.exists():
            try:
                self.product_dir.mkdir()
            except OSError as exc:
                self.log.error(
                    f'Unable to create "{self.product_dir}" directory: '
                    f'{exc.message}'
                )
                sys.exit(1)

        try:
            cmd = [self.repo_bin, 'init', '-u',
                   self.manifest_repo,
                   '-g', 'all', '-m', self.new_manifest]
            if self.reporef_dir is not None:
                cmd.extend(['--reference', self.reporef_dir])
            subprocess.check_output(cmd,
                                    cwd=self.product_dir, stderr=subprocess.STDOUT
                                    )
        except subprocess.CalledProcessError as exc:
            self.log.error(f'The "repo init" command failed: {exc.output}')
            sys.exit(1)

        # From now on, use the "repo" wrapper from the .repo directory,
        # to prevent "A new version is available" warning messages.
        # This assumes that all "repo" commands will be invoked with
        # cwd=self.product_dir, so this relative path will work.
        self.repo_bin = os.path.join(".repo", "repo", "repo")

        try:
            subprocess.check_output(
                [self.repo_bin, 'sync', '--jobs=6', '--force-sync'],
                cwd=self.product_dir, stderr=subprocess.STDOUT
            )
        except subprocess.CalledProcessError as exc:
            self.log.error(f'The "repo sync" command failed: {exc.output}')
            sys.exit(1)

        # This is needed for manifests with projects not locked down
        # (e.g. spock.xml)
        try:
            with open('new.xml', 'w') as fh:
                subprocess.check_call(
                    [self.repo_bin, 'manifest', '-r'],
                    stdout=fh, cwd=self.product_dir
                )
        except subprocess.CalledProcessError as exc:
            self.log.error(f'The "repo manifest -r" command failed: {exc.output}')
            sys.exit(1)

    def diff_manifests(self):
        """
        Generate the diffs between the two manifests via the command
        'repo diffmanifests'.  Only return the project lines, not
        the actual commit differences.
        """

        new_xml = pathlib.Path('new.xml').resolve()

        try:
            diffs = subprocess.check_output(
                [self.repo_bin, 'diffmanifests', '--raw',
                 self.old_manifest, new_xml],
                cwd=self.product_dir, stderr=subprocess.STDOUT
            ).decode()
        except subprocess.CalledProcessError as exc:
            self.log.error(f'The "repo diffmanifests" command failed: {exc.output}')
            sys.exit(1)

        return [
            line for line in diffs.strip().split('\n')
            if not line.startswith(' ')
        ]

    def generate_diff(self, repo_path, commit_sha):
        """"""

        return None  # Short-circuit

        project_dir = self.product_dir / repo_path
        repo = dulwich.repo.Repo(str(project_dir.resolve()))

        commit = repo[bytes(commit_sha, 'utf-8')]
        prev_commit = repo[commit.parents[0]]

        fh = io.BytesIO()
        dulwich.patch.write_tree_diff(fh, repo.object_store, prev_commit.tree,
                                      commit.tree)

        return [line for line in fh.getvalue().decode().split('\n')]

    @staticmethod
    def compare_summaries(old_summary, new_summary):
        """"""

        # If two commits have *exactly* the same summary line, and that
        # summary line isn't trivially short, we can assume it was a
        # cherry-pick
        if old_summary == new_summary and len(old_summary) > 10:
            return True
        else:
            return False

    """
    Pre-compiled regex for SHA
    """
    sha_regex = re.compile(r'[0-9a-f]{40}')

    """
    Pre-compiled regex for tag reference
    """
    tag_regex = re.compile(r'refs/tags/.*')

    def get_commit_sha(self, project, commit):
        """
        Find the latest commit SHA from the given branch/tag/SHA
        """

        # If 'commit' is already a SHA, just return it
        if MissingCommits.sha_regex.fullmatch(commit) is not None:
            # Already a SHA, just return it
            return commit

        # Not a SHA, so we'll ask git to turn it into a SHA. If 'commit'
        # looks like a tag reference, use it directly; otherwise, assume
        # it's a branch name, and prepend the remote name to disambiguate.
        if MissingCommits.tag_regex.fullmatch(commit) is not None:
            git_ref = commit
        else:
            # $REPO_REMOTE is set by 'repo forall'
            git_ref = f'$REPO_REMOTE/{commit}'

        try:
            commit_sha = subprocess.check_output(
                [self.repo_bin, 'forall', project, '-c',
                 f'git show-ref --hash {git_ref}'],
                cwd=self.product_dir, stderr=subprocess.STDOUT
            ).decode()
        except subprocess.CalledProcessError as exc:
            self.log.error(f'The "repo forall" command failed: {exc.output}')
            sys.exit(1)

        return commit_sha.strip()

    def show_needed_commits(self, repo_path, change_info):
        """
        Determine 'missing' commits for a given project based on
        two commit SHAs for the project.  This is done by doing
        a 'git log' on the symmetric difference of the two commits
        in forward and reversed order, then comparing the summary
        content from the latter to find a matching entry in the
        former, which is a strong indication that the commit was
        properly merged into the project at the time of the target
        manifest.
        Print out any possible matches along with any 'missing'
        commits to allow user to determine what might still need
        to be merged forward.
        """

        if repo_path in self.ignore_projects:
            return
        # Also don't care about third-party Go stuff
        if repo_path.startswith("godeps"):
            return

        old_commit, new_commit, old_diff, new_diff = change_info
        missing = [
            '/usr/bin/git', 'log', '--oneline', '--cherry-pick',
            '--right-only', '--no-merges'
        ]

        old_commit = self.get_commit_sha(repo_path, old_commit)
        new_commit = self.get_commit_sha(repo_path, new_commit)

        project_dir = self.product_dir / repo_path
        try:
            old_results = subprocess.check_output(
                missing + [f'{old_commit}...{new_commit}'],
                cwd=project_dir, stderr=subprocess.STDOUT
            ).decode()
        except subprocess.CalledProcessError as exc:
            self.log.error(f'The "git log" command for project "{repo_path}" '
                  f'failed: {exc.stdout}')
            sys.exit(1)

        if old_results:
            rev_commits = old_results.strip().split('\n')
        else:
            rev_commits = list()

        try:
            new_results = subprocess.check_output(
                missing + [f'{new_commit}...{old_commit}'],
                cwd=project_dir, stderr=subprocess.STDOUT
            ).decode()
        except subprocess.CalledProcessError as exc:
            self.log.error(f'The "git log" command for project "{repo_path}" '
                  f'failed: {exc.stdout}')
            sys.exit(1)

        backport_message = ""
        possible_missing_message = ""
        missing_message = ""

        if new_results:
            for commit in new_results.strip().split('\n'):
                sha, comment = commit.split(' ', 1)

                if any(c.startswith(sha[:7]) for c in self.ignored_commits):
                    continue

                match = True
                for rev_commit in rev_commits:
                    rev_sha, rev_comment = rev_commit.split(' ', 1)
                    if self.compare_summaries(rev_comment, comment):
                        break
                else:
                    match = False
                backports = self.backports_of(get_tickets(comment))
                is_a_backport = False
                if backports:
                    with pushd(self.product_dir / repo_path):
                        with open('.git/HEAD') as f:
                            head = f.read().strip()
                        gitlog = subprocess.Popen(
                            ['git', 'log', '--oneline', new_commit], stdout=subprocess.PIPE)
                        try:
                            matches = subprocess.check_output(
                                ["grep", "-E"] + ['|'.join(backports)], stdin=gitlog.stdout).decode("ascii").strip().split("\n")
                            if len(matches) > 0:
                                is_a_backport = True
                        except subprocess.CalledProcessError as exc:
                            if exc.returncode == 1:
                                # grep most likely failed to find a match.
                                pass
                            else:
                                self.log.warning("Exception:", exc.output)

                # If it's a backport, keep log for later but don't count it as
                # a missing commit
                if is_a_backport:
                    backport_message += (
                        f'                 [Backport] {sha[:8]} {comment:.80}\n'
                        f'                        of: '
                    )
                    backport_message += '\n                            '.join(
                        [f'{x:.80}' for x in matches]
                    )
                    backport_message += '\n'
                    continue

                # At this point we know we have something to report. Save the
                # message.
                if match:
                    possible_missing_message += (
                        f'    [Possible commit match] {sha[:8]} {comment:.80}\n'
                        f'              Check commit: {rev_sha[:8]} {rev_comment:.80}\n'
                    )
                else:
                    missing_message += (
                        f'          [No commit match] {sha[:8]} {comment:.80}\n'
                    )

            # Print project header, if anything to report
            if missing_message != "" or possible_missing_message != "" or backport_message != "":
                self.log.info("")
                self.log.info(f'Project {repo_path}:')

            if missing_message != "":
                self.log.info("")
                self.log.info(missing_message)
                self.missing_commits_found = True

            # This is *usually* OK, and it's just too hard to verify everything
            # by hand, so go ahead and assume it's OK.
            # I'm leaving the code in place to display this if we want
            # it again in future. - Ceej Jun 10 2022
            if possible_missing_message != "":
                self.log.info("")
                self.log.info("This project has commits that were matched by commit message summary")

            # We believe this to be working now, so stop outputting the
            # message - makes it too hard to see actual issues.
            # I'm leaving the code in place to display this if we want
            # it again in future. - Ceej Jun 06 2021
            if backport_message != "":
                self.log.info("")
                self.log.info("This project has backports that were correctly identified")

    def determine_diffs(self):
        """
        This manages the main workflow:
         - Sync the repo checkout to the target release/branch
         - Generate the diffs between the two manifests
         - Determine any "missing" commits by comparing SHAs
           for each project via 'git log'; this includes knowing
           about merged projects and handling them correctly
        """

        self.repo_sync()
        diffs = self.diff_manifests()
        changes = dict()

        # Create dictionary with all the relevant changes; this avoids
        # any added or removed projects that were not part of a merge
        # process at some point in the past
        for entry in diffs:
            if entry.startswith('A '):
                _, repo_path, current_commit = entry.split()

                if repo_path in self.post_merge:
                    changes[repo_path] = (
                        'added', current_commit,
                        self.generate_diff(repo_path, current_commit)
                    )
            elif entry.startswith('R '):
                _, repo_path, final_commit = entry.split()

                if repo_path in self.pre_merge:
                    changes[repo_path] = (
                        'removed', final_commit,
                        self.generate_diff(repo_path, final_commit)
                    )
            elif entry.startswith('C '):
                _, repo_path, old_commit, new_commit = entry.split()
                changes[repo_path] = (
                    'changed', old_commit, new_commit,
                    self.generate_diff(repo_path, old_commit),
                    self.generate_diff(repo_path, new_commit)
                )
            else:
                self.log.warning(f'Unhandled entry, skipping: {entry}')

        self.log.info(f"\n\n* Missing commits from {self.old_manifest} "
                      f"to {self.new_manifest}...\n")

        # Perform commit diffs, handling merged projects by diffing
        # the merged project against each of the projects the were
        # merged into it
        for repo_path, change_info in changes.items():
            if change_info[0] == 'changed':
                change_info = change_info[1:]
                self.show_needed_commits(repo_path, change_info)
            elif change_info[0] == 'added':
                _, new_commit, new_diff = change_info

                for pre in self.merge_map[repo_path]:
                    _, old_commit, old_diff = changes[pre]
                    change_info = (old_commit, new_commit, old_diff, new_diff)
                    self.show_needed_commits(repo_path, change_info)

        if not self.missing_commits_found:
            self.log.info("")
            self.log.info("No missing commits discovered!\n")
        self.log.info("")

def main():
    """
    Parse the command line, initialize logging and key information,
    create manifest paths and perform the missing commits check
    """

    parser = argparse.ArgumentParser(
        description='Determine potential missing commits'
    )
    parser.add_argument('-d', '--debug', action='store_true',
                        help='Show additional information during run')
    parser.add_argument('-i', '--ignore', dest='ignore_file',
                        help='File to store "ignored" commits',
                        default='ignored_commits.txt')
    parser.add_argument('-m', '--merge', dest='merge_file',
                        help='File to store "merged" projects info',
                        default='merged_projects.txt')
    parser.add_argument('product', help='Product to check')
    parser.add_argument('old_manifest', help='Base manifest to check against')
    parser.add_argument('new_manifest', help='Current manifest to verify')
    parser.add_argument('--reporef_dir',
                        help='Path to repo mirror reference directory')
    parser.add_argument('--manifest_repo', help='Git URL to manifest repo')
    args = parser.parse_args()

    # Set up logging
    logger = logging.getLogger(__name__)
    logger.setLevel(logging.DEBUG)

    ch = logging.StreamHandler()
    if not args.debug:
        ch.setLevel(logging.INFO)

    logger.addHandler(ch)

    # Read in 'ignored' commits
    # Form of each line is: '<project> <commit SHA> <optional comment>'
    ignored_commits = list()

    try:
        with open(args.ignore_file) as fh:
            for entry in fh.readlines():
                if entry.startswith('#'):
                    continue   # Skip comments
                try:
                    project, commit = entry.split()[0:2]
                except ValueError:
                    logger.warning(f'Malformed line in ignored commits file, '
                                   f'skipping: {entry}')
                else:
                    ignored_commits.append(commit)
    except FileNotFoundError:
        logger.warning(f'Ignored commits file, {args.ignore_file}, '
                       f'not found.  Continuing...')

    # Read in 'merged' projects information
    # Form of each line is: '<merged project> [<original project> [...]]'
    pre_merge = list()
    post_merge = list()
    merge_map = dict()

    try:
        with open(args.merge_file) as fh:
            for entry in fh.readlines():
                if entry.startswith('#'):
                    continue   # Skip comments

                try:
                    post, *pre = entry.split()
                except ValueError:
                    logger.warning(f'Empty line in merged projects file, '
                                   f'skipping')
                else:
                    if pre:
                        pre_merge.extend(pre)
                        post_merge.append(post)
                        merge_map[post] = pre
                    else:
                        logger.warning(f'Malformed line in merged projects '
                                       f'file, skipping: {entry}')
    except FileNotFoundError:
        logger.warning(f'Merged projects file, {args.merge_file}, '
                       f'not found.  Continuing...')

    # Setup file paths and search for missing commits
    product_dir = pathlib.Path(args.product)
    old_manifest = pathlib.Path(args.old_manifest)
    new_manifest = pathlib.Path(args.new_manifest)
    reporef_dir = pathlib.Path(args.reporef_dir)

    miss_comm = MissingCommits(
        logger, product_dir, old_manifest, new_manifest, args.manifest_repo,
        reporef_dir, ignored_commits, pre_merge, post_merge, merge_map
    )
    miss_comm.determine_diffs()

    if miss_comm.missing_commits_found:
        sys.exit(1)

if __name__ == '__main__':
    main()


from sklearn.cross_validation import train_test_split
#from sklearn.model_selection import train_test_split
from cnn import larger_model
import pickle
from keras.utils import np_utils
from sklearn.metrics import confusion_matrix
import numpy as np
def load_data(file_path):
    with open(file_path,'rb') as f:
        data = pickle.load(f)
    return data

def train():
    X,y = load_data('data/ecg_data.pkl')
    print(X.shape)
#    X = X.reshape(X.shape[0],X.shape[1],X.shape[2]).astype('float32')
    print(X.shape)
    X_train,X_test,y_train,y_test = train_test_split(X,y,test_size=0.2)
    num_classes = 4
    shape = X[0].shape
    print('x_train:{0}\nX_test:{1}\ny_train:{2}\ny_test{3}\n'.format(X_train.shape,X_test.shape,y_train.shape,y_test.shape))
    model = larger_model(num_classes,shape)
    model.fit(X_train,y_train,validation_data=(X_test,y_test),epochs=20,batch_size=32)
    model.save('cnn.model')
    scores = model.evaluate(X_test,y_test,verbose=0)
    print('Accuracy:{}'.format(scores))
    y_predict = model.predict(X_test)
    y_predict = np.argmax(y_predict,axis=1).tolist()
    y_actual = np.argmax(y_test,axis=1).tolist()
    print('confusion matrix:')
    print(confusion_matrix(y_actual,y_predict))

if __name__ == "__main__":
    train()


#!/usr/bin/env python
import sys
import socket
import struct
import math
import random
import time
import select

class Network:
   def __init__(self, bot, name, ip):
      self.bot = bot
      self.initMsg()
      self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
      self.s.connect((ip, 3490))
      self.send("n %s\n" % name)
      self.readall()
      self.send("b 8\n")

   def initMsg(self):
      self.cntHead = 8
      self.cntHead2 = -1
      self.cntBody = -1
      self.bufHead = b''
      self.bufHead2 = b''
      self.bufBody = b''
      self.msgID = 0
      self.pid = -1

   def send(self, message):
      self.s.send(bytes(message, 'UTF-8'))

   def readall(self):
      self.s.settimeout(1)
      while True:
         try:
            self.s.recv(1024)
         except socket.timeout:
            break

   def shoot(self, a):
      self.send("%lf\n" % a)

   def requestEnergy(self):
      self.send("u\n")

   def step(self):
      while self.stepSingle() > 0:
         pass

   def stepSingle(self):
      r, w, x = select.select((self.s, ), (), (), 0)
      if len(r) == 0:
         return 0
      if self.cntHead > 0:
         data = self.s.recv(self.cntHead)
         self.cntHead -= len(data)
         self.bufHead += data
         if self.cntHead == 0:
            self.evalHead()
      elif self.cntHead2 > 0:
         data = self.s.recv(self.cntHead2)
         self.cntHead2 -= len(data)
         self.bufHead2 += data
         if self.cntHead2 == 0:
            self.evalHead2()
      elif self.cntBody > 0:
         data = self.s.recv(self.cntBody)
         self.cntBody -= len(data)
         self.bufBody += data
         if self.cntBody == 0:
            self.evalBody()
      return len(r)

   def evalHead(self):
      self.msgID, self.pid = struct.unpack('II', self.bufHead)
      if self.msgID == 1:                    # MSG_OWNID
         self.bot.ownId(self.pid)
         self.initMsg()
      elif self.msgID == 2:                  # MSG_PLAYERLEAVE
         self.bot.playerLeave(self.pid)
         self.initMsg()
      elif self.msgID == 3:                  # MSG_PLAYERPOS
         self.cntBody = 8
      elif self.msgID == 5:                  # MSG_SHOTBEGIN
         self.cntBody = 16
      elif self.msgID == 6:                  # MSG_SHOTFIN
         self.cntHead2 = 20
      elif self.msgID == 7:                  # MSG_GAMEMODE
         self.cntBody = 4
      elif self.msgID == 8:                  # MSG_OWNENERGY
         self.cntBody = 8
      else:
         print("unexpected message id in head: %d" % self.msgID)

   def evalHead2(self):
      if self.msgID == 6:                    # MSG_SHOTFIN
         a, v, n = struct.unpack("ddI", self.bufHead2)
         self.cntBody = n * 8
      else:
         print("unexpected message id in head2: %d" % self.msgID)

   def evalBody(self):
      if self.msgID == 3:                    # MSG_PLAYERPOS
         fx, fy = struct.unpack("ff", self.bufBody)
         self.bot.playerPos(self.pid, fx, fy)
      elif self.msgID == 5:                  # MSG_SHOTBEGIN
         a, v = struct.unpack("dd", self.bufBody)
         self.bot.shotBegin(self.pid, a, v)
      elif self.msgID == 6:                  # MSG_SHOTFIN
         a, v, n = struct.unpack("ddI", self.bufHead2)
         curve = []
         for i in range(n):
            curve.append(struct.unpack_from("ff", self.bufBody, 8 * i))
         self.bot.shotFin(self.pid, a, v, curve)
      elif self.msgID == 7:                  # MSG_GAMEMODE
         m, = struct.unpack("I", self.bufBody)
         if m != 3:
            print("unsupported gamemode %d, expect strangeness" % m)
      elif self.msgID == 8:                  # MSG_OWNENERGY
         e, = struct.unpack("d", self.bufBody)
         self.bot.ownEnergy(e)
      else:
         print("unexpected message id in body: %d" % self.msgID)
      self.initMsg()

class Stupobot:
   def __init__(self, ip):
      self.net = Network(self, "Stupobot OOP", ip)
      self.angle = 0
      self.mypid = None
      self.energy = 0
      self.nextTime = time.time()

   def step(self):
      self.net.step()
      t = time.time()
      if t > self.nextTime:
         self.net.requestEnergy()

   def ownId(self, pid):
      self.mypid = pid
      print ("My id: %d" % self.mypid)

   def playerLeave(self, pid):
      print("Player %d left" % pid)

   def playerPos(self, pid, fx, fy):
      print("Player %d at %f, %f" % (pid, fx, fy))

   def shotBegin(self, pid, a, v):
      print("Player %d begun shot (v: %lf, a: %lf)" % (pid, v, a))
      if pid == self.mypid:
         self.energy -= v
         self.angle = (self.angle + 361 / 5) % 360
         if self.energy >= v:
            self.net.shoot(self.angle)
         else:
            self.net.requestEnergy()

   def shotFin(self, pid, a, v, curve):
      print("Player %d finished shot (v: %lf, a: %lf) with %d segments" % (pid, v, a, len(curve)))
      for fx, fy in curve:
         print("   Curve: %f, %f" % (fx, fy))

   def ownEnergy(self, e):
      print("Own energy: %lf" % e)
      if e < 50:
         self.nextTime = time.time() + 1
      else:
         self.nextTime += 2 # will be updated when all shots are fired, should never take more than 2s
         self.energy = e
         self.net.shoot(self.angle)

if __name__ == "__main__":
   bot = Stupobot("127.0.0.1")

   while True:
      bot.step()
      time.sleep(0.1)


from django.core.management import BaseCommand

from football.management.commands_data.football_data import GAMES_DATA, TEAMS_DATA
from football.models import Game, Team

def insert_teams():
    for name, points in TEAMS_DATA:
        Team.objects.create(name=name, points=points)

def insert_games():
    for team_home_id, team_home_goals, team_away_id, team_away_goals in GAMES_DATA:
        Game.objects.create(team_home_id=team_home_id, team_home_goals=team_home_goals,
                            team_away_id=team_away_id, team_away_goals=team_away_goals)

class Command(BaseCommand):
    help = "Insert data about football league to data base."

    def handle(self, *args, **kwargs):
        insert_teams()
        insert_games()
        print("Data load successfully!")


import numpy as np
import pandas as pd

# 1. Load DS and make pivot table
df = pd.read_csv('df_to_prepare_for_multiple_analysis.csv')
df.head()
df = df[['date', 'name', 'volume', 'close']]
df_last = pd.pivot_table(df, values=['volume', 'close'], index=['date'], columns=['name'])
df_last.index = pd.to_datetime(df_last.index)
sorted(list(np.unique(df_last[np.any(df_last.isnull(), axis=1)].index)))
df_last = df_last.dropna()
list(df_last)

# 2. Prepare Rt: returns (shor will be Rt) for each cryprocyrrency
Rt = df_last[[i for i in list(df_last) if i[0] == 'close']]
Rt = Rt.pct_change()
Rt[pd.isnull(Rt)] = 0
Rt.columns = ['R_' + i[1] for i in list(Rt)]

# 3. Prepare vt : cryptocyrrency 50-day de-trended measure of trading activity as described in
# "Dynamic volume-Return relation of individual stocks. Rev. Financial Stud. 15, 1005–1047."
Vt_temp = df_last[[i for i in list(df_last) if i[0] == 'volume']]
Vt_temp = np.log(Vt_temp + 0.00000255)
trend_lasts = 50
Vt = []
for i in range(0, Vt_temp.shape[0]):
    if i < trend_lasts:
        Vt.append(np.array([0] * Vt_temp.shape[1]))
        continue
    Vt.append(Vt_temp.iloc[i, :].as_matrix() - np.mean(Vt_temp.iloc[i - trend_lasts:i - 1, :]).as_matrix())
Vt = pd.DataFrame(np.asarray(Vt)).set_index(Rt.index)
Vt.columns = list(Vt_temp)
Vt.columns = ['V_' + i[1] for i in list(Vt)]

# 4. Prepare Rt x vt
RVt = pd.DataFrame(np.array(Vt) * np.array(Rt)).set_index(Rt.index)
RVt.columns = ['RV_' + i[1] for i in list(Vt_temp)]

# 5. Save tables
res_table = pd.concat([RVt, Vt, Rt], axis=1)
res_table.to_csv('df_prepared_for_multiple_analysis.csv', header=True, index=True)


from flask import Blueprint, jsonify, session, request
from flask_login import login_required, current_user
from app.models import db, Review
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from ..forms.review_form import ReviewForm
from datetime import date

review_routes = Blueprint('reviews', __name__)

db_url = "sqlite:///dev.db"
engine = create_engine(db_url)
SessionFactory = sessionmaker(bind=engine)
session = SessionFactory()

#Get review by Id
@review_routes.route('/<int:reviewId>')
def review(reviewId):
    review = Review.query.get(reviewId)
    return review.to_dict()

#Get reviews by product Id
@review_routes.route('/products/<int:productId>')
def reviewsByProduct(productId):
    reviews = Review.query.filter(Review.product_id == productId)
    return [review.to_dict() for review in reviews]

#Create a new review
@review_routes.route('/products/<int:productId>/new', methods=['POST'])
@login_required
def create_Review(productId):
    form = ReviewForm()
    form['csrf_token'].data = request.cookies['csrf_token']
    existing_review = Review.query.filter(Review.product_id == productId)
    if existing_review is True:
        return {'error':'Review for Product already exists'},403
    if form.validate_on_submit():
        review = Review(
            title = form.data['title'],
            body = form.data['body'],
            rating = form.data['rating'],
            created_at = date.today(),
            user_id = current_user.id,
            product_id = productId,
        )
        db.session.add(review)
        db.session.commit()
        return review.to_dict()
    return {'errors': 'error'}, 401

#Delete an Existing Review
@review_routes.route("/delete/<int:reviewId>", methods=['GET','POST','DELETE'])
@login_required
def delete_review(reviewId):
  review = Review.query.get(reviewId)
  db.session.delete(review)
  db.session.commit()
  return {'message':'deleted'}

#Edit and Existing review
@review_routes.route('/edit/<int:reviewId>', methods=['GET','POST','PUT'])
@login_required
def edit_review(reviewId):
    form = ReviewForm()
    review = Review.query.get(reviewId)
    form['csrf_token'].data = request.cookies['csrf_token']
    review.title = form.data['title']
    review.body = form.data['body']
    review.rating = form.data['rating']
    review.edited_at = date.today()
    db.session.commit()
    return review.to_dict()

###########test stuff#############
session.close()
engine.dispose()


import os
import json
import boto3

def lambda_handler(event, context):
    cf_client = boto3.client('cloudformation')
    stack_set_name = 'deleteresources'  # Name for the CloudFormation StackSet
    template_url = os.environ['TEMPLATE_URL']  # URL from the CloudFormation parameter
    target_ou = os.environ['TARGET_OU']  # Target OU from the CloudFormation parameter
    target_account = os.environ['TARGET_ACCOUNT']  # Target account ID from the CloudFormation parameter
    region = 'us-east-1'  # Deployment region

    try:
        # Create the CloudFormation stack set
        cf_client.create_stack_set(
            StackSetName=stack_set_name,
            TemplateURL=template_url,
            Capabilities=['CAPABILITY_NAMED_IAM'],
            PermissionModel='SERVICE_MANAGED',
            AutoDeployment={
                'Enabled': True,
                'RetainStacksOnAccountRemoval': False
            },
            Parameters=[
                # Add any necessary parameters here
            ]
        )

        # Deploy the stack set to the specified OU or account
        cf_client.create_stack_instances(
            StackSetName=stack_set_name,
            DeploymentTargets={
                'OrganizationalUnitIds': [target_ou],
                # Alternatively, for a specific account: 'Accounts': [target_account]
            },
            Regions=[region]
        )

        return {
            'statusCode': 200,
            'body': json.dumps(f'StackSet {stack_set_name} deployment initiated.')
        }

    except Exception as e:
        return {
            'statusCode': 500,
            'body': json.dumps(f'Error: {str(e)}')
        }


import argparse
import json

import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.transforms.trigger import AfterWatermark, AfterCount, AccumulationMode, AfterProcessingTime
from datasketch import HyperLogLog
from apache_beam.io.gcp.bigquery import WriteToBigQuery

ACCURACY = 14
PROJECT_ID = 'stream-processing-384807'
DATASET_ID = 'stackoverflow'
TABLE_ID = 'unique-tag-count'

class AddWindowInfo(beam.DoFn):
    def process(self, x, window=beam.DoFn.WindowParam):
        d = {}
        d['count'] = float(x)
        d["window_start"] = window.start.to_utc_datetime()
        d["window_end"] = window.end.to_utc_datetime()
        print(d)
        yield d

# Custom combine function for HyperLogLog
class HyperLogLogCombineFn(beam.CombineFn):
    def create_accumulator(self):
        return HyperLogLog(p=ACCURACY)  # p is used to control accuracy, higher is more accuracy

    def add_input(self, accumulator, element):
        tags = element['tags'].split('|') if element[
            'tags'] else []  # Split the tags if there are multiple separated by '|'
        for tag in tags:
            accumulator.update(tag.encode('utf-8'))  # Encode tag as bytes
        return accumulator

    def merge_accumulators(self, accumulators):
        merged_hll = HyperLogLog(p=ACCURACY)
        for hll in accumulators:
            merged_hll.merge(hll)
        return merged_hll

    def extract_output(self, accumulator):
        return accumulator.count()

def process_pubsub_message(element):
    message_data = json.loads(element.decode('utf-8'))
    return message_data

def run(argv=None):
    # Create a Pipeline using Apache Beam
    parser = argparse.ArgumentParser()
    known_args, pipeline_args = parser.parse_known_args(argv)

    pipeline_options = PipelineOptions(pipeline_args)
    pipeline = beam.Pipeline(options=pipeline_options)

    # Read from Pub/Sub and process the messages
    messages = (
            pipeline
            | 'Read from Pub/Sub' >> beam.io.ReadFromPubSub(
        subscription='projects/stream-processing-384807/subscriptions/stack-overflow-sub')
            | 'Process Pub/Sub Messages' >> beam.Map(process_pubsub_message)
    )

    # Apply the HyperLogLog transform
    total_distinct_count = (
            messages
            | beam.WindowInto(beam.window.FixedWindows(60 * 5))
            # Group the messages into 5-minute  windows
            | beam.CombineGlobally(HyperLogLogCombineFn()).without_defaults()
    )

    # Convert the data to rows for writing to BigQuery
    rows = total_distinct_count | beam.ParDo(AddWindowInfo())

    # Write the data to BigQuery
    table_schema = {
        'fields': [
            {'name': 'window_start', 'type': 'TIMESTAMP'},
            {'name': 'window_end', 'type': 'TIMESTAMP'},
            {'name': 'count', 'type': 'FLOAT64'}
        ]
    }

    write_to_bq = WriteToBigQuery(
        table=PROJECT_ID + ':' + DATASET_ID + '.' + TABLE_ID,
        schema=table_schema,
        write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND,
        create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED
    )

    rows | 'Write to BigQuery' >> write_to_bq

    # Execute the pipeline
    result = pipeline.run()
    result.wait_until_finish()

if __name__ == '__main__':
    run()


import torch
from torch import nn
from transformers import AutoModel, AutoTokenizer

class CrossEncoder(nn.Module):
    def __init__(self,
                 encoder= None,
                 model_checkpoint="vinai/phobert-base-v2",
                 representation=0,
                 dropout=0.1,
                 device=torch.device("cuda" if torch.cuda.is_available() else "cpu")):
        super(CrossEncoder, self).__init__()
        if encoder:
            self.encoder=encoder
        else:
            self.encoder = AutoModel.from_pretrained(model_checkpoint)
        self.representation = representation
        self.classifier = nn.Sequential(nn.Dropout(p=dropout),
                                        nn.Linear(self.encoder.config.hidden_size,2))
        self.tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
        self.device = device
        self.sep_token = " . " + self.tokenizer.sep_token + " "

    def forward(
        self,
        input_ids,
        attention_mask,
        token_type_ids=None):

        outputs = self.encoder(input_ids,
                                attention_mask,
                                token_type_ids)

        sequence_output = outputs['last_hidden_state']
        sequence_output = sequence_output.masked_fill(~attention_mask[..., None].bool(), 0.0)

        if self.representation > -2:
            output = sequence_output[:, self.representation, :]
        elif self.representation == -10:
            output = sequence_output.sum(dim=1) / attention_mask.sum(dim=1)[..., None]
        elif self.representation == -100:
            output = outputs[1]

        logits = self.classifier(output)

        return logits

    def predict(self, cross_samples, max_len):
        cross_texts = [q + self.sep_token + p for (q,p) in cross_samples]
        C = self.tokenizer.batch_encode_plus(cross_texts,
                                             padding='max_length',
                                             truncation=True,
                                             max_length=max_len,
                                             return_tensors='pt')
        with torch.no_grad():
            logits = self.forward(C['input_ids'].to(self.device),
                            C['attention_mask'].to(self.device))
            probs = nn.functional.softmax(logits, dim=-1)
        return probs


###############
# IMPORTAÇÕES #
###############
import pandas as pd
import keras
from keras.models import Sequential
from keras.layers import Dense, Dropout
from keras.wrappers.scikit_learn import KerasClassifier
from sklearn.model_selection import cross_val_score

###############################
#  LEITURA DOS ARQUIVOS .CSV  #
###############################
previsores = pd.read_csv('entradas-breast.csv')
classe = pd.read_csv('saidas-breast.csv')

###############################
#       FUNÇÃO CRIAR REDE     #
###############################
def criarRede():
    classificador = Sequential()
    classificador.add(Dense(units = 16, activation = 'relu',
                        kernel_initializer = 'uniform', input_dim = 30))
    classificador.add(Dropout(0.2))
    classificador.add(Dense(units = 16, activation = 'relu',
                        kernel_initializer = 'normal'))
    classificador.add(Dropout(0.2))
    classificador.add(Dense(units = 1, activation = 'sigmoid'))
    otimizador = keras.optimizers.Adam(lr = 0.001, decay = 0.0001, clipvalue = 0.5)
    classificador.compile(optimizer = otimizador, loss = 'binary_crossentropy',
                      metrics = ['binary_accuracy'])
    return classificador

#############################################
#      DEFINIÇÃO DAS ÉPOCAS E BATCH_SIZE    #
#############################################
classificador = KerasClassifier(build_fn = criarRede,
                                epochs = 500,
                                batch_size = 50)
resultados = cross_val_score(estimator = classificador,
                             X = previsores, y = classe,
                             cv = 10, scoring = 'accuracy')

###############################
#  GERAR RESULTADO E DESVIO   #
###############################
media = resultados.mean()
desvio = resultados.std()


def show(matrix):
    for row in matrix:
        print("\t".join(map(str, row)))
    print()

def in_bounds(x, y, m, n):
    if x < m or x >= n:
        return False
    elif y < m or y >= n:
        return False
    return True

def perimeter(layer, n):
    x = layer
    y = layer

    for dx, dy in [(0, 1), (1, 0), (0, -1), (-1, 0)]:
        while in_bounds(x + dx, y + dy, layer, n - layer):
            x = x + dx
            y = y + dy
            yield x, y

def rotate(matrix, n):
    for i in range(n // 2):
        prev = matrix[i][i]

        for x, y in perimeter(i, n):
            matrix[x][y], prev = prev, matrix[x][y]

if __name__ == '__main__':
    n = 5
    matrix = [[n * i + j for j in range(n)] for i in range(n)]

    show(matrix)
    rotate(matrix, n)
    show(matrix)


"""Tests."""
import datetime

from handlers import (add_new_idea, get_ideas_on_week,
                      register_user_or_find_existed)
from utils import ideas_to_text

def test_register_user(test_db, user_data):
    """
    Test user registration.

    Parameters:
        test_db: Db session.
        user_data: User data.
    """
    user = register_user_or_find_existed(user_data=user_data, db=test_db)
    assert user.telegram_id == user_data['telegram_id']  # noqa: S101

def test_register_user_if_user_restarted_bot(test_db, user_data):
    """
    Test user registration if user restarted the bot.

    Parameters:
        test_db: Db session.
        user_data: User data.
    """
    first_user = register_user_or_find_existed(
        user_data=user_data,
        db=test_db,
    )

    second_user = register_user_or_find_existed(
        user_data=user_data,
        db=test_db,
    )
    assert first_user.telegram_id == user_data['telegram_id']  # noqa: S101
    assert first_user.telegram_id == second_user.telegram_id  # noqa: S101
    assert first_user.id == second_user.id  # noqa: S101

def test_add_new_idea(test_db, user_data):
    """
    Test adding new idea.

    Parameters:
        test_db: Db session.
        user_data: User data.
    """
    user = register_user_or_find_existed(user_data=user_data, db=test_db)
    idea_data = {
        'user_id': user.id,
        'body': 'text',
        'date_created': datetime.datetime.now(),
    }
    idea = add_new_idea(idea_data=idea_data, db=test_db)

    assert idea.user_id == idea_data['user_id']  # noqa: S101
    assert idea.body == idea_data['body']  # noqa: S101
    assert idea.date_created == idea_data['date_created']  # noqa: S101

def test_get_ideas(test_db, user_data):  # noqa: WPS210
    """
    Test getting ideas.

    Parameters:
        test_db: Db session.
        user_data: User data.
    """
    user = register_user_or_find_existed(user_data=user_data, db=test_db)

    user_data = {'telegram_id': 1234}
    user2 = register_user_or_find_existed(user_data=user_data, db=test_db)

    idea_data = {
        'user_id': user.id,
        'body': 'text',
        'date_created': datetime.datetime.now(),
    }
    idea_data2 = {
        'user_id': user2.id,
        'body': 'text',
        'date_created': datetime.datetime.now(),
    }
    add_new_idea(idea_data=idea_data, db=test_db)
    add_new_idea(idea_data=idea_data, db=test_db)
    add_new_idea(idea_data=idea_data2, db=test_db)

    ideas = get_ideas_on_week(user=user, db=test_db)
    assert len(list(ideas)) == 2  # noqa: S101

def test_ideas_to_text(test_db, user_data):
    """
    Test ideas to text.

    Parameters:
        test_db: Db session.
        user_data: User data.
    """
    user = register_user_or_find_existed(user_data=user_data, db=test_db)

    idea_data = {
        'user_id': user.id,
        'body': 'text',
        'date_created': datetime.datetime.now(),
    }
    add_new_idea(idea_data=idea_data, db=test_db)
    add_new_idea(idea_data=idea_data, db=test_db)
    ideas = list(get_ideas_on_week(user=user, db=test_db))

    mess = ideas_to_text(ideas)

    assert type(mess).__name__ == 'str'  # noqa: S101


from bs4 import BeautifulSoup
import urllib2, re, datetime, os

url='http://apod.nasa.gov/apod/'
response=urllib2.urlopen(url)
content=response.read()
response.close()
soup = BeautifulSoup(content)
url=url+str(re.search(r'"([A-Za-z0-9_\./\\-]*)"', str(soup.find_all('a')[1])).group())[1:-1]
#print url
response=urllib2.urlopen(url)
store = open ( str(datetime.date.today())+".jpg" , 'w')
store.write ( response.read())
store.close()
response.close()

os.system("sqlite3 ~/Library/Application\ Support/Dock/desktoppicture.db \"update data set value = '"+str(os.getcwd())+ "/" +str(datetime.date.today())+".jpg"+"'\";")
os.system("killall Dock")


import numpy as np
from PIL import Image
import os
import torch
from torchvision import transforms
import torchvision.models as models
import pickle

imagefolder='/home/liel/lrp_toolbox/lrp_output/val/' #write here the directory of the outputs of lrp/sa (the directory includes the images themseld and the scores files).
files=os.listdir(imagefolder)
channels=3
h=224
width=224
imagenum=0
lrp_vals={}
images={}
lrp_vals_all={}
#creating the scores dictionary from the text files
for f in files:
	if 'as' in f:

		images[f.split('_as')[0]]=f
		print(f)
	if 'rawhm' in f:
		with open(imagefolder+f,'r') as img:
			imagename=f.split('_raw')[0]
			#print(imagename)
			for i,line in enumerate(img):
				if i>=2:
					l=line.split(' ')[:-1]
					if i-imagenum*h-2>=h:
						imagenum+=1
					for j,val in enumerate(l):
						key=str(imagenum)+','+str(i-imagenum*h-2)+','+str(j)
						if key not in lrp_vals:
							lrp_vals[key]=float(val)
						else:
							lrp_vals[key]+=float(val)

				elif  i==0:
					channels=int(line.rstrip('\n'))
				else:
					h,w=line.split(' ')[:2]
					h=int(h)
					w=int(w)

			lrp_vals_sorted=sorted(lrp_vals.items(),key=lambda item: item[1],reverse=True)[:50]
			lrp_vals_all[imagename]=lrp_vals_sorted
			lrp_vals={}
		img.close()
		#print(lrp_vals_sorted[:9])
		imagenum=0

label_file='/home/liel/lrp_toolbox/caffe-master-lrp/data/ilsvrc12/val.txt'
labels={}
with open(label_file,'r') as l:
	for line in l:
		img=line.split(' ')
		labels[img[0]]=int(img[1])

googlenet = models.googlenet(pretrained=True)#we used pytorch pretrained googlenet model for the pertubations evaluation
acc=0
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                 std=[0.229, 0.224, 0.225])
googlenet.eval()
original_score=0
scores=[]

for j in range(25):
	acc=0
	for i,img in enumerate(images):
		imgs=Image.open(imagefolder+images[img])

		trans=transforms.ToTensor()
		imgs=trans(imgs)
		imgs=normalize(imgs)

		#the pertubation phase
		for k in range(j):
			#print(k)
			loc=lrp_vals_all[img][k][0].split(',')
			color=int(loc[0])
			loc=loc[1:]

			top,right,left,bottom=(int(loc[0])+4,int(loc[1])+4,int(loc[1])-4,int(loc[0])-4)
			if bottom<0:
				bottom=0
			if right>223:
				right=223
			if left<0:
				left=0
			if top>223:
				top=223
			imgs[:,bottom:top+1,left:right+1]=torch.randint(low=0,high=256,size=(3,top+1-bottom,right+1-left))

		imgs=torch.unsqueeze(imgs, 0)
		#print(imgs.size())

		outputs=googlenet(imgs)
		#print(torch.sort(outputs, descending=True)[:10])
		_,indexs = torch.sort(outputs,dim=1,descending=True)
		indexs=indexs[0,:5]
		#print(index,labels[img])
		if i==0:
			print(indexs)
		if torch.tensor(labels[img]) in indexs:
			acc+=1
	if j==0:
		original_score=acc/len(list(images.keys()))
		print(original_score)
	score=acc/len(list(images.keys()))
	scores.append(score/original_score)
	print(score,score/original_score)

#rename the file according to the algorithm that created the scores in the image folder - of the algorthm is lrp write lrp, if sa write sa
pickle.dump(scores,open('scores_lrp_9x9_1.pkl','wb'))


from django.urls import path

from .views import *

app_name = 'weather'

urlpatterns = [
    path('', MainPage.as_view(), name='index'),
    path('requests/', RequestsList.as_view(), name='requests-list'),
    path('requests/<int:pk>/delete/', RequestDelete.as_view(), name='request-delete'),
    path('delete/<int:pk>/', GetUserLocationWeather.as_view(), name='get-user-location-weather'),
]


import codecs
import json
import random
import re
import os

from scrapy.spiders import Spider

class FollowersSpider(Spider):
    """
    从programmableWeb上爬取每个api的follower信息，爬取全部的
    """
    name = 'pw_followers_all'
    start_urls = ['https://www.programmableweb.com']
    download_delay = 0.2  # 设置爬取网页的间隔，避免速度过快导致被封停

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        print("init")
        with codecs.open("./proxyIP/https_verified.txt", 'r', encoding='utf-8') as reader:
            self.proxies = reader.readlines()
        self.crawled_api_ids = []  # 记录已经爬取的api的id
        self.target_number = 0  # 记录还需要爬取的api的数目
        self.page_counter = 0  # 记录本次启动爬虫爬取的页面的数目
        if not os.path.exists("api_name_num_followers_mapping_all.txt"):   # 如果不存在文件，则创建
            open("api_name_num_followers_mapping_all.txt", 'w')
        if os.path.exists("api_name_num_followers_mapping_8459.txt"):  # 检查8,459个api的n_followers数据是否存在
            with codecs.open("api_name_num_followers_mapping_8459.txt", 'r', encoding='utf-8') as reader:
                lines = reader.readlines()
                for line in lines:
                    self.crawled_api_ids.append(line.split(" ")[0])
        with codecs.open("api_name_num_followers_mapping_all.txt", 'r', encoding='utf-8') as reader:
            lines = reader.readlines()
            for line in lines:
                self.crawled_api_ids.append(line.split(" ")[0])
        self.writer = codecs.open("api_name_num_followers_mapping_all.txt", 'a', encoding='utf-8')
        with codecs.open("../apis/all/apis.json", 'r', encoding='utf-8') as reader:
            self.api_list = json.load(reader)

        print("原数据共有", len(self.api_list), "个api")
        print("已经爬取到", len(self.crawled_api_ids), "个api")
        self.target_number = len(self.api_list) - len(self.crawled_api_ids)
        print("仍需爬取", self.target_number, "个api")

    def parse(self, response):
        for api in self.api_list:
            if api['api_id'] not in self.crawled_api_ids:  # 说明当前api的follower信息还没被爬取过
                url = api['api_pw_url']
                request = response.follow(url, callback=self.parse_one)
                request.meta['api_id'] = api['api_id']  # 将api_id作为参数传递过去
                yield request
                # yield response.follow(url, callback=self.parse_one)

    def parse_one(self, response):
        """
        爬取到一个页面后，处理并保存数据
        :param response:
        :return:
        """
        self.page_counter += 1
        print("已爬取到第", self.page_counter, "个界面")
        print("id", response.meta['api_id'])
        contents = response.xpath(
            '//section[@id="block-views-api-followers-row-top"]/div[@class="block-title"][1]/span[1]').extract()
        if len(contents) > 0:
            span_content = contents[0]
            follower_num = int(re.findall(re.compile(r'[(](.*?)[)]', re.S), span_content)[0])
            self.writer.write(str(response.meta['api_id']) + " " + str(follower_num) + "\n")
            self.writer.flush()
        else:
            print("未找到followers信息")
            self.writer.write(str(response.meta['api_id']) + " 0\n")
            self.writer.flush()

    def get_random_proxy(self):
        """
        随机获取一个可用的代理
        :return:
        """
        proxy = random.choice(self.proxies)
        return proxy


import time
import board
from analogio import AnalogIn
import digitalio
import microcontroller
import busio

time_interval = 0.5

analog_in = AnalogIn(board.IO12)

def get_voltage(pin):
    return (pin.value * 3.3) / 65536

def test_voltage_measure():
    while True:
        print(get_voltage(analog_in))
        time.sleep(time_interval)

led = digitalio.DigitalInOut(board.IO41)
led.direction = digitalio.Direction.OUTPUT

def test_blink():
    while True:
        led.value = True
        time.sleep(time_interval)
        led.value = False
        time.sleep(time_interval)

def test_temperature_measure():
    while True:
        print(microcontroller.cpu.temperature)
        time.sleep(time_interval)

i2c = busio.I2C(board.IO11, board.IO10)

def i2c_scan():
    while True:
        print("I2C addresses:",[hex(device) for device in i2c.scan()])
        time.sleep(2)

#test_blink()
test_voltage_measure()
#test_temperature_measure()


import view
import model

current_position = 'MAIN'
current_class = None
current_lesson = None
current_student = None
classes_data = None

def start():
    global current_position, current_class, classes_data, current_lesson, current_student
    while 1:
        current_class_name = None if current_class == None else model.get_class_name(
            current_class)
        now_position_menu = view.print_menu(
            current_position, current_class_name, current_lesson, current_student, classes_data)
        if (now_position_menu == 2):
            exit()
        if (now_position_menu == 1):
            view.print_class_list(model.get_class_list())
            view.press_enter()
        if (now_position_menu == 3):
            current_position = 'CLASS'
            current_class = view.select_class()
            classes_data = model.select_class(current_class)
            current_lesson = None
            current_student = None
        if (now_position_menu == 6):
            current_position = 'MAIN'
            current_class = None
            current_lesson = None
            classes_data = None
            current_student = None
        if (now_position_menu == 4 and current_class != None):
            view.print_lessons(classes_data)
            view.press_enter()
        if (now_position_menu == 5 and current_class != None):
            current_position = 'LESSON'
            current_lesson = view.select_lesson()
            current_student = None
        if (now_position_menu == 10 and current_class != None):
            current_position = 'CLASS'
            current_lesson = None
            current_student = None
        if (now_position_menu == 8 and current_class != None):
            current_position = 'STUDENT'
            current_student = view.select_student()
        if (now_position_menu == 7 and current_class != None):
            view.print_students(current_lesson, classes_data)
            view.press_enter()
        if (now_position_menu == 9 and current_class != None):
            if (model.enter_rating(view.enter_rating(), current_lesson, current_student, classes_data)) == False:
                view.print_error_rating()
        if (now_position_menu == 11 and current_class != None):
            model.save_all(current_class, classes_data)
            view.print_save_ok()


from .views import (
    CountAllView,
    GetActions,
    Datasets,
    GetAdverseEventTargetPath,
    GetAverageSimilarity,
    GetGlobalAverageSimilarity,
    # GetGwas,
    # GetHGene,
    # GetHProtein,
    # GetIntact,
    # GetPathway,
    # GetReactome,
    # GetSignor,
    GetTargetAdverseEventPath,
    GetTargetByAdverseEventView,
    GetWeightedAverageSimilarity,
    RoutesListAPIView,
    CountView,
    GetAdverseEventByTargetView,
    # GetPheno,
    SuggestHintView,
    GetSimilarity

)
from django.urls import path

from . import views

app_name = "search"

urlpatterns = [

    # path('api/pheno/<str:target>/', GetPheno.as_view(), name='pheno'),
    # path('api/gwas/<str:target>/', GetGwas.as_view(), name='gwas'),
    # path('api/hgene/<str:target>/', GetHGene.as_view(), name='hgene'),
    # path('api/hprotein/<str:target>/', GetHProtein.as_view(), name='hprotein'),
    # path('api/intact/<str:target>/', GetIntact.as_view(), name='intact'),
    # path('api/pathway/<str:target>/', GetPathway.as_view(), name='pathway'),
    # path('api/reactome/<str:target>/', GetReactome.as_view(), name='reactome'),
    # path('api/signor/<str:target>/', GetSignor.as_view(), name='signor'),

    # Return the list of routes in the Django site
    path('api/routes/', RoutesListAPIView.as_view(), name='api-routes-list'),

    # Return an array of actions for the specified target
    path("api/actions/", GetActions.as_view(), name='get_actions'),
    path('api/actions/<str:target>/', GetActions.as_view(), name='get_actions_target'),

    path("api/descriptors", views.DescriptorListView.as_view(), name="descriptors"),

    # Return list of all node similarity scores associated to a target
    path('api/similarity/<str:descriptor>/<str:target>/', GetSimilarity.as_view(), name='similarity'),
    path('api/average_similarity/<str:target>/', GetAverageSimilarity.as_view(), name='average_similarity'),
    path('api/weighted_average_similarity/<str:target>/', GetWeightedAverageSimilarity.as_view(), name='weighted_average_similarity'),
    path('api/global_average_similarity/<int:min_descriptors>/', GetGlobalAverageSimilarity.as_view(), name='global_average_similarity'),

    # Upload one or more entities in a comma-separated file
    path('api/csv/', views.upload_csv, name='upload_csv'),

    # Return the content of a previously uploaded comma-separated file
    path('api/csv/<str:file_id>/', views.get_csv, name='get_csv'),

    # Clear out the database
    path('api/clear/', views.clear, name='clear'),

    # Initialize entities (all or of the specified type) from the OpenTargets store
    path('api/init/', views.init_type, name='init_all'),
    path('api/init/<str:type_string>/', views.init_type, name='init_type'),

    # Add a single gene entity to the database
    path('api/gene/<str:id>/', views.gene, name='gene'),

    # GET: Return an array of all known datasets (both active and inactive)
    # POST: Modify the active status of one or more datasets
    path('api/datasets/', Datasets.as_view(), name='datasets'),

    # This route returns an array of adverse events associated with a specific target, optionally filtered by action types. It requires the drug target symbol as a path parameter
    # target: Drug Symbol
    path('api/weight/<str:target>/', GetAdverseEventByTargetView.as_view(), name='get_adverse_event_weights_from_ae'),

    # This route returns an array of weights (log likelihood ratios) of adverse events associated with a specific target. It requires the drug target symbol and adverse event ID (meddraId) as path parameters.
    # target: Drug Symbol, ae: meddraId
    path('api/weight/<str:target>/<str:ae>/', GetAdverseEventByTargetView.as_view(), name='get_weights_target_ae'),

    # This route returns an array of targets associated with a specific adverse event, optionally filtered by action types. It requires the meddraId as a path parameter
    # target: ae: meddraId
    path('api/ae-weight/<str:ae>/', GetTargetByAdverseEventView.as_view(), name='get_target_weights_from_ae'),

    # This route returns an array of weights (log likelihood ratios) of and adverse events association for several targets. It requires the adverse event ID (meddraId) and drug target symbol as path parameters.
    # target: Drug Symbol, ae: meddraId
    path('api/ae-weight/<str:ae>/<str:target>/', GetTargetByAdverseEventView.as_view(), name='get_weights_ae_target'),

    # These paths define API routes for querying paths from a target to one or all adverse events
    # associated with it, optionally filtered by drug and action.
    # target_symbol: Symbol, adverse_event: meddraId, drug_id: chemblId
    path('api/ae/path/<str:target_symbol>/', GetAdverseEventTargetPath.as_view(), name='get_paths_target'),
    path('api/ae/path/<str:target_symbol>/<str:adverse_event>/', GetAdverseEventTargetPath.as_view(), name='get_paths_target_ae'),
    path('api/ae/path/<str:target_symbol>/<str:adverse_event>/<str:drug_id>/', GetAdverseEventTargetPath.as_view(), name='get_paths_target_ae_drug'),

    # These paths define API routes for querying paths from an adverse event to one or all targets
    # associated with it, optionally filtered by drug and action.
    # adverse_event: meddraId, target_symbol: Symbol, drug_id: chemblId
    path('api/target/path/<str:adverse_event>/', GetTargetAdverseEventPath.as_view(), name='get_paths_ae'),
    path('api/target/path/<str:adverse_event>/<str:target_symbol>/', GetTargetAdverseEventPath.as_view(), name='get_paths_ae_target'),
    path('api/target/path/<str:adverse_event>/<str:target_symbol>/<str:drug_id>/', GetTargetAdverseEventPath.as_view(), name='get_paths_ae_target_drug'),

    # Return an array of Cytoscape entities representing paths from a target to one or all adverse events associated with it, optionally filtered by drug and action
    path('api/count/', CountAllView.as_view(), name='count_all'),
#     path('api/count/', CountView.as_view(), name='count_all'),
    path('api/count/<str:type_string>/', CountView.as_view(), name='count_entity'),

    # Health check
    path('api/info/', views.info, name='info'),

    # Return an array of suggested entities in response to a hint (beginning of the name)
    path('api/suggest/<str:entity_type>/<str:hint>/', SuggestHintView.as_view(), name='suggest_hint'),

    # Return an array of actions for the specified target
    path('api/actions/<str:target>/', views.actions, name='actions_target'),

    # # Clear out the database
    path('api/clear/', views.clear, name='clear'),

    # # Initialize entities (all or of the specified type) from the OpenTargets store
    # path('api/init/', views.init_type, name='init_all'),
    # path('api/init/<str:type_string>/', views.init_type, name='init_type'),

    # # Add a single gene entity to the database
    # path('api/gene/<str:id>/', views.gene, name='gene'),

    # # Return an array of all known datasets (both active and inactive)
    # path('api/datasets/', views.datasets, name='datasets'),

    # # Modify the active status of one or more datasets
    # path('api/enable_datasets/', views.enable_datasets, name='enable_datasets'),

    # # Return an array of adverse events associated with a specific target, optionally filtered by action
    # path('api/weight/<str:target>/', views.get_adverse_event, name='get_adverse_event'),

    # # Return an array of weights of adverse events associated with a specific target, optionally filtered by action
    # path('api/weight/<str:target>/<str:ae>/', views.get_weights_target_ae, name='get_weights_target_ae'),

    # # Return an array of Cytoscape entities representing paths from a target to one or all adverse events associated with it, optionally filtered by drug and action
    # path('api/ae/path/<str:target>/', views.get_paths_target_ae_drug_view, name='get_paths_target_ae'),
    # path('api/ae/path/<str:target>/<str:ae>/', views.get_paths_target_ae_drug_view, name='get_paths_target_ae_ae'),
    # path('api/ae/path/<str:target>/<str:ae>/<str:drug_id>/', views.get_paths_target_ae_drug_view, name='get_paths_target_ae_drug'),

    # # Return an array of Cytoscape entities representing paths from a target to one or all adverse events associated with it, optionally filtered by drug and action
    # path('api/count/<str:type_string>/', views.count, name='count'),

    # # Health check
    # path('api/info/', views.info, name='info'),

    # # Return an array of suggested entities in response to a hint (beginning of the name)
    # path('api/suggest/<str:hint>/', views.suggest_hint, name='suggest_hint'),

    # # Return an array of all actions in the database
    # path('api/actions/', views.actions, name='actions'),

    # # Return an array of actions for the specified target
    # path('api/actions/<str:target>/', views.actions, name='actions_target')

    # # TODO Add some paths for retrieving similarity information?

]