import numpy as np
from matplotlib.pyplot import *
from sklearn import linear_model
from sklearn.utils import ConvergenceWarning
from methods import findPeaks, createCustomMatrix
import itertools, sys, pickle, datetime, os, warnings, ipdb

DIRNAME = 'out_lasso_path'

def init():
    """Initialize parameters and variables."""
    time_i, time_f, dt = 0.0, 100.0, 0.02043359821692
    search_low, search_high, search_num = 10.0, 30.0, 1000
    delta_p_range = np.linspace(-9, 9, 40)
    SNR_range = np.linspace(1, 10, 40)
    p = 20.0

    noise_std_range = 1.0 / np.sqrt(SNR_range)
    time = np.arange(time_i, time_f, dt)

    search_periods = np.linspace(search_low, search_high, search_num)

    A = createCustomMatrix(time, search_periods)

    m = itertools.product(delta_p_range, SNR_range)
    variables = [a[0] + (a[1],) for a in zip(itertools.product(
        delta_p_range, SNR_range), range(delta_p_range.size*SNR_range.size))]

    grid = np.array(list(m)).reshape((delta_p_range.size, SNR_range.size, 2))

    return A, time, p, delta_p_range, SNR_range, variables, grid, search_periods

def createAlphaDirs(alpha):
    """Create necessary directories and filenames."""
    alphadir = os.path.join(DIRNAME, 'alpha_{:.1e}'.format(alpha))
    plotdir = os.path.join(alphadir, 'plots')
    rfile = os.path.join(alphadir, 'results.txt')
    figfile = os.path.join(alphadir, 'plot.png')
    if not os.path.isdir(plotdir):
        os.makedirs(plotdir)
    return alphadir, plotdir, rfile, figfile

def createArrays(N_dp, N_snr, search_num):
    """Initialize empty arrays for filling later."""
    diffs = np.empty((N_dp, N_snr))
    diffs.fill(np.nan)
    powers = np.empty((N_dp, N_snr, search_num))
    return diffs, powers

def plotImage(diffs, delta_p_range, SNR_range, alpha):
    """Create the main difference image plot."""
    fig = figure()
    img = imshow(diffs, aspect='auto', interpolation='nearest', extent=[
        SNR_range[0], SNR_range[-1], delta_p_range[-1], delta_p_range[0]])
    colorbar()
    title('$\\Delta(\\Delta P_\\mathrm{{input}}, \\Delta P_\\mathrm{{output}})$; '
          'alpha = {:.1e}'.format(alpha))
    xlabel('SNR ($1/\\sigma_\\mathrm{{noise}}^2$)')
    ylabel('Input $\\Delta$ period')
    return fig, img

def LassoDiffWrapper(args):
    return LassoDiff(*args)

def LassoDiff(clf, A, time, p, delta_p, SNR, i=None):
    flux = np.sin(2*np.pi/p * time) + np.sin(2*np.pi/(p + delta_p) * time)
    flux += np.random.normal(0, flux.std()/np.sqrt(SNR), flux.size)

    convWarning = False
    with warnings.catch_warnings(record=True) as w:
        # clf.fit(A, flux)
        clf = linear_model.lasso_path(A, flux)

        if len(w) == 1 and type(w[0].message) is ConvergenceWarning:
            convWarning = True

    # ipdb.set_trace()
    coeffs = clf[1][:,99]

    power = coeffs[:A.shape[1]//2]**2 + coeffs[A.shape[1]//2:A.shape[1]//2 * 2]**2

    peaks = findPeaks(power)
    if peaks.size > 1:
        d2P_dp2 = np.gradient(np.gradient(power))
        p1_i, p2_i = peaks[-4:][d2P_dp2[peaks[-4:]].argsort()][:2]
    else:
        p1_i, p2_i = power.argsort()[-2:]

    return i, power, p1_i, p2_i, convWarning

def recordProgress(start, n, N_pix, rfile):
    stop = datetime.datetime.now()
    avg = (stop - start).total_seconds() / n
    remain = datetime.timedelta(seconds = avg * (N_pix - n))
    eta = stop + remain
    progress = '{:d}/{:d} pixels completed. Avg {:.2f} sec/pix, {:s} remaining, ETA {:s}    '.format(n, N_pix, avg,
        str(remain).split('.')[0], eta.strftime('%x %I:%M:%S %p'))
    with open(rfile, 'w') as f:
        f.write(progress+'\n')
    sys.stdout.write('\r'+progress)
    sys.stdout.flush()

def update(i, diffs, diff, powers, power, fig, img):
    """Update the data arrays and difference image plot."""
    r, c = np.unravel_index(i, diffs.shape)
    diffs[r,c] = diff
    powers[r,c] = power
    img.set_data(diffs)
    img.set_clim(np.nanmin(diffs), np.nanmax(diffs))
    fig.canvas.draw()
    fig.savefig(figfile, dpi=200)

def plotCoefficients(variables, i, search_periods, p1_i, p2_i, diff, power, alpha, plotdir, convWarning):
    """Plot the coefficients."""
    dp, SNR = variables[i][:2]
    p1_input, p2_input = 20, 20 + dp
    p1_output, p2_output = search_periods[[p1_i, p2_i]]
    f, ax = subplots()
    ax.set_title('$P_1 = {:.2f}, P_2 = {:.2f},$ SNR = {:.2f}, alpha = {:.1e}'.format(p1_input, p2_input, SNR, alpha))
    ax.set_xlabel('Period [days]')
    ax.set_ylabel('Lasso Power')
    ax.axvline(p1_input, 0, 1, c='b')
    ax.axvline(p2_input, 0, 1, c='b', label='Input periods')
    color = 'r' if convWarning else 'g'
    label = 'Convergence Warning' if convWarning else 'Lasso converged'
    loc = 'upper right' if dp < 0 else 'upper left'
    ax.plot(search_periods, power, color+'o-', ms=4, label=label)
    ax.plot(search_periods[[p1_i, p2_i]], power[[p1_i, p2_i]], 'o',
        mec='m', mew=2, mfc='None', ms=10, label='Detected Periods')
    ax.margins(0.05, 0.05)
    ax.legend(fontsize=10, loc=loc)
    # if dp < 0:
    #     kwargs = {'xy':(0.99, 0.8), 'ha':'right'}
    # else:
    #     kwargs = {'xy':(0.01, 0.8), 'ha':'left'}
    # ax.annotate('$\\Delta P_\\mathrm{{in}} = {0:.2f}$\n'
    #     '$\\Delta P_\\mathrm{{out}} = {1:.2f}$\n'
    #     '$\\Delta(\\Delta P_\\mathrm{{in}}, \\Delta P_\\mathrm{{out}}) = {2:.2f}$'.format(abs(dp), abs(p1_output - p2_output), diff),
    #     xycoords='axes fraction', va='top', **kwargs)
    f.savefig(os.path.join(plotdir, '{:d}.png'.format(i)), dpi=200)
    close(f)

def alphaFinal(diffs, powers, grid, alphadir, rfile, start):
    """Save data and output last line to progress file."""
    data = {
        'diffs': diffs,
        'powers': powers,
        'grid': grid
    }
    with open(os.path.join(alphadir, 'data.pkl'), 'wb') as dataFile:
        pickle.dump(data, dataFile)
    with open(rfile, 'a') as f:
        f.write('Saved data file. Took {} to complete.\n'.format(
            str(datetime.datetime.now() - start).split('.')[0]))

if __name__ == '__main__':

    A, time, p, delta_p_range, SNR_range, variables, grid, search_periods = init()
    N_pix = delta_p_range.size * SNR_range.size

    a0 = 0.00001
    # alpha_vars = (a0*2, a0/4, a0/2, a0, a0*4)
    # for alpha in alpha_vars:

    alpha = 0.00001

    alphadir, plotdir, rfile, figfile = createAlphaDirs(alpha)
    diffs, powers = createArrays(delta_p_range.size, SNR_range.size, search_periods.size)
    fig, img = plotImage(diffs, delta_p_range, SNR_range, alpha)
    clf = linear_model.Lasso(alpha=alpha, fit_intercept=False)
    args = [(clf, A, time, p,) + a for a in variables]
    start = datetime.datetime.now()

    for n, (i, power, p1_i, p2_i, convWarning) in enumerate(map(LassoDiffWrapper, args), 1):
        recordProgress(start, n, N_pix, rfile)
        p1, p2 = search_periods[[p1_i, p2_i]]
        diff = abs(abs(p1 - p2) - abs(variables[i][0]))
        update(i, diffs, diff, powers, power, fig, img)
        plotCoefficients(variables, i, search_periods, p1_i, p2_i, diff, power, alpha, plotdir, convWarning)

    close(fig)

    alphaFinal(diffs, powers, grid, alphadir, rfile, start)


# -*- coding:UTF-8 -*-

import tushare as ts
import numpy as np
import pandas as pd
import os
import time
import datetime
import sys
import math
sys.path.append("..")
from common import base_common
from common import np_common
import tushare_data
import feature

def CreateDSFa3DSplitMTFunc(param, msg):
    param.CreateDSFa3DSplitStock(msg)

class DSFa3DDataset():
    def __init__(self,
                 o_data_source,
                 o_feature):
        self.data_source = o_data_source
        self.feature = o_feature
        self.date_list = self.data_source.date_list
        self.code_list = self.data_source.code_list
        self.date_index_map = self.data_source.date_index_map
        self.code_index_map = self.data_source.code_index_map

    def FileNameDSFa3DDataset(self):
        temp_file_name = './data/dataset/DSFa3D_%s_%s.npy' %(\
            self.data_source.setting_name, \
            self.feature.setting_name)
        return temp_file_name

    def FileNameDSFa3DSplit(self, ts_code):
        temp_file_name = './data/DSFa3D/%s_%s_%s.npy' %(\
            self.data_source.setting_name_stock, \
            self.feature.setting_name, \
            ts_code)
        return temp_file_name

    def CreateDSFa3DSplitStock(self, ts_code):
        data_unit_date_index = self.feature.index_date
        pp_data = self.data_source.LoadStockPPData(ts_code)
        if len(pp_data) == 0:
            return
        dateset_index2 = self.code_index_map[ts_code]
        for day_loop in range(0, len(pp_data)):
            data_unit = self.feature.GetDataUnit(pp_data, day_loop)
            if len(data_unit) == 0:
                continue
            temp_date = int(data_unit[data_unit_date_index])
            if temp_date > self.data_source.EndDate():
                continue
            if temp_date < self.data_source.start_date:
                break
            dateset_index1 = self.date_index_map[temp_date]
            self.dataset[dateset_index1][dateset_index2] = data_unit
        sys.stdout.write("DSFa - %-4d : %s 100%%\n" % (self.code_index_map[ts_code], ts_code))

    def CreateDSFa3DDataset(self):
        dataset_file_name = self.FileNameDSFa3DDataset()
        self.dataset = np.zeros((len(self.date_list), len(self.code_list), self.feature.unit_size))
        # base_common.ListMultiThread(CreateDSFa3DSplitMTFunc, self, 1, self.code_list)
        for ts_code in self.code_list:
            self.CreateDSFa3DSplitStock(ts_code)
        base_common.MKFileDirs(dataset_file_name)
        np.save(dataset_file_name, self.dataset)
        self.dataset = None

    def GetDSFa3DDataset(self):
        dataset_file_name = self.FileNameDSFa3DDataset()
        if not os.path.exists(dataset_file_name):
            self.CreateDSFa3DDataset()
        return np.load(dataset_file_name)

if __name__ == "__main__":
    data_source = tushare_data.DataSource(20000101, '', '', 1, 20000101, 20200106, False, False, True)
    data_source.ShowStockCodes()

    o_feature = feature.Feature(30, feature.FUT_D5_NORM, 1, False, False)

    o_dataset = DSFa3DDataset(data_source, o_feature)
    temp_dataset = o_dataset.GetDSFa3DDataset()
    print("dataset: {}".format(temp_dataset.shape))


# © 2015-2016 Akretion - Alexis de Lattre <alexis.delattre@akretion.com>
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html).

from odoo import _, api, models
from odoo.exceptions import ValidationError

class ResBank(models.Model):
    _inherit = "res.bank"

    @api.constrains("bic")
    def check_bic_length(self):
        for bank in self:
            if bank.bic and len(bank.bic) not in (8, 11):
                raise ValidationError(
                    _(
                        "A valid BIC contains 8 or 11 characters. The BIC '%s' "
                        "contains %d characters, so it is not valid."
                    )
                    % (bank.bic, len(bank.bic))
                )

# starting from v9, on res.partner.bank bank_bic is a related of bank_id.bic


from tensorflow.python.eager import backprop
from tensorflow.python.framework import test_util
from tensorflow.python.platform import test

class OpsDefsTest(test.TestCase):
  """Test utils."""

  def _assertOpAndComposite(self, vars_, compute_op, compute_composite, kwargs,
                            op_kwargs=None):
    if op_kwargs is None:
      op_kwargs = kwargs
    if test_util.IsMklEnabled():
      self.skipTest("Not compatible with oneDNN custom ops.")

    # compute with op.
    with backprop.GradientTape() as gt:
      for var_ in vars_:
        gt.watch(var_)
      y = compute_op(**op_kwargs)  # uses op and decomposites by the graph pass.
      grads = gt.gradient(y, vars_)  # uses registered gradient function.

    # compute with composition
    with backprop.GradientTape() as gt:
      for var_ in vars_:
        gt.watch(var_)
      re_y = compute_composite(**kwargs)  # uses composite function.
      re_grads = gt.gradient(re_y, vars_)  # uses gradients compposite function.

    for v, re_v in zip(y, re_y):
      self.assertAllClose(v, re_v)
    for g, re_g in zip(grads, re_grads):
      self.assertAllClose(g, re_g)


import cv2
import numpy as np
import matplotlib.pyplot as plt

img=cv2.imread('pic.jpg',cv2.IMREAD_COLOR)

px=img[55,55]
print(px)                  #to print the color value of that co ordinate

img[55,55]=[255,255,0]     #changing the color of that coordinate
print(px)

img[100:150,100:150]=[255,255,255]       #changing that image portion to white-ROI(region of image)

face=img[40:100,50:100]                  #copy and pasting regions of space
img[0:60,0:50]=face

cv2.imshow('img',img)
cv2.waitKey(0)
cv2.destroyAllWindows()


"""
The player's AI code
    Functions here are called by clock.py to run the AI code
"""

import random
import math

from clientLogic.logging import logPrint
from clientLogic import clientData, commands

def onConnect():
    """
    Called when the player initially connects to the server but before the tank first spawns
    """
    commands.setInfo("Python player instance running the example AI.\n" +
                     "Fork me at https://github.com/JoelEager/pyTanks.Player")

def onSpawn():
    """
    Called when the tank spawns in a new game
    """
    pass

def onTick(elapsedTime):
    """
    Called once every frame while the tank is alive
    :param elapsedTime: The time elapsed, in seconds, since the last frame
    """
    gs = clientData.gameState

    # Collided so try to get moving again
    if not gs.myTank.moving:
        commands.turn((math.pi / 4) * random.randint(0, 7))
        commands.go()
        logPrint("Turned and starting moving", 2)

    # Shooting logic
    if gs.myTank.canShoot and random.randint(0, 4) == 0:
        # Select a target
        random.shuffle(gs.tanks)
        for target in gs.tanks:
            if target.alive:
                # Do the math
                deltaX = abs(gs.myTank.x - target.x)
                if deltaX == 0: return
                deltaY = gs.myTank.y - target.y
                angle = math.atan(deltaY / deltaX)

                if target.x < gs.myTank.x:
                    angle = math.pi - angle

                commands.fire(angle)
                logPrint("Fired", 2)

                break


class Pattern_Sixteen:
    '''
        9
        9 8
        9 8 7
        9 8 7 6
        9 8 7 6 5
        9 8 7 6 5 4
        9 8 7 6 5 4 3
        9 8 7 6 5 4 3 2
        9 8 7 6 5 4 3 2 1
    '''

    def __init__(self, steps=9):
        self.steps = steps

    def method_one(self):
        print('\nMethod One')

        _range = [str(i) for i in range(1, self.steps + 1)][::-1]

        for gr in range(1, len(_range) + 1):
            print(' '.join(_range[:gr]))

    def method_two(self):
        print('\nMethod Two')

        for i in range(self.steps, 0, -1):
            for j in range(self.steps, i - 1, -1):
                print(j, end=' ')

            print()

if __name__ == '__main__':
    pattern_sixteen = Pattern_Sixteen()

    pattern_sixteen.method_one()
    pattern_sixteen.method_two()


a = "3aabacbebebe"
n = int(a[0])
listik = list()
setik = set()
word = str()
index_last_letter = len(a) - 1
for i in range(1, len(a)):
    index = i - 1
    for letter in a[i:]:
        if len(setik) < n or (len(setik) == n and letter in setik):
            setik.add(letter)
            word += letter
            index += 1
            if index == index_last_letter:
                listik.append(word)
                setik.clear()
                word = str()
        else:
            listik.append(word)
            setik.clear()
            word = str()
            break
print(listik)

result = max(*listik, key=lambda x: len(x))
print(result)


#!/usr/bin/env python3

# Source: https://github.com/cheran-senthil/PyRival/blob/master/pyrival/data_structures/SortedList.py

from bisect import bisect_left as lower_bound
from bisect import bisect_right as upper_bound

class FenwickTree:
    def __init__(self, x):
        bit = self.bit = list(x)
        size = self.size = len(bit)
        for i in range(size):
            j = i | (i + 1)
            if j < size:
                bit[j] += bit[i]

    def update(self, idx, x):
        """updates bit[idx] += x"""
        while idx < self.size:
            self.bit[idx] += x
            idx |= idx + 1

    def __call__(self, end):
        """calc sum(bit[:end])"""
        x = 0
        while end:
            x += self.bit[end - 1]
            end &= end - 1
        return x

    def find_kth(self, k):
        """Find largest idx such that sum(bit[:idx]) <= k"""
        idx = -1
        for d in reversed(range(self.size.bit_length())):
            right_idx = idx + (1 << d)
            if right_idx < self.size and self.bit[right_idx] <= k:
                idx = right_idx
                k -= self.bit[idx]
        return idx + 1, k

class SortedList:
    block_size = 700

    def __init__(self, iterable=()):
        self.macro = []
        self.micros = [[]]
        self.micro_size = [0]
        self.fenwick = FenwickTree([0])
        self.size = 0
        for item in iterable:
            self.insert(item)

    def insert(self, x):
        i = lower_bound(self.macro, x)
        j = upper_bound(self.micros[i], x)
        self.micros[i].insert(j, x)
        self.size += 1
        self.micro_size[i] += 1
        self.fenwick.update(i, 1)
        if len(self.micros[i]) >= self.block_size:
            self.micros[i:i + 1] = self.micros[i][:self.block_size >> 1], self.micros[i][self.block_size >> 1:]
            self.micro_size[i:i + 1] = self.block_size >> 1, self.block_size >> 1
            self.fenwick = FenwickTree(self.micro_size)
            self.macro.insert(i, self.micros[i + 1][0])

    def pop(self, k=-1):
        i, j = self._find_kth(k)
        self.size -= 1
        self.micro_size[i] -= 1
        self.fenwick.update(i, -1)
        return self.micros[i].pop(j)

    def __getitem__(self, k):
        i, j = self._find_kth(k)
        return self.micros[i][j]

    def count(self, x):
        return self.upper_bound(x) - self.lower_bound(x)

    def __contains__(self, x):
        return self.count(x) > 0

    def lower_bound(self, x):
        i = lower_bound(self.macro, x)
        return self.fenwick(i) + lower_bound(self.micros[i], x)

    def upper_bound(self, x):
        i = upper_bound(self.macro, x)
        return self.fenwick(i) + upper_bound(self.micros[i], x)

    def _find_kth(self, k):
        return self.fenwick.find_kth(k + self.size if k < 0 else k)

    def __len__(self):
        return self.size

    def __iter__(self):
        return (x for micro in self.micros for x in micro)

    def __repr__(self):
        return str(list(self))

class S:
    def __init__(self, l, r, i): self.l, self.r, self.i, self.ans = l, r + l, i, 0

# start -> max answer
n, best = int(input()), SortedList([(-1, 0)])
ps = sorted((S(*map(int, input().split()), i) for i in range(n)), key=lambda s: (-s.r, s.l))
for s in ps:
    # s.ans   = prev(best.upper_bound(s.l))->second + 1;
    it = best.upper_bound((s.l, n))
    s.ans = best[it - 1][1] + 1
    # auto it = best.insert_or_assign(s.l, s.ans).first;
    if best[it - 1][0] == s.l: best.pop(it - 1)
    else: it += 1
    best.insert((s.l, s.ans))
    # if(next(it) != best.end() && next(it)->second <= s.ans) best.erase(next(it));
    if it < len(best) and best[it][1] <= s.ans: best.pop(it)
print(*(s.ans - 1 for s in sorted(ps, key=lambda s: s.i)))


__author__ = 'Rio'
from yacs.config import CfgNode as CN
config = CN()
config.image_dir = '../data/images'
config.ann_path = '../data/my_annotation.json'
config.models = '../save_models/model_best.pth'

# Data loader settings
config.max_seq_length = 60
config.threshold = 3
config.num_workers = 0  # the number of workers for dataloader
config.batch_size = 16  # the number of samples for a batch
config.evaluate_batch = 1 # the number of samples in evaluate part

# Model settings (for visual extractor)
config.visual_extractor = 'resnet101'
config.visual_extractor_pretrained = True  # 'whether to load the pretrained visual

# Model settings (for Transformer)
config.d_model = 512  # the dimension of Transformer.
config.d_ff = 512  # the dimension of FFN.
config.d_vf = 2048  # the dimension of the patch features.
config.num_heads = 8  # the number of heads in Transformer.
config.num_layers = 3  # the number of layers of Transformer.
config.dropout = 0.1  # the dropout rate of Transformer.
config.logit_layers = 1  # the number of the logit layer.
config.bos_idx = 0  # the index of <bos>.
config.eos_idx = 0  # the index of <eos>.
config.pad_idx = 0  # the index of <pad>.
config.use_bn = 0  # whether to use batch normalization.
config.drop_prob_lm = 0.5  # the dropout rate of the output layer.

# Sample related
config.sample_n = 1  # the sample number per image.
config.output_logsoftmax = 1  # whether to output the probabilities.
config.decoding_constraintt = 0  # whether decoding constraint.

# Trainer settings
config.n_gpu = 1  # the number of gpus to be used.
config.epochs = 50  # the number of training epochs.
config.save_dir = '../result/models/'  # the patch to save the save_models.
config.record_dir = '../result/records/'  # the patch to save the results of experiments
config.save_period = 1  # the saving period.
config.monitor_mode = 'max'  # choices=['min', 'max'],help='whether to max or min the metric.
config.monitor_metric = 'BLEU_4'  # the metric to be monitored.
config.early_stop = 100  # the patience of training.

# Optimization
config.optim = 'Adam'  # the type of the optimizer.
config.lr_ve = 5e-5  # the learning rate for the visual extractor.
config.lr_ed = 1e-3  # the learning rate for the remaining parameters.
config.weight_decay = 5e-5  # 'the weight decay.
config.amsgrad = True

# Learning Rate Scheduler
config.lr_scheduler = 'StepLR'  # the type of the learning rate scheduler.
config.step_size = 28  # he step size of the learning rate scheduler.
config.gamma = 0.1  # the gamma of the learning rate scheduler.

# Others
config.seed = 9233
config.resume = None  # whether to resume the training from existing checkpoints.

# RNN_Encoder
config.embedding_vector = 300
config.nhidden = 512
config.nlayers = 1
config.bidirectional = True
config.rnn_type = 'LSTM'

# text_image_losses.py
config.cuda = True
config.train_smooth_gamma3 = 10.0
config.train_smooth_gamma2 = 5.0
config.train_smooth_gamma1 = 4.0
config.attn_pth = '../attn_pth/'


# Solution for Project Euler Problem 16

# Import Functions
from functions import *

# Constants:
x = 2
exponent = 1000
total_sum = 0
digits = str(x ** exponent)

# Main
for i in range(len(digits)):
    total_sum += int(digits[i])
print(total_sum)


# -*- coding: utf-8 -*-

# TODO MAKE ME A PACKAGE!

from datetime import datetime as dt
from datetime import timedelta as td
from copy import deepcopy

import airflow
from airflow import DAG
from airflow.operators.subdag_operator import SubDagOperator
from airflow.operators.dummy_operator import DummyOperator
from airflow.executors.celery_executor import CeleryExecutor

from airflow.contrib.operators.bigquery_to_gcs import BigQueryToCloudStorageOperator
from airflow.contrib.operators.dataproc_operator import DataprocClusterCreateOperator
from airflow.contrib.operators.dataproc_operator import DataProcPySparkOperator

from debussy.operators.basic import StartOperator, FinishOperator
from debussy.operators.bigquery import BigQueryTableFlushOperator, BigQueryRawToClean
from debussy.operators.datastore import DatastoreGetObjectOperator
from debussy.operators.datastore import DatastorePutObjectOperator
from debussy.operators.extractors import DatastoreExtractorTemplateOperator
from debussy.operators.extractors import JDBCExtractorTemplateCustomQueryOperator
from debussy.operators.extractors import JDBCExtractorTemplateOperator
from debussy.operators.notification import SlackOperator
from debussy.operators.python import MonthlyBranchPython

# INTERNALS BEGIN

def _create_subdag(subdag_func, parent_dag, task_id, phase, default_args,
    trigger_rule='all_success'):
    """Creates a subdag, initiated by a StartOperator and ended by a FinishOperator.
    Also, uses an internal function, passed as argument, to create the other subdag
    operators in between.
    """
    subdag = DAG(
        dag_id='{}.{}'.format(parent_dag.dag_id, task_id),
        schedule_interval=None,
        catchup=False,
        default_args=default_args
    )

    begin_task = StartOperator(phase, trigger_rule=trigger_rule, **default_args)
    subdag >> begin_task
    end_task = FinishOperator(phase, trigger_rule=trigger_rule, **default_args)
    subdag >> end_task
    subdag_func(begin_task, end_task)

    return SubDagOperator(
        subdag=subdag,
        task_id=task_id,
        dag=parent_dag,
        executor=CeleryExecutor()
    )

def _create_bigquery_simple_mirror_subdag(parent_dag, project_params, extractor_class,
    task_id_format, default_args):
    """Creates a subdag for simple mirroring, without "raw" and "clean" stages.
    """
    project_id = project_params['project_id']
    config = project_params['config']

    metadata_params_get = dict(project_params['metadata_params_get'])
    metadata_params_put = dict(project_params['metadata_params_put'])

    extractor_params = dict(project_params['extract'])
    extractor_params['project'] = project_id
    extractor_params['config'] = config
    extractor_params.update(default_args)

    table = extractor_params['table']

    def _internal(begin_task, end_task):
        datastore_get_task = DatastoreGetObjectOperator(**metadata_params_get)
        extract_task = extractor_class(**extractor_params)
        datastore_put_task = DatastorePutObjectOperator(**metadata_params_put)

        begin_task >> datastore_get_task >> extract_task >> datastore_put_task
        datastore_put_task >> end_task

    return _create_subdag(
        _internal,
        parent_dag,
        task_id_format.format(table),
        table,
        default_args
    )

def _create_bigquery_mirror_subdag(parent_dag, project_params, extractor_class,
    conversor_wrapper, task_id_format, default_args):
    """Creates a subdag to mirror a table from a given source to BigQuery.
    """
    project_id = project_params['project_id']
    config = project_params['config']

    metadata_params_get = dict(project_params['metadata_params_get'])
    metadata_params_put = dict(project_params['metadata_params_put'])

    extractor_params = dict(project_params['extract'])
    extractor_params['project'] = project_id
    extractor_params['config'] = config
    extractor_params.update(default_args)

    bigquery_params = dict(project_params['bigquery'])
    bigquery_pool = bigquery_params['pool']
    bigquery_table = bigquery_params['table']
    raw_table_name = bigquery_params['raw_table_name']
    clean_table_name = bigquery_params['clean_table_name']

    def _internal(begin_task, end_task):
        datastore_get_task = DatastoreGetObjectOperator(**metadata_params_get)

        bq_flush_raw_task = BigQueryTableFlushOperator(
            project=project_id,
            config=config,
            table=bigquery_table,
            target_table_path=raw_table_name,
            pool=bigquery_pool,
            task_id='flush-raw-table-{}'.format(bigquery_table),
            **default_args
        )

        extract_task = extractor_class(**extractor_params)

        raw2clean_task = BigQueryRawToClean(
            project=project_id,
            config=config,
            table=bigquery_table,
            target_table_path=clean_table_name,
            source_table_path=raw_table_name,
            conversor_wrapper=conversor_wrapper,
            pool=bigquery_pool,
            **default_args
        )

        datastore_put_task = DatastorePutObjectOperator(**metadata_params_put)

        begin_task >> datastore_get_task >> bq_flush_raw_task >> extract_task
        extract_task >> raw2clean_task >> datastore_put_task >> end_task

    return _create_subdag(
        _internal,
        parent_dag,
        task_id_format.format(bigquery_table),
        bigquery_table,
        default_args
    )

def _create_full_load_to_bigquery(parent_dag, project_params, extractor_class,
    conversor_wrapper, task_id_format, default_args):
    """Creates a subdag to perform a full load of a table from a given source to BigQuery.
    """
    project_id = project_params['project_id']
    config = project_params['config']

    extractor_params = dict(project_params['extract'])
    extractor_params['project'] = project_id
    extractor_params['config'] = config
    extractor_params.update(default_args)

    bigquery_params = dict(project_params['bigquery'])
    bigquery_pool = bigquery_params['pool']
    bigquery_table = bigquery_params['table']
    raw_table_name = bigquery_params['raw_table_name']
    clean_table_name = bigquery_params['clean_table_name']

    def _internal(begin_task, end_task):
        bq_flush_raw_task = BigQueryTableFlushOperator(
            project=project_id,
            config=config,
            table=bigquery_table,
            target_table_path=raw_table_name,
            pool=bigquery_pool,
            task_id='flush-raw-table-{}'.format(bigquery_table),
            **default_args
        )

        extract_task = extractor_class(**extractor_params)

        bq_flush_clean_task = BigQueryTableFlushOperator(
            project=project_id,
            config=config,
            table=bigquery_table,
            target_table_path=clean_table_name,
            task_id='flush-clean-table-{}'.format(bigquery_table),
            pool=bigquery_pool,
            **default_args
        )

        raw2clean_task = BigQueryRawToClean(
            project=project_id,
            config=config,
            table=bigquery_table,
            target_table_path=clean_table_name,
            source_table_path=raw_table_name,
            conversor_wrapper=conversor_wrapper,
            pool=bigquery_pool,
            **default_args
        )

        begin_task >> bq_flush_raw_task >> extract_task >> bq_flush_clean_task
        bq_flush_clean_task >> raw2clean_task >> end_task

    return _create_subdag(
        _internal,
        parent_dag,
        task_id_format.format(bigquery_table),
        bigquery_table,
        default_args
    )

# INTERNALS END

# EXTERNALS BEGIN

def create_notification_subdag(parent_dag, env_level, phase, message, default_args):
    """Creates a subdag that notifies a given message in a slack channel.
    """
    def _internal(begin_task, end_task):
        notification_task = SlackOperator(phase, message, **default_args)
        begin_task >> notification_task >> end_task

    return _create_subdag(
        _internal,
        parent_dag,
        '{}_notification_subdag'.format(phase),
        phase,
        default_args
    )

def create_jdbc_bigquery_simple_mirror_subdag(parent_dag, project_params, default_args):
    return _create_bigquery_simple_mirror_subdag(
        parent_dag,
        project_params,
        JDBCExtractorTemplateCustomQueryOperator,
        'table_{}_simple_mirror_subdag',
        default_args
    )

def create_sqlserver_bigquery_mirror_subdag(parent_dag, project_params, conversor_wrapper,
    default_args):
    """DEPRECATED

    Creates a subdag for incremental mirroring of a SQL Server table into BigQuery.
    """
    return _create_bigquery_mirror_subdag(
        parent_dag,
        project_params,
        JDBCExtractorTemplateOperator,
        conversor_wrapper,
        'table_{}_mirror_subdag',
        default_args
    )

def create_jdbc_bigquery_mirror_subdag(parent_dag, project_params, conversor_wrapper,
    default_args):
    """Creates a subdag for incremental mirroring of a SQL Server table into BigQuery.
    """
    return _create_bigquery_mirror_subdag(
        parent_dag,
        project_params,
        JDBCExtractorTemplateOperator,
        conversor_wrapper,
        'table_{}_mirror_subdag',
        default_args
    )

def create_sqlserver_bigquery_full_load_subdag(parent_dag, project_params, conversor_wrapper,
    default_args):
    """Creates a subdag for full load of a SQL Server table into BigQuery.
    """
    return _create_full_load_to_bigquery(
        parent_dag,
        project_params,
        JDBCExtractorTemplateOperator,
        conversor_wrapper,
        'table_{}_full_load_subdag',
        default_args
    )

def create_datastore_bigquery_mirror_subdag(parent_dag, project_params, conversor_wrapper,
    default_args):
    return _create_bigquery_mirror_subdag(
        parent_dag,
        project_params,
        DatastoreExtractorTemplateOperator,
        conversor_wrapper,
        '{}_mirror_subdag',
        default_args
    )

def create_simple_pyspark_subdag(parent_dag, project_params, zone, region,
    storage_bucket, subnetwork_uri, main_file, extra_files, job_name, arguments,
    default_args):
    """Creates a simplified Dataproc cluster for processing a PySpark script.
    To make things as simple as possible, the cluster is fixed as a standard-2 master (no HA),
    4 standard-4 workers (2 of which preemptible), 200gb fixed disk size and GCS/BQ
    connectors (no other packages can be installed). The cluster auto-deletes with a 10min
    idle timeout, and only internal IPs. This subdag aims for a single, pure PySpark process,
    without much personalization and dependencies.
    :param parent_dag: the parent dag that this subdag will be associated
    :type parent_dag: DAG
    :param project_params: dict with project-wide configurations, such as project_id, dataset_id, etc
    :type project_params: dict
    :param zone: the GCP zone (e.g. us-central1-b) where the cluster will be created
    :type zone: str
    :param region: the GCP region (e.g. us-central1) where the cluster will be created
    :type region: str
    :param storage_bucket: the bucket where temporary data for the cluster will be stored
    :type storage_bucket: str
    :param subnetwork_uri: the full URI of the subnetwork where the Dataproc cluster will execute
        (e.g.: projects/project-id/regions/us-central1/subnetworks/subnet-id)
    :type subnetwork_uri: str
    :param main_file: the path of the file to be executed (should be in a GCS bucket that the cluster will have access to)
    :type main_file: str
    :param extra_files: the list of support files to be passed to the cluster (must be a Python file like .py or .egg)
    :type extra_files: list|str
    :param job_name: the name of the job that will be displayed in the cluster's jobs list
    :type job_name: str
    :param arguments: the list of arguments to be passed to the main file
    :type arguments: list|str
    """
    project_id = project_params['project_id']

    def _internal(begin_task, end_task):
        task_id = 'cluster-{}'.format(job_name)
        cluster_name = '{}-cluster'.format(job_name)

        cluster = DataprocClusterCreateOperator(
            task_id=task_id,
            project_id=project_id,
            zone=zone,
            cluster_name=cluster_name,
            num_workers=2,
            num_preemptible_workers=2,
            storage_bucket=storage_bucket,
            master_machine_type='n1-standard-2',
            master_disk_size=200,
            worker_machine_type='n1-standard-4',
            worker_disk_size=200,
            init_actions_uris=['gs://dataproc-initialization-actions/connectors/connectors.sh'],
            metadata={
                'gcs-connector-version': '1.9.16',
                'bigquery-connector-version': '0.13.16'
            },
            subnetwork_uri=subnetwork_uri,
            internal_ip_only=True,
            region=region,
            idle_delete_ttl=600,
            **default_args
        )

        job = DataProcPySparkOperator(
            task_id=job_name,
            main=main_file,
            arguments=arguments,
            pyfiles=extra_files,
            job_name=job_name,
            cluster_name=cluster_name,
            region=region,
            **default_args
        )

        begin_task >> cluster >> job >> end_task

    return _create_subdag(
        _internal,
        parent_dag,
        'pyspark-{}'.format(job_name),
        job_name,
        default_args,
        trigger_rule='all_done'
    )

# EXTERNALS END


import pkgutil

import tensorflow as tf

from tensorflow.python import tf2
from tensorflow.python.platform import test

class ModuleTest(test.TestCase):

  def testCanLoadWithPkgutil(self):
    out = pkgutil.find_loader('tensorflow')
    self.assertIsNotNone(out)

  def testDocString(self):
    self.assertIn('TensorFlow', tf.__doc__)
    self.assertNotIn('Wrapper', tf.__doc__)

  def testDict(self):
    # Check that a few modules are in __dict__.
    # pylint: disable=pointless-statement
    tf.nn
    tf.keras
    tf.image
    # pylint: enable=pointless-statement
    self.assertIn('nn', tf.__dict__)
    self.assertIn('keras', tf.__dict__)
    self.assertIn('image', tf.__dict__)

  def testName(self):
    self.assertEqual('tensorflow', tf.__name__)

  def testBuiltInName(self):
    # range is a built-in name in Python. Just checking that
    # tf.range works fine.
    if tf2.enabled():
      self.assertEqual(
          'tf.Tensor([1 2 3 4 5 6 7 8 9], shape=(9,), dtype=int32)',
          str(tf.range(1, 10)))
    else:
      self.assertEqual('Tensor("range:0", shape=(9,), dtype=int32)',
                       str(tf.range(1, 10)))

  def testCompatV2HasCompatV1(self):
    # pylint: disable=pointless-statement
    tf.compat.v2.compat.v1.keras
    # pylint: enable=pointless-statement

  def testSummaryMerged(self):
    # pylint: disable=pointless-statement
    tf.summary.image
    # If we use v2 API, check for create_file_writer,
    # otherwise check for FileWriter.
    if hasattr(tf, '_major_api_version') and tf._major_api_version == 2:
      tf.summary.create_file_writer
    else:
      tf.compat.v1.summary.FileWriter
    # pylint: enable=pointless-statement

  def testPythonModuleIsHidden(self):
    self.assertNotIn('python', dir(tf))

if __name__ == '__main__':
  test.main()


'''
如何设置文件的缓冲
'''

import os
path=os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

#全缓冲,buffering为缓冲区域大小2048
# ：直到buffer区域填满，再写入
f=open(path+"\\file\\5_3.txt",'w',buffering=2048)
f.write('+'*2048)

#行缓冲，buffering=1
# f=open(path+"\\file\\5_3.txt",'w',buffering=1)
# f.write('aaa')
# f.write('\n')

#无缓冲，buffering=0
# f=open(path+"\\file\\5_3.txt",'w',buffering=0)


from django.conf import settings

def mesads_settings(request):
    """Expose settings starting with MESADS_ to templates."""
    return {
        key: getattr(settings, key)
        for key in dir(settings)
        if key.startswith("MESADS_")
    }


#!/usr/bin/env python
# coding: utf-8

"""
# 'CAROM_Supplementary.py' contains the code for producing the majority of the supplementary
# models and figures described in the CAROM-ML paper.
#
# For the main figures and models, refer to 'CAROM_script.py'. Several other
# supplementary analyses are carried out in the 'CAROM_zscoreRegression.py' and
# 'CAROM_algorithmComparison.py' files.
#
# The script is broken into the following sections:
# Part 1: Data import and setup
# Part 2: Training the supplementary ML models (e.g Yeast-only)
# Part 3: Explaining the organism-specific models using the SHAP package
# Part 4: Cell-cycle acetylation predictions using the no-G0 model
# Part 5: Validating the various cell-cycle models
#           -e.g caromMdl_G1 --> G0 data replaced w/ G1, then validated on
#            G0/S/G2
# Part 6: Validating the Phos-only (binary) model
#
#
# For details on using the custom functions in this script, use the help
# function (e.g help(carom.xgb_func)).

@author: Kirk Smith
"""

import pandas as pd
import numpy as np
import os
import matplotlib.pyplot as plt
import seaborn as sns
from functions import carom
import pickle
from collections import Counter
import xlsxwriter

#%%

### Part 1: Data import and setup ###

# primary dataset
df_carom = pd.read_csv("../Supplementary data/caromDataset.csv")
# create dataframes for each organism type
df_ecoli = df_carom.loc[df_carom.Organism==1]
df_yeast = df_carom.loc[df_carom.Organism==2]
df_human = df_carom.loc[df_carom.Organism==3]
# carom dataset with no cell-cycle data:
df_noLee = pd.read_csv("../Supplementary data/caromDataset_NoCellCycle.csv")
# dataset where background gene cross-reference is not applied:
df_noBackground =pd.read_csv("../Supplementary data/caromDataset_NoBackground.csv")
# cell cycle datasets:
df_g0Z = pd.read_excel("../Supplementary data/CellCycle_Scaled.xlsx",
                       sheet_name="CellCycle_G0")
df_g1Z = pd.read_excel("../Supplementary data/CellCycle_Scaled.xlsx",
                       sheet_name="CellCycle_G1")
df_sZ = pd.read_excel("../Supplementary data/CellCycle_Scaled.xlsx",
                       sheet_name="CellCycle_S")
df_g2Z = pd.read_excel("../Supplementary data/CellCycle_Scaled.xlsx",
                       sheet_name="CellCycle_G2")
df_CellCycle = pd.concat([df_g1Z,df_sZ,df_g2Z])

df_g1 = pd.read_csv("C:/Users/kirksmi/Documents/CAROM/final_datasets/LeeG1_Norm.csv")
df_s = pd.read_csv("C:/Users/kirksmi/Documents/CAROM/final_datasets/LeeS_Norm.csv")
df_g2 = pd.read_csv("C:/Users/kirksmi/Documents/CAROM/final_datasets/LeeG2_Norm.csv")

df_helaCholest = pd.read_csv("C:/Users/kirksmi/Documents/CAROM/final_datasets/HelaCholest_Norm.csv")
#%%
# view variables and data types
print(df_carom.dtypes)

# define feature names
feature_names = df_carom.columns[3:16]   # index version
print("Predictor features: ", feature_names)

feats_list = list(feature_names)   # list version
featsPlusOrg = feats_list + ["Organism"]

# define class names
class_names = ["Phos","Unreg","Acetyl"]

# view number of gene-rxn pairs by class
print("Number of regulated gene-rxn pairs:\n", df_carom['Target'].value_counts())
# view number of gene-rxn pairs by organism types
print("Number of each cell type:\n", df_carom['Organism'].value_counts())

#%%
### Feature correlation heatmaps

# create folder for saving figures
path = "../figures/correlation_heatmaps"
try:
    os.mkdir(path)
except OSError:
    print ("Creation of the directory %s failed" % path)
else:
    print ("Successfully created the directory %s " % path)

# plot correlation heatmaps
for df, condition in zip([df_ecoli[feature_names], df_yeast[feature_names],
                          df_human[feature_names], df_carom[featsPlusOrg]],
                         ["ecoli","yeast","human","allOrganisms"]):
    corrmat = df.corr()
    top_corr_features = corrmat.index
    # plot heat map
    plt.figure(figsize=(10,10))
    sns.heatmap(df[top_corr_features].corr(),annot=True,cmap="RdYlGn",
                     vmin=-0.75, vmax=0.75,fmt='.2f')
    plt.savefig("../figures/correlation_heatmaps/{}_CorrHeatmap.png".format(condition),
                     bbox_inches='tight', dpi=600)

#%%

### Part 2: Train supplementary XGBoost models ###

# E. coli
[ecoli_mdl, ecoli_scores]= carom.xgb_func(
   X=df_ecoli[feature_names],
   y=df_ecoli['Target'],
   num_iter=5,
   condition='Ecoli_NoWeights',
   class_names=class_names,
   imbalance="none")
filename1 = '../models/ecoli_XGBmodel.sav'
pickle.dump(ecoli_mdl, open(filename1, 'wb'))

# yeast
[yeast_mdl, yeast_scores]= carom.xgb_func(
   X=df_yeast[feature_names],
   y=df_yeast['Target'],
   num_iter=5,
   condition='Yeast_NoWeights',
   class_names=class_names,
   imbalance="none")
filename1 = '../models/yeast_XGBmodel.sav'
pickle.dump(yeast_mdl, open(filename1, 'wb'))

# human
[human_mdl, human_scores]= carom.xgb_func(
   X=df_human[feature_names],
   y=df_human['Target'],
   num_iter=5,
   condition='Human_NoWeights',
   class_names=class_names,
   imbalance="none")
filename1 = '../models/human_XGBmodel.sav'
pickle.dump(human_mdl, open(filename1, 'wb'))

# organism-type included as feature
[caromOrg_mdl, caromOrg_scores]= carom.xgb_func(
   X=df_carom[featsPlusOrg],
   y=df_carom['Target'],
   num_iter=5,
   condition='caromOrg_NoW',
   class_names=class_names,
   imbalance="none")
filename1 = '../models/caromOrg_XGBmodel.sav'
pickle.dump(caromOrg_mdl, open(filename1, 'wb'))

# no cell-cycle data included
[caromMdl_NoG0, caromScores_NoG0]= carom.xgb_func(
   X=df_noLee[feature_names],
   y=df_noLee['Target'],
   num_iter=5,
   condition='NoG0',
   class_names=class_names,
   imbalance="none")
filename1 = '../models/carom_XGBmodel_NoG0.sav'
pickle.dump(caromMdl_NoG0, open(filename1, 'wb'))

# no removal of non-annotated background genes
[caromMdl_background, caromScores_background]= carom.xgb_func(
   X=df_noBackground[feature_names],
   y=df_noBackground['Target'],
   num_iter=5,
   condition='NoBackground',
   class_names=class_names,
   imbalance="none")
filename1 = '../models/carom_XGBmodel_NoBackground.sav'
pickle.dump(caromMdl_background, open(filename1, 'wb'))

## train models for the various cell-cycle phase datasets
# G1
df_caromG1 = pd.concat([df_noLee, df_g1Z])

[caromMdl_G1, caromScores_G1]= carom.xgb_func(
   X=df_caromG1[feature_names],
   y=df_caromG1['Target'],
   num_iter=5,
   condition='G1',
   class_names=class_names,
   imbalance="none")
pickle.dump(caromMdl_G1, open('../models/carom_XGBmodel_G1.sav', 'wb'))

# S
df_caromS = pd.concat([df_noLee, df_sZ])

[caromMdl_S, caromScores_S]= carom.xgb_func(
   X=df_caromS[feature_names],
   y=df_caromS['Target'],
   num_iter=5,
   condition='S',
   class_names=class_names,
   imbalance="none")
pickle.dump(caromMdl_S, open('../models/carom_XGBmodel_S.sav', 'wb'))

# G2
df_caromG2 = pd.concat([df_noLee, df_g2Z])

[caromMdl_G2, caromScores_G2]= carom.xgb_func(
   X=df_caromG2[feature_names],
   y=df_caromG2['Target'],
   num_iter=5,
   condition='G2',
   class_names=class_names,
   imbalance="none")
pickle.dump(caromMdl_G2, open('../models/carom_XGBmodel_G2.sav', 'wb'))
#%%

# train model on G0 only

[caromMdl_onlyG0, caromScores_onlyG0]= carom.xgb_func(
   X=df_g0Z[feature_names],
   y=df_g0Z['Target'],
   num_iter=5,
   condition='onlyG0',
   class_names=["Phos","Unreg"],
   imbalance="none")

[caromMdl_onlyG1, caromScores_onlyG1]= carom.xgb_func(
   X=df_g1Z[feature_names],
   y=df_g1Z['Target'],
   num_iter=5,
   condition='onlyG1',
   class_names=["Phos","Unreg"],
   imbalance="none")

[caromMdl_onlyS, caromScores_onlyS]= carom.xgb_func(
   X=df_sZ[feature_names],
   y=df_sZ['Target'],
   num_iter=5,
   condition='onlyS',
   class_names=["Phos","Unreg"],
   imbalance="none")

[caromMdl_onlyG2, caromScores_onlyG2]= carom.xgb_func(
   X=df_g2Z[feature_names],
   y=df_g2Z['Target'],
   num_iter=5,
   condition='onlyG2',
   class_names=["Phos","Unreg"],
   imbalance="none")
#%%

# single phase models, no tuning

classifier = xgboost.XGBClassifier(objective='binary:logistic',
                    n_estimators=150,
                    use_label_encoder=False,
                    eval_metric='logloss',
                    random_state=123)

classifier2 = xgboost.XGBClassifier(objective='multi:softprob',
                    n_estimators=150,
                    use_label_encoder=False,
                    eval_metric='mlogloss',
                    num_class=3,
                    random_state=123)

classifier_onlyG0 = classifier.fit(X=df_g0Z[feature_names],
   y=pd.Series(le.transform(df_g0Z['Target'])))

classifier_onlyG1 = classifier.fit(X=df_g1Z[feature_names],
   y=pd.Series(le.transform(df_g1Z['Target'])))

classifier_onlyS = classifier.fit(X=df_sZ[feature_names],
   y=pd.Series(le.transform(df_sZ['Target'])))

classifier_onlyG2 = classifier.fit(X=df_g2Z[feature_names],
   y=pd.Series(le.transform(df_g2Z['Target'])))

classifier_onlyG1_fc = classifier.fit(X=df_g1[feature_names],
   y=pd.Series(le.transform(df_g1['Target'])))

classifer_caromG1 = classifier2.fit(df_caromG1[feature_names],
   y=pd.Series(le.transform(df_caromG1['Target'])))

#%%

# test one phase models

df = pd.concat([df_g1Z, df_sZ, df_g2Z])
[scores_g1, G1_PredictPhos, G1_ypred] = carom.mdl_predict(
                          mdl=caromMdl_onlyG0,
                          X=df[feature_names],
                          y=df["Target"],
                          condition="onlyG0",
                          class_names=["Phos","Unreg"],
                          gene_reactions=df[['genes','reaction','Phase']],
                          confusion_mat=True, bar=False)

df = pd.concat([df_g0Z, df_sZ, df_g2Z])
[scores_g1, G1_PredictPhos, G1_ypred] = carom.mdl_predict(
                          mdl=caromMdl_onlyG1,
                          X=df[feature_names],
                          y=df["Target"],
                          condition="onlyG1",
                          class_names=["Phos","Unreg"],
                          gene_reactions=df[['genes','reaction','Phase']],
                          confusion_mat=True, bar=False)

df = pd.concat([df_g0Z, df_g1Z, df_g2Z])
[scores_g1, G1_PredictPhos, G1_ypred] = carom.mdl_predict(
                          mdl=caromMdl_onlyS,
                          X=df[feature_names],
                          y=df["Target"],
                          condition="onlyS",
                          class_names=["Phos","Unreg"],
                          gene_reactions=df[['genes','reaction','Phase']],
                          confusion_mat=True, bar=False)

df = pd.concat([df_g0Z, df_g1Z, df_sZ])
[scores_g1, G1_PredictPhos, G1_ypred] = carom.mdl_predict(
                          mdl=caromMdl_onlyG2,
                          X=df[feature_names],
                          y=df["Target"],
                          condition="onlyG2",
                          class_names=["Phos","Unreg"],
                          gene_reactions=df[['genes','reaction','Phase']],
                          confusion_mat=True, bar=False)

#%%

## train G1 fold change models
df_caromG1 = pd.concat([df_noLee, df_g1])

[caromMdl_G1, caromScores_G1]= carom.xgb_func(
   X=df_caromG1[feature_names],
   y=df_caromG1['Target'],
   num_iter=5,
   condition='G1_FC',
   class_names=class_names,
   imbalance="none")

[caromMdl_onlyG1, caromScores_onlyG1]= carom.xgb_func(
   X=df_g1[feature_names],
   y=df_g1['Target'],
   num_iter=5,
   condition='onlyG1_FC',
   class_names=["Unreg","Phos"],
   imbalance="none")

#%%
df_sg2 = pd.concat([df_s, df_g2])

carom.mdl_predict(mdl=classifer_caromG1,
                X=df_sg2[feature_names],
                y=df_sg2["Target"],
                condition="testG1Mdl",
                gene_reactions=df_sg2[['genes','reaction','Phase']],
                confusion_mat=True, bar=True, swarm=False,
                pairwise=False, boxplot=False,
                explainer=None)

#%%
## train binary models (Unreg vs Acetyl, Unreg vs Phos)

# Acetyl
df_acetyl = pd.concat([df_carom[feature_names],df_carom.Acetyl],axis=1).dropna()

[AcetylMdl, AcetylScores]= carom.xgb_func(
   X=df_acetyl[feature_names],
   y=df_acetyl['Acetyl'],
   num_iter=5,
   condition='Acetyl',
   class_names=["Unreg","Acetyl"],
   imbalance="none")
pickle.dump(AcetylMdl, open('../models/acetyl_XGBmodel.sav', 'wb'))

#Phos
df_phos = pd.concat([df_carom[feature_names], df_carom.Phos],axis=1).dropna()

[PhosMdl_shallow, PhosScores_shallow]= carom.xgb_func(
   X=df_phos[feature_names],
   y=df_phos['Phos'],
   num_iter=5,
   condition='Phos',
   class_names=["Unreg","Phos"],
   imbalance="none",
   depth='shallow')
pickle.dump(PhosMdl_shallow, open('../models/phosShallow_XGBmodel.sav', 'wb'))

 #%%
## load the models from disk
ecoli_mdl = pickle.load(open('../models/ecoli_XGBmodel.sav', 'rb'))
yeast_mdl = pickle.load(open('../models/yeast_XGBmodel.sav', 'rb'))
human_mdl = pickle.load(open('../models/human_XGBmodel.sav', 'rb'))
caromOrg_mdl = pickle.load(open('../models/caromOrg_XGBmodel.sav', 'rb'))
caromMdl_NoG0 = pickle.load(open('../models/carom_XGBmodel_NoG0.sav', 'rb'))
caromMdl = pickle.load(open('../models/carom_XGBmodel.sav', 'rb'))
phosMdl = pickle.load(open('../models/phosShallow_XGBmodel.sav', 'rb'))

#%%
### Part 3: SHAP analysis for various organism-type models

[ecoli_explainer, ecoli_shapValues] = carom.shap_func(
    xgbModel = ecoli_mdl,
    X = df_ecoli[feature_names],
    condition = "Ecoli",
    class_names = ["Phosphorylation","Unregulated","Acetylation"])

[yeast_explainer, yeast_shapValues] = carom.shap_func(
    xgbModel = yeast_mdl,
    X = df_yeast[feature_names],
    condition = "Yeast",
    class_names = ["Phosphorylation","Unregulated","Acetylation"])

[human_explainer, human_shapValues] = carom.shap_func(
    xgbModel = human_mdl,
    X = df_human[feature_names],
    condition = "Human",
    class_names = ["Phosphorylation","Unregulated","Acetylation"])

[caromOrg_explainer, caromOrg_shapValues] = carom.shap_func(
    xgbModel = caromOrg_mdl,
    X = df_carom[featsPlusOrg],
    condition = "CAROMwithOrg",
    class_names = ["Phosphorylation","Unregulated","Acetylation"])

#%%

### Part 4: Acetylation predictions for cell-cycle data with no-G0 model ###

[G0_acetylGenes, G0_phosGenes, G0_ypred] = carom.mdl_predict(
                                                    mdl=caromMdl_NoG0,
                                                    X=df_g0Z[feature_names],
                                                    condition="PredictG0_NoG0Mdl",
                                                    class_names=class_names,
                                                    gene_reactions=df_g0Z[['genes','reaction']])
G0_acetylGenes["Phase"] = np.tile("G0",(len(G0_acetylGenes),1))

[G1_acetylGenes, G1_phosGenes, G1_ypred] = carom.mdl_predict(
                                                    mdl=caromMdl_NoG0,
                                                    X=df_g1Z[feature_names],
                                                    condition="PredictG1_NoG0Mdl",
                                                    class_names=class_names,
                                                    gene_reactions=df_g1Z[['genes','reaction']])
G1_acetylGenes["Phase"] = np.tile("G1",(len(G1_acetylGenes),1))

[S_acetylGenes, S_phosGenes, S_ypred] = carom.mdl_predict(
                                                    mdl=caromMdl_NoG0,
                                                    X=df_sZ[feature_names],
                                                    condition="PredictS_NoG0Mdl",
                                                    class_names=class_names,
                                                    gene_reactions=df_sZ[['genes','reaction']])
S_acetylGenes["Phase"] = np.tile("S",(len(S_acetylGenes),1))

[G2_acetylGenes, G2_phosGenes, G2_ypred] = carom.mdl_predict(
                                                    mdl=caromMdl_NoG0,
                                                    X=df_g2Z[feature_names],
                                                    condition="PredictG2_NoG0Mdl",
                                                    class_names=class_names,
                                                    gene_reactions=df_g2Z[['genes','reaction']])
G2_acetylGenes["Phase"] = np.tile("G2",(len(G2_acetylGenes),1))
#%%

# combine predictions and write to file
LeeAcetylPred_AllPhases = pd.concat([G0_acetylGenes,G1_acetylGenes,
                                            S_acetylGenes, G2_acetylGenes])
LeePhosPred_AllPhases = pd.concat([G0_phosGenes,G1_phosGenes,
                                            S_phosGenes, G2_phosGenes])
LeeAcetylPred_AllPhases.to_csv('../results/CellCycleAcetylPred_AllPhases.csv',
                                index=False)

# get unique Acetyl genes and rxns
uniqueLeeAcetylGenes = LeeAcetylPred_AllPhases.genes.unique()
uniqueLeeAcetylRxns = LeeAcetylPred_AllPhases.reaction.unique()

uniqueLeePhosGenes = LeePhosPred_AllPhases.genes.unique()
#%%

array = [uniqueLeeAcetylGenes,
         uniqueLeeAcetylRxns]

workbook = xlsxwriter.Workbook('../results/CellCycleAcetylPred_Unique.xlsx')
worksheet = workbook.add_worksheet()

array = [uniqueLeeAcetylGenes,
         uniqueLeeAcetylRxns]
row = 0
for col, data in enumerate(array):
    worksheet.write_column(row, col, data)
workbook.close()

# plot number of acetyl predictions per phase
acetylCounts = list(Counter(LeeAcetylPred_AllPhases.Phase).values())
fig,ax = plt.subplots()
sns.barplot(x=["G0","G1","S","G2"],y=acetylCounts)
plt.ylabel("# of Acetyl. Predictions",fontsize=16)
ax.tick_params(axis="both", labelsize=16)

#%%

### Part 5: Validating the various cell-cycle models ###

# G1 model
df = pd.concat([df_g0Z, df_sZ, df_g2Z])
[scores_g1, G1_PredictAcetyl, G1_PredictPhos, G1_ypred] = carom.mdl_predict(
                          mdl=caromMdl_G1,
                          X=df.loc[:,feature_names],
                          y=df.loc[:,"Target"],
                          condition="G1_mdl", class_names=class_names,
                          gene_reactions=df[['genes','reaction','Phase']],
                          confusion_mat=True, bar=True, swarm=False,
                          pairwise=False, boxplot=False,
                          explainer=None)

# S model
df = pd.concat([df_g0Z, df_g1Z, df_g2Z])
[scores_s, S_PredictAcetyl, S_PredictPhos, S_ypred] = carom.mdl_predict(
                          mdl=caromMdl_S,
                          X=df.loc[:,feature_names],
                          y=df.loc[:,"Target"],
                          condition="S_mdl", class_names=class_names,
                          gene_reactions=df[['genes','reaction','Phase']],
                          confusion_mat=True, bar=True, swarm=False,
                          pairwise=False, boxplot=False,
                          explainer=None)

# G2 model
df = pd.concat([df_g0Z, df_g1Z, df_sZ])
[scores_g2, G2_PredictAcetyl, G2_PredictPhos, G2_ypred] = carom.mdl_predict(
                          mdl=caromMdl_G2,
                          X=df.loc[:,feature_names],
                          y=df.loc[:,"Target"],
                          condition="G2_mdl", class_names=class_names,
                          gene_reactions=df[['genes','reaction','Phase']],
                          confusion_mat=True, bar=True, swarm=False,
                          pairwise=False, boxplot=False,
                          explainer=None)
#%%

# Part 6: Validating the binary Phos model ###

X = df_CellCycle.loc[:,feature_names]
y = df_CellCycle.loc[:,"Phos"]
print(y.value_counts())

[phosMdl_scores, phosMdl_PredictPhos, phosMdl_Ypred] = carom.mdl_predict(
                          mdl=PhosMdl_shallow, X=X, y=y,
                          condition="PhosMdl", class_names=["Unreg","Phos"],
                          gene_reactions=df_CellCycle[['genes','reaction','Phase']],
                          confusion_mat=True, bar=True, swarm=False,
                          pairwise=False, boxplot=False,
                          explainer=None)


import discord
from discord.ext import commands
from itertools import cycle
import json
global to_delete
to_delete = []

class help_channel():

    all_channels = []

    def __init__(self, channel_id, guild_id):
        self.channel_id = channel_id
        self.Is_occupated = False
        self.all_channels.append(self)
        self.user = []
        self.guild_id = guild_id

with open(r"C:\Users\ariso\Desktop\Snake projects\aribot\cogs\data.json", "r") as f:
    data = json.load(f)
for x in data["serwers"]:
    for y in x:
        for helps in x[y]["help_channels"]:
            help_channel(helps, y)

def get_channel_for_guild(guild_id, wanted_channel):
    for x in data["serwers"]:
        for y in x:
            if int(y) == guild_id:
                return int(x[y][wanted_channel])

class coggs(commands.Cog):
    def __init__(self, client):
        self.client = client

    @commands.command(brief="brief description of your problem")
    @commands.cooldown(1, 30, commands.BucketType.user)
    async def get_help(self, ctx, *, info):
        await ctx.channel.purge(limit=1)
        await ctx.channel.send(f"{ctx.author} your request has been send")
        guild_id = ctx.guild.id
        for x in help_channel.all_channels:

            if x.Is_occupated == False and int(x.guild_id) == ctx.guild.id:
                print("Check")
                x.Is_occupated = True
                print(x.Is_occupated)
                channel = self.client.get_channel(int(x.channel_id))
                channel2 = self.client.get_channel(
                    get_channel_for_guild(guild_id, "helpers_channel"))
                await channel.set_permissions(ctx.author, send_messages=True)
                await channel.send(f"hi, {ctx.author} please describe your problem here\nSoon one of the helpers will come here to solve your problem  ")
                x.user.append(ctx.author)
                emb = discord.Embed(title=f"{ctx.author} is looking for help with:",
                                    description=f"{info}\n\n*Reat with ✅ to take request*\n*react with ❌ to close this request* ", color=0xFF0000)
                await channel2.send(embed=emb)
                break

    @commands.command(brief="mods only don't try it")
    @commands.has_permissions(manage_messages=True)
    async def close(self, ctx):
        for x in help_channel.all_channels:
            if self.client.get_channel(int(x.channel_id)) == ctx.channel:
                await ctx.channel.purge(limit=1000)
                await ctx.channel.set_permissions(x.user[0], send_messages=False)
                break

    @commands.Cog.listener()
    async def on_raw_reaction_add(self, payload):

        channel = self.client.get_channel(payload.channel_id)

        if channel == self.client.get_channel(get_channel_for_guild(payload.guild_id, "helpers_channel")):
            print("check0")
            msg = await channel.fetch_message(payload.message_id)
            msge = msg.embeds
            title = msge[0].title

            msg_splited = title.split(" ")
            for x in help_channel.all_channels:

                print(type(payload.guild_id))
                print(type(int(x.guild_id)))

                if len(x.user) != 0 and int(x.guild_id) == payload.guild_id:
                    print("check 1")
                    if str(x.user[0]) == msg_splited[0]:
                        print("check 2")
                        channel2 = self.client.get_channel(int(x.channel_id))
                        break

        if str(payload.emoji.name) == "❌":

            if channel == self.client.get_channel(get_channel_for_guild(payload.guild_id, "helpers_channel")):
                print(len(to_delete))
                for u in to_delete:
                    await channel2.set_permissions(u, send_messages=False)

                msg = await channel.fetch_message(payload.message_id)
                await msg.delete()
        elif str(payload.emoji.name) == "✅":
            if channel == self.client.get_channel(get_channel_for_guild(payload.guild_id, "helpers_channel")):
                user = await self.client.fetch_user(payload.user_id)
                print(user)
                print(type(user))
                to_delete.append(user)

                await channel2.set_permissions(user, send_messages=True)
                await channel2.send(f"It's your lucky day {user} is here to help you")

def setup(client):
    client.add_cog(coggs(client))


import RPi.GPIO as GPIO
import time
import asyncio

class LED:

    def __init__(self):
            # GPIO.setmode(GPIO.BOARD)
            pass

    async def swichRfidOnGreen(self):
        """Switches on green Green LED on and off for the RFID subsystem
        """
        GPIO.setmode(GPIO.BCM)
        GPIO.setwarnings(False)

        GPIO.setup(24,GPIO.OUT)
        GPIO.output(24,GPIO.HIGH)

        # GPIO.setup(18,GPIO.OUT)
        # GPIO.output(18,GPIO.HIGH)
        # time.sleep(1)
        time.sleep(0.7)
        # await asyncio.sleep(0.7)

        # GPIO.output(18,GPIO.LOW)
        GPIO.output(24,GPIO.LOW)

    async def swichRfidOnRed(self):
        """Switches on green RED LED on and off for the RFID subsystem
        """
        GPIO.setmode(GPIO.BCM)
        GPIO.setwarnings(False)

        # GPIO.setup(16,GPIO.OUT)
        # GPIO.output(16,GPIO.HIGH)
        GPIO.setup(23,GPIO.OUT)
        GPIO.output(23,GPIO.HIGH)

        # time.sleep(1)
        time.sleep(0.7)
        # await asyncio.sleep(0.7)
        # GPIO.output(16,GPIO.LOW)
        GPIO.output(23,GPIO.LOW)

    async def swichOnTempRed(self):
        """Switches on green RED LED on and off for the Temperature subsystem
        """
        GPIO.setmode(GPIO.BCM)
        GPIO.setwarnings(False)

        # GPIO.setup(11,GPIO.OUT)
        # GPIO.output(11,GPIO.HIGH)
        # time.sleep(1)

        # GPIO.output(11,GPIO.LOW)

        GPIO.setup(17,GPIO.OUT)
        GPIO.output(17,GPIO.HIGH)
        # time.sleep(1)
        time.sleep(0.7)
        # await asyncio.sleep(0.7)

        GPIO.output(17,GPIO.LOW)

    async def swichOnTempGreen(self):
        """Switches on green GREEEN LED on and off for the TEMP subsystem
        """
        GPIO.setmode(GPIO.BCM)
        GPIO.setwarnings(False)

        # GPIO.setup(13,GPIO.OUT)
        # GPIO.output(13,GPIO.HIGH)
        # time.sleep(1)

        # GPIO.output(13,GPIO.LOW)
        GPIO.setup(27,GPIO.OUT)
        GPIO.output(27,GPIO.HIGH)
        # time.sleep(1)
        time.sleep(0.7)
        # await asyncio.sleep(0.7)

        GPIO.output(27,GPIO.LOW)

# LEDObj = LED()
# LEDObj.swichRfidOnGreen()
# LEDObj.swichRfidOnRed()
# LEDObj.swichOnTempRed()
# LEDObj.swichOnTempGreen()
# GPIO.cleanup()
# GPIO.setmode(GPIO.BOARD)
# GPIO.setwarnings(False)
# GPIO.setup(23,GPIO.OUT)
# GPIO.setup(24,GPIO.OUT)
# print("LED on")
# GPIO.output(23,GPIO.HIGH)
# GPIO.output(24,GPIO.HIGH)
# time.sleep(1)
# print("LED off")
# GPIO.output(23,GPIO.LOW)
# GPIO.output(24,GPIO.LOW)


# -*- coding: utf-8 -*-

import xlrd

# excelFile = xlrd.open_workbook(r'bike.xlsx')

def read03Excel(path):
    workbook = xlrd.open_workbook(path)
    sheets = workbook.sheet_names()
    worksheet = workbook.sheet_by_name(sheets[1])
    for i in range(0, worksheet.nrows):
        row = worksheet.row(i)
        for j in range(0, len(row)):
            print(str(worksheet.cell_value(i, j)) + "|", end=' ')

        print()

read03Excel("bike.xlsx")

# 原因：xlrd升级后不支持xlsx文件读取了，通过降低版本
# pip3 uninstall xlrd
# pip3 install xlrd==1.2.0


class Node:
    def __init__(self,data):
        self.data = data
        self.next = None

class linkedLists:
    def __init__(self):
        self.head = None

    def traverseNode(self):
        currentValue = self.head

        while currentValue != None:
            if currentValue.data == 'u':
                newNode = Node('ORA')
                newNode.next = currentValue.next
                currentValue.next = newNode
            print(currentValue.data)
            currentValue = currentValue.next

    def deleteNode(self):
        currentValue = self.head
        count = 0

        print("============================")
        while currentValue != None:
            count = count + 1

            if count == 1:
                print(currentValue.data)
                currentValue = currentValue.next.next
            else:
                print(currentValue.data)
                currentValue = currentValue.next

if __name__ == '__main__':

    lList = linkedLists()

    lList.head = Node('s')
    secondNode = Node('u')
    thirdNode = Node('s')

    lList.head.next = secondNode
    secondNode.next = thirdNode

    lList.traverseNode()

    lList.deleteNode()


"""empty message

Revision ID: 248a30ecba33
Revises: 9d64fd56c7e4
Create Date: 2023-01-12 12:34:30.398192

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '248a30ecba33'
down_revision = '9d64fd56c7e4'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('podimage',
    sa.Column('pod_id', sa.Integer(), nullable=False),
    sa.Column('image_id', sa.Integer(), nullable=False),
    sa.ForeignKeyConstraint(['image_id'], ['image.id'], ),
    sa.ForeignKeyConstraint(['pod_id'], ['pod.id'], ),
    sa.PrimaryKeyConstraint('pod_id', 'image_id')
    )
    op.drop_index('ix_pod_image_id', table_name='pod')
    op.drop_constraint('pod_image_id_fkey', 'pod', type_='foreignkey')
    op.drop_column('pod', 'image_id')
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('pod', sa.Column('image_id', sa.INTEGER(), autoincrement=False, nullable=True))
    op.create_foreign_key('pod_image_id_fkey', 'pod', 'image', ['image_id'], ['id'])
    op.create_index('ix_pod_image_id', 'pod', ['image_id'], unique=False)
    op.drop_table('podimage')
    # ### end Alembic commands ###


#!/usr/bin/python
import os
import platform
import sys
from time import sleep

# extra libs
sys.path.insert(0, os.getcwd() + '/')  # If the user runs this on previous path
sys.path.insert(0, os.getcwd() + '/libs')  # If the user runs this on previous path
sys.path.insert(0, os.getcwd() + '/../libs')
sys.path.insert(0, os.getcwd() + '/../')
import colorama
from colorama import Fore
from drivers.NRF52_dongle import NRF52Dongle
from scapy.layers.bluetooth4LE import *
from scapy.layers.bluetooth import *
from timeout_lib import start_timeout, disable_timeout, update_timeout

# Default master address
master_address = '5d:36:ac:90:0b:20'
access_address = 0x9a328370
# Normal pairing request for secure pairing (uncomment the following to choose pairing request method)
# pairing_iocap = 0x01  # DisplayYesNo
# pairing_iocap = 0x03  # NoInputNoOutput
pairing_iocap = 0x04  # KeyboardDisplay
# paring_auth_request = 0x00  # No bounding
# paring_auth_request = 0x01  # Bounding
# paring_auth_request = 0x08 | + 0x01  # Le Secure Connection + bounding
# paring_auth_request = 0x04 | 0x01  # MITM + bounding
paring_auth_request = 0x08 | 0x04 | 0x01  # Le Secure Connection + MITM + bounding

# Internal vars
SCAN_TIMEOUT = 2
none_count = 0
end_connection = False
connecting = False
current_key_size = 6
accepted_keys = []

# Autoreset colors
colorama.init(autoreset=True)

# Get serial port from command line
if len(sys.argv) >= 2:
    serial_port = sys.argv[1]
elif platform.system() == 'Linux':
    serial_port = '/dev/ttyACM0'
elif platform.system() == 'Windows':
    serial_port = 'COM1'
else:
    print(Fore.RED + 'Platform not identified')
    sys.exit(0)

print(Fore.YELLOW + 'Serial port: ' + serial_port)

# Get advertiser_address from command line (peripheral addr)
if len(sys.argv) >= 3:
    advertiser_address = sys.argv[2].upper()
else:
    advertiser_address = 'A4:C1:38:D8:AD:B8'

print(Fore.YELLOW + 'Advertiser Address: ' + advertiser_address.lower())

def crash_timeout():
    print(Fore.RED + "No advertisement from " + advertiser_address.lower() +
          ' received\nThe device may have crashed!!!')
    disable_timeout('scan_timeout')

def scan_timeout():
    global connecting
    connecting = False
    scan_req = BTLE() / BTLE_ADV() / BTLE_SCAN_REQ(
        ScanA=master_address,
        AdvA=advertiser_address)
    driver.send(scan_req)
    start_timeout('scan_timeout', SCAN_TIMEOUT, scan_timeout)

def set_security_settings(pkt):
    global paring_auth_request
    # Change security parameters according to slave security request
    # paring_auth_request = pkt[SM_Security_Request].authentication
    print(Fore.YELLOW + 'Slave requested authentication of ' + hex(pkt[SM_Security_Request].authentication))
    print(Fore.YELLOW + 'We are using authentication of ' + hex(paring_auth_request))

def send_termination_indication():
    pkt = BTLE(access_addr=access_address) / BTLE_DATA() / CtrlPDU() / LL_TERMINATE_IND()
    driver.send(pkt)

def check_range(array, left, right):
    return len([x for x in array if left <= x <= right]) > 0

def send_pairing_request():
    pairing_req = BTLE(access_addr=access_address) / BTLE_DATA() / L2CAP_Hdr() / SM_Hdr() / SM_Pairing_Request(
        iocap=pairing_iocap,
        oob=0,
        authentication=paring_auth_request,
        max_key_size=current_key_size,
        initiator_key_distribution=0x07,
        responder_key_distribution=0x07)
    driver.send(pairing_req)

# Open serial port of NRF52 Dongle
driver = NRF52Dongle(serial_port, '115200', logs_pcap=True, pcap_filename='knob_ble_tester.pcap')
# Send scan request
scan_req = BTLE() / BTLE_ADV() / BTLE_SCAN_REQ(
    ScanA=master_address,
    AdvA=advertiser_address)
driver.send(scan_req)

start_timeout('scan_timeout', SCAN_TIMEOUT, scan_timeout)

print(Fore.YELLOW + 'Waiting advertisements from ' + advertiser_address)
while True:
    pkt = None
    # Receive packet from the NRF52 Dongle
    data = driver.raw_receive()
    if data:
        # Decode Bluetooth Low Energy Data
        pkt = BTLE(data)  # Receive plain text Link Layer
        # if packet is incorrectly decoded, you may not be using the dongle
        if pkt is None:
            none_count += 1
            if none_count >= 4:
                print(Fore.RED + 'NRF52 Dongle not detected')
                sys.exit(0)
            continue
        elif BTLE_DATA in pkt and BTLE_EMPTY_PDU not in pkt:
            update_timeout('scan_timeout')
            # Print slave data channel PDUs summary
            print(Fore.MAGENTA + "RX <--- " + pkt.summary()[7:])

        # --------------- Process Link Layer Packets here ------------------------------------
        # Check if packet from advertised is received
        if BTLE_SCAN_RSP in pkt and pkt.AdvA == advertiser_address.lower() and connecting == False:
            connecting = True
            update_timeout('scan_timeout')
            disable_timeout('crash_timeout')
            conn_rx_packet_counter = 0
            conn_tx_packet_counter = 0
            encryption_enabled = False
            print(Fore.GREEN + advertiser_address.upper() + ': ' + pkt.summary()[7:] + ' Detected')
            # Send connection request to advertiser
            conn_request = BTLE() / BTLE_ADV(RxAdd=pkt.TxAdd, TxAdd=0) / BTLE_CONNECT_REQ(
                InitA=master_address,
                AdvA=advertiser_address,
                AA=access_address,  # Access address (any)
                crc_init=0x179a9c,  # CRC init (any)
                win_size=2,  # 2.5 of windows size (anchor connection window size)
                win_offset=1,  # 1.25ms windows offset (anchor connection point)
                interval=16,  # 20ms connection interval
                latency=0,  # Slave latency (any)
                timeout=50,  # Supervision timeout, 500ms (any)
                chM=0x1FFFFFFFFF,  # Any
                hop=5,  # Hop increment (any)
                SCA=0,  # Clock tolerance
            )
            # Yes, we're sending raw link layer messages in Python. Don't tell Bluetooth SIG as this is forbidden by
            # them!!!
            driver.send(conn_request)

        elif BTLE_DATA in pkt and connecting == True:
            connecting = False
            print(Fore.GREEN + 'Slave Connected (Link Layer data channel established)')
            if SM_Security_Request in pkt:
                set_security_settings(pkt)
            if LL_VERSION_IND in pkt:
                pkt = BTLE(access_addr=access_address) / BTLE_DATA() / CtrlPDU() / LL_VERSION_IND(version='4.2')
                driver.send(pkt)
            # Send Feature request
            pkt = BTLE(access_addr=access_address) / BTLE_DATA() / CtrlPDU() / LL_FEATURE_REQ(
                feature_set='le_encryption+le_data_len_ext')
            driver.send(pkt)

        elif SM_Security_Request in pkt:
            set_security_settings(pkt)

        elif LL_FEATURE_RSP in pkt:
            pkt = BTLE(access_addr=access_address) / BTLE_DATA() / CtrlPDU() / LL_LENGTH_REQ(
                max_tx_bytes=247 + 4, max_rx_bytes=247 + 4)
            driver.send(pkt)

        elif LL_LENGTH_RSP in pkt or LL_UNKNOWN_RSP in pkt:
            pkt = BTLE(access_addr=access_address) / \
                  BTLE_DATA() / L2CAP_Hdr() / ATT_Hdr() / ATT_Exchange_MTU_Request(mtu=247)
            driver.send(pkt)

        elif ATT_Exchange_MTU_Response in pkt:
            # Send version indication request
            pkt = BTLE(access_addr=access_address) / BTLE_DATA() / CtrlPDU() / LL_VERSION_IND(version='4.2')
            driver.send(pkt)

        elif LL_VERSION_IND in pkt:
            send_pairing_request()

        elif LL_LENGTH_REQ in pkt:
            pkt = BTLE(access_addr=access_address) / BTLE_DATA() / CtrlPDU() / LL_LENGTH_RSP(
                max_tx_bytes=247 + 4, max_rx_bytes=247 + 4)
            driver.send(pkt)
            send_pairing_request()

        # THE ATTACK STARTS HERE !!!!
        elif SM_Pairing_Response in pkt:
            # Pairing request accepted
            # ediv and rand are 0 on first time pairing
            end_connection = True
            print(Fore.YELLOW + 'Slave accepted key size of ' + str(current_key_size))
            accepted_keys.append(current_key_size)
            current_key_size += 1
            send_termination_indication()

        elif LL_REJECT_IND in pkt or SM_Failed in pkt:
            print(Fore.GREEN + 'Slave rejected key size of ' + str(current_key_size))
            current_key_size += 1
            end_connection = True
            send_termination_indication()

        if end_connection == True:
            end_connection = False
            scan_req = BTLE() / BTLE_ADV() / BTLE_SCAN_REQ(
                ScanA=master_address,
                AdvA=advertiser_address)
            print(Fore.YELLOW + 'Connection reset')

            print(Fore.YELLOW + 'Waiting advertisements from ' + advertiser_address)
            driver.send(scan_req)
            start_timeout('crash_timeout', 7, crash_timeout)

        if current_key_size > 17:
            print(Fore.MAGENTA + 'Key sizes accepted by peripheral: ' + str(accepted_keys))

            if check_range(accepted_keys, 0, 6):
                print(Fore.RED + 'Peripheral accepts key size lower than 7!!!')

            elif check_range(accepted_keys, 7, 15):
                print(Fore.RED + 'Peripheral allows key entropy reduction. The key size range is '
                                 '[' + str(min(accepted_keys)) + ',' + str(max(accepted_keys)) + ']')
            elif check_range(accepted_keys, 16, 17):
                print(Fore.RED + 'Peripheral accepts key size greater than 16. Non-compliance!!!')
            elif check_range(accepted_keys, 16, 16) and not check_range(accepted_keys, 7, 15):
                print(Fore.GREEN + 'Peripheral only accepts key size of 16. We are good to go!!!')
            else:
                print(Fore.RED + 'Something went wrong during testing, check if the peripheral accepts pairing')

            print(Fore.YELLOW + 'Test finished')
            exit(0)

sleep(0.01)


# This script computes the halo bispectrum in real- and redshift-space. It takes as input
# the first and last number of the wanted realizations, the cosmology and the snapnum
# In redshift-space it computes the bispectrum along the 3 different axes.
import argparse
from mpi4py import MPI
import numpy as np
import sys,os
import readgadget,readfof
import redshift_space_library as RSL
import Pk_library as PKL
import MAS_library as MASL

###### MPI DEFINITIONS ######
comm   = MPI.COMM_WORLD
nprocs = comm.Get_size()
myrank = comm.Get_rank()

# read the first and last realization to identify voids
parser = argparse.ArgumentParser(description="This script computes the bispectrum")
parser.add_argument("first",   help="first realization number", type=int)
parser.add_argument("last",    help="last  realization number", type=int)
parser.add_argument("cosmo",   help="folder with the realizations")
parser.add_argument("snapnum", help="snapshot number",          type=int)
args = parser.parse_args()
first, last, cosmo, snapnum = args.first, args.last, args.cosmo, args.snapnum

# This routine computes and saves the bispectrum
def find_Pk(folder, snapdir, snapnum, grid, MAS, do_RSD, axis, threads,
            fixed_Mmin, Mmin, Nhalos, fpk, save_multipoles):

    if os.path.exists(fpk):  return 0

    # read header
    head     = readgadget.header(snapdir)
    BoxSize  = head.boxsize/1e3  #Mpc/h
    Omega_m  = head.omega_m
    Omega_l  = head.omega_l
    redshift = head.redshift
    Hubble   = 100.0*np.sqrt(Omega_m*(1.0+redshift)**3+Omega_l)#km/s/(Mpc/h)
    h        = head.hubble

    # read halo catalogue
    FoF   = readfof.FoF_catalog(folder, snapnum, long_ids=False,
                                swap=False, SFR=False, read_IDs=False)
    pos_h = FoF.GroupPos/1e3            #Mpc/h
    mass  = FoF.GroupMass*1e10          #Msun/h
    vel_h = FoF.GroupVel*(1.0+redshift) #km/s
    if fixed_Mmin:
        indexes = np.where(mass>Mmin)[0]
        pos_h = pos_h[indexes];  vel_h = vel_h[indexes];  del indexes
    else:
        indexes = np.argsort(mass)[-Nhalos:] #take the Nhalos most massive halos
        pos_h = pos_h[indexes];  vel_h = vel_h[indexes];  del indexes

    # move halos to redshift-space
    if do_RSD:  RSL.pos_redshift_space(pos_h, vel_h, BoxSize, Hubble, redshift, axis)

    # calculate Pk
    delta = np.zeros((grid,grid,grid), dtype=np.float32)
    MASL.MA(pos_h, delta, BoxSize, MAS)
    delta /= np.mean(delta, dtype=np.float64);  delta -= 1.0
    Pk = PKL.Pk(delta, BoxSize, axis, MAS, threads)

    # save results to file
    hdr = ('Nhalos=%i BoxSize=%.3f'%(pos_h.shape[0],BoxSize))
    if save_multipoles:
        np.savetxt(fpk, np.transpose([Pk.k3D, Pk.Pk[:,0], Pk.Pk[:,1], Pk.Pk[:,2]]),
                   delimiter='\t', header=hdr)
    else:
        np.savetxt(fpk, np.transpose([Pk.k3D, Pk.Pk[:,0]]),
                   delimiter='\t', header=hdr)

root = '/simons/scratch/fvillaescusa/pdf_information'
##################################### INPUT #########################################
# Pk parameters
grid    = 1024
MAS     = 'CIC'
threads = 2

# halo parameters
fixed_Mmin = True   #whether fix Mmin or nbar
Mmin       = 3.2e13 #Msun/h; fixed Mmin
Nhalos     = 150000 #also consider the Nhalos more massive of the sim (nbar fixed)

# output folder name
root_out = '/simons/scratch/fvillaescusa/pdf_information/Pk/halos_Mmin_3.2e13/'
#####################################################################################

# find the redshift
z = {4:0, 3:0.5, 2:1, 1:2, 0:3}[snapnum]

# create output folder if it does not exist
if myrank==0:
    if not(os.path.exists(root_out+cosmo)):  os.system('mkdir %s/%s/'%(root_out,cosmo))

# get the realizations each cpu works on
numbers = np.where(np.arange(args.first, args.last)%nprocs==myrank)[0]
numbers = np.arange(args.first, args.last)[numbers]

######## standard simulations #########
for i in numbers:

    folder  = '%s/Halos/%s/%d'%(root,cosmo,i) #folder with the halo catalogue
    if not(os.path.exists(folder)):  continue
    snapdir = '%s/%s/%d/snapdir_%03d/snap_%03d'%(root,cosmo,i,snapnum,snapnum)

    # real-space
    fpk = '%s/%s/Pk_%d_z=%s.txt'%(root_out,cosmo,i,z)
    do_RSD, axis, save_multipoles = False, 0, False
    find_Pk(folder, snapdir, snapnum, grid, MAS, do_RSD, axis, threads,
            fixed_Mmin, Mmin, Nhalos, fpk, save_multipoles)

    # redshift-space
    for axis in [0,1,2]:
        fpk = '%s/%s/Pk_RS%d_%d_z=%s.txt'%(root_out,cosmo,axis,i,z)
        do_RSD, save_multipoles = True, True
        find_Pk(folder, snapdir, snapnum, grid, MAS, do_RSD, axis, threads,
                fixed_Mmin, Mmin, Nhalos, fpk, save_multipoles)

###### paired fixed realizations ######
for i in numbers:

    for pair in [0,1]:
        folder  = '%s/Halos/%s/NCV_%d_%d'%(root,cosmo,pair,i) #halo catalogue folder
        if not(os.path.exists(folder)):  continue
        snapdir = '%s/%s/NCV_%d_%d/snapdir_%03d/snap_%03d'\
                  %(root,cosmo,pair,i,snapnum,snapnum)

        # real-space
        fpk = '%s/%s/Pk_NCV_%d_%d_z=%s.txt'%(root_out,cosmo,pair,i,z)
        do_RSD, axis, save_multipoles = False, 0, False
        find_Pk(folder, snapdir, snapnum, grid, MAS, do_RSD, axis, threads,
                fixed_Mmin, Mmin, Nhalos, fpk, save_multipoles)

        # redshift-space
        for axis in [0,1,2]:
            fpk = '%s/%s/Pk_RS%d_NCV_%d_%d_z=%s.txt'%(root_out,cosmo,axis,pair,i,z)
            do_RSD, save_multipoles = True, True
            find_Pk(folder, snapdir, snapnum, grid, MAS, do_RSD, axis, threads,
                    fixed_Mmin, Mmin, Nhalos, fpk, save_multipoles)