import os
import pandas as pd
import numpy as np
import tensorflow as tf
import re
import time
from core.seq_to_seq import *
import pickle

def pickleStuff(filename, stuff):
    save_stuff = open(filename, "wb")
    pickle.dump(stuff, save_stuff)
    save_stuff.close()

def loadStuff(filename):
    saved_stuff = open(filename,"rb")
    stuff = pickle.load(saved_stuff)
    saved_stuff.close()
    return stuff

def count_words(count_dict, text):
    for sentence in text:
        for word in sentence.split():
            if word not in count_dict:
                count_dict[word] = 1
            else:
                count_dict[word] += 1

# Remove questions and answers that are shorter than 2 words and longer than 20 words.
min_line_length = 2
max_line_length = 25
threshold = 3

def initial_actions():
    with open('a', mode="r") as a:
        answers = a.readlines()

    with open('q', mode="r") as q:
        questions = q.readlines()
        # print(questions)

    # Clean the data
    clean_questions = []
    for question in questions:
        clean_questions.append(clean_text(question))

    # print(clean_questions)

    clean_answers = []
    for answer in answers:
        clean_answers.append(clean_text(answer))

    # print(clean_answers)

    # Filter out the questions that are too short/long
    short_questions_temp = []
    short_answers_temp = []

    i = 0
    for question in clean_questions:
        if len(question.split()) >= min_line_length and len(question.split()) <= max_line_length:
            short_questions_temp.append(question)
            short_answers_temp.append(clean_answers[i])
        i += 1

    # Filter out the answers that are too short/long
    short_questions = []
    short_answers = []

    i = 0
    for answer in short_answers_temp:
        if len(answer.split()) >= min_line_length and len(answer.split()) < max_line_length:
            short_answers.append(answer)
            short_questions.append(short_questions_temp[i])
        i += 1

    # Compare the number of lines we will use with the total number of lines.
    print("# of questions:", len(short_questions))
    print("# of answers:", len(short_answers))
    print("% of data used: {}%".format(round(len(short_questions) / len(questions), 4) * 100))

    # Create a dictionary for the frequency of the vocabulary
    vocab = {}
    for question in short_questions:
        for word in question.split():
            if word not in vocab:
                vocab[word] = 1
            else:
                vocab[word] += 1

    for answer in short_answers:
        for word in answer.split():
            if word not in vocab:
                vocab[word] = 1
            else:
                vocab[word] += 1

    # Remove rare words from the vocabulary.
    # We will aim to replace fewer than 5% of words with <UNK>
    # You will see this ratio soon.
    count = 0
    for k, v in vocab.items():
        if v >= threshold:
            count += 1

    print("Size of total vocab:", len(vocab))
    print("Size of vocab we will use:", count)
    return vocab, short_answers, short_questions

def data_to_int():
    vocab, short_answers, short_questions = initial_actions()

    # In case we want to use a different vocabulary sizes for the source and target text,
    # we can set different threshold values.
    # Nonetheless, we will create dictionaries to provide a unique integer for each word.
    questions_vocab_to_int = {}

    word_num = 0
    for word, count in vocab.items():
        if count >= threshold:
            questions_vocab_to_int[word] = word_num
            word_num += 1

    answers_vocab_to_int = {}

    word_num = 0
    for word, count in vocab.items():
        if count >= threshold:
            answers_vocab_to_int[word] = word_num
            word_num += 1

    # Add the unique tokens to the vocabulary dictionaries.
    codes = ['<PAD>', '<EOS>', '<UNK>', '<GO>']

    for code in codes:
        questions_vocab_to_int[code] = len(questions_vocab_to_int) + 1

    for code in codes:
        answers_vocab_to_int[code] = len(answers_vocab_to_int) + 1

    # Create dictionaries to map the unique integers to their respective words.
    # i.e. an inverse dictionary for vocab_to_int.
    questions_int_to_vocab = {v_i: v for v, v_i in questions_vocab_to_int.items()}
    answers_int_to_vocab = {v_i: v for v, v_i in answers_vocab_to_int.items()}

    # Check the length of the dictionaries.
    print(len(questions_vocab_to_int))
    print(len(questions_int_to_vocab))
    print(len(answers_vocab_to_int))
    print(len(answers_int_to_vocab))

    # Add the end of sentence token to the end of every answer.
    for i in range(len(short_answers)):
        short_answers[i] += ' <EOS>'

    # Convert the text to integers.
    # Replace any words that are not in the respective vocabulary with <UNK>
    questions_int = []
    for question in short_questions:
        ints = []
        for word in question.split():
            if word not in questions_vocab_to_int:
                ints.append(questions_vocab_to_int['<UNK>'])
            else:
                ints.append(questions_vocab_to_int[word])
        questions_int.append(ints)

    answers_int = []
    for answer in short_answers:
        ints = []
        for word in answer.split():
            # print(word)
            if word not in answers_vocab_to_int:
                ints.append(answers_vocab_to_int['<UNK>'])
            else:
                ints.append(answers_vocab_to_int[word])
        answers_int.append(ints)

    # Check the lengths
    print(len(questions_int))
    print(len(answers_int))

    pickleStuff("answers_int_to_vocab.p", answers_int_to_vocab)
    pickleStuff("questions_int_to_vocab.p", questions_int_to_vocab)

    # Calculate what percentage of all words have been replaced with <UNK>
    word_count = 0
    unk_count = 0

    for question in questions_int:
        for word in question:
            if word == questions_vocab_to_int["<UNK>"]:
                unk_count += 1
            word_count += 1

    for answer in answers_int:
        for word in answer:
            if word == answers_vocab_to_int["<UNK>"]:
                unk_count += 1
            word_count += 1

    unk_ratio = round(unk_count / word_count, 4) * 100

    print("Total number of words:", word_count)
    print("Number of times <UNK> is used:", unk_count)
    print("Percent of words that are <UNK>: {}%".format(round(unk_ratio, 3)))

    return questions_vocab_to_int, answers_vocab_to_int, questions_int, answers_int

def pad_sentence_batch(sentence_batch, vocab_to_int):
    """Pad sentences with <PAD> so that each sentence of a batch has the same length"""
    max_sentence = max([len(sentence) for sentence in sentence_batch])
    return [sentence + [vocab_to_int['<PAD>']] * (max_sentence - len(sentence)) for sentence in sentence_batch]

def batch_data(questions, answers, batch_size):
    """Batch questions and answers together"""
    questions_vocab_to_int, answers_vocab_to_int, _, __ = data_to_int()

    for batch_i in range(0, len(questions)//batch_size):
        start_i = batch_i * batch_size
        questions_batch = questions[start_i:start_i + batch_size]
        answers_batch = answers[start_i:start_i + batch_size]
        pad_questions_batch = np.array(pad_sentence_batch(questions_batch, questions_vocab_to_int))
        pad_answers_batch = np.array(pad_sentence_batch(answers_batch, answers_vocab_to_int))

        # Need the lengths for the _lengths parameters summary = answer
        pad_answers_lengths = []
        for answer in pad_answers_batch:
            pad_answers_lengths.append(len(answer))

        pad_questions_lengths = []
        for question in pad_questions_batch:
            pad_questions_lengths.append(len(question))

        yield pad_questions_batch, pad_answers_batch, pad_answers_lengths, pad_questions_lengths

def clean_text(text):
    '''Clean text by removing unnecessary characters and altering the format of words.'''

    text = text.lower()

    text = re.sub(r"\n", "", text)
    text = re.sub(r"i'm", "i am", text)
    text = re.sub(r"he's", "he is", text)
    text = re.sub(r"she's", "she is", text)
    text = re.sub(r"it's", "it is", text)
    text = re.sub(r"that's", "that is", text)
    text = re.sub(r"what's", "that is", text)
    text = re.sub(r"where's", "where is", text)
    text = re.sub(r"how's", "how is", text)
    text = re.sub(r"\'ll", " will", text)
    text = re.sub(r"\'ve", " have", text)
    text = re.sub(r"\'re", " are", text)
    text = re.sub(r"\'d", " would", text)
    text = re.sub(r"\'re", " are", text)
    text = re.sub(r"won't", "will not", text)
    text = re.sub(r"can't", "cannot", text)
    text = re.sub(r"n't", " not", text)
    text = re.sub(r"n'", "ng", text)
    text = re.sub(r"'bout", "about", text)
    text = re.sub(r"'til", "until", text)
    text = re.sub(r"[-()\"#/@;:<>{}`+=~|.!?,]", "", text)

    return text

def run_run():
    print('ebi swiney')
    if os.path.exists("sorted_answers.p"):
        print("LOAD FROM FILE")
        sorted_answers = loadStuff("sorted_answers.p")
        sorted_questions = loadStuff("sorted_questions.p")
        questions_vocab_to_int = loadStuff("questions_vocab_to_int.p")
        answers_vocab_to_int = loadStuff("answers_vocab_to_int.p")
    else:
        print("WTF!&")
        questions_vocab_to_int, answers_vocab_to_int, questions_int, answers_int = data_to_int()
        # questions_int,

        # Sort questions and answers by the length of questions.
        # This will reduce the amount of padding during training
        # Which should speed up training and help to reduce the loss

        sorted_questions = []
        sorted_answers = []

        for length in range(1, max_line_length + 1):
            for i in enumerate(questions_int):
                if len(i[1]) == length:
                    sorted_questions.append(questions_int[i[0]])
                    sorted_answers.append(answers_int[i[0]])

        print(len(sorted_questions))
        print(len(sorted_answers))
        print()
        for i in range(3):
            print(sorted_questions[i])
            print(sorted_answers[i])
            print()

        pickleStuff("questions_vocab_to_int.p", questions_vocab_to_int)
        pickleStuff("answers_vocab_to_int.p", answers_vocab_to_int)

        pickleStuff("sorted_answers.p", sorted_answers)
        pickleStuff("sorted_questions.p", sorted_questions)

    # Validate the training with 10% of the data
    train_valid_split = int(len(sorted_questions) * 0.05)

    # Split the questions and answers into training and validating data
    train_questions = sorted_questions[train_valid_split:]
    train_answers = sorted_answers[train_valid_split:]

    valid_questions = sorted_questions[:train_valid_split]
    valid_answers = sorted_answers[:train_valid_split]

    # Reset the graph to ensure that it is ready for training
    tf.reset_default_graph()
    # Start the session
    # Build the graph
    train_graph = tf.Graph()
    # Set the graph to default to ensure that it is ready for training
    with train_graph.as_default():

        # Load the model inputs
        input_data, targets, lr, keep_prob, answer_length, max_answer_length, questions_length = model_inputs()

        # Find the shape of the input data for sequence_loss
        input_shape = tf.shape(input_data)

        # Create the training and inference logits
        train_logits, inference_logits = seq2seq_model(
            input_data=tf.reverse(input_data, [-1]),
            target_data=targets,
            keep_prob=keep_prob,
            batch_size=batch_size,
            questions_length=questions_length,
            answers_vocab_size=len(answers_vocab_to_int),
            questions_vocab_size=len(questions_vocab_to_int),
            enc_embedding_size=100,
            dec_embedding_size=100,
            rnn_size=rnn_size,
            num_layers=num_layers,
            questions_vocab_to_int=questions_vocab_to_int,
            answer_length=answer_length,
            max_answer_length=max_line_length
        )

        # Create tensors for the training logits and inference logits
        training_logits = tf.identity(train_logits[0].rnn_output, 'logits')
        inference_logits = tf.identity(inference_logits[0].sample_id, name='predictions')

        # Create the weights for sequence_loss, the sould be all True across since each batch is padded
        masks = tf.sequence_mask(answer_length, max_answer_length, dtype=tf.float32, name='masks')

        with tf.name_scope("optimization"):
            # Loss function
            cost = tf.contrib.seq2seq.sequence_loss(
                training_logits,
                targets,
                masks)

            # Optimizer
            # learning_rate = tf.Variable(tf.float32)
            optimizer = tf.train.AdamOptimizer(lr)

            # Gradient Clipping
            gradients = optimizer.compute_gradients(cost)
            capped_gradients = [(tf.clip_by_value(grad, -5., 5.), var) for grad, var in gradients if grad is not None]
            train_op = optimizer.apply_gradients(capped_gradients)
    print("Graph is built.")
    graph_location = "tb/"
    print(graph_location)
    train_writer = tf.summary.FileWriter(graph_location)
    train_writer.add_graph(train_graph)

    # Train the Model
    learning_rate_decay = 0.95
    min_learning_rate = 0.0001
    display_step = 20  # Check training loss after every 20 batches
    stop_early = 0
    stop = 20  # If the update loss does not decrease in 20 consecutive update checks, stop training
    per_epoch = 3  # Make 3 update checks per epoch
    update_check = (len(sorted_questions) // batch_size // per_epoch) - 1

    update_loss = 0
    batch_loss = 0
    summary_update_loss = []  # Record the update losses for saving improvements in the model

    sess = tf.Session()
    checkpoint = "ddd/best_model.ckpt"
    with tf.Session(graph=train_graph) as sess:
        sess.run(tf.global_variables_initializer())
        learning_rate = 0.01

        for epoch_i in range(1, epochs + 1):
            for batch_i, (questions_batch, answers_batch, answers_lengths, questions_lengths) in enumerate(
                    batch_data(train_questions, train_answers, batch_size)):
                start_time = time.time()

                """
                print(questions_batch)
                # [[1605 4277 1361]]

                print(answers_batch)
                # [[1361 2095 5436]]

                print(answers_lengths)
                # [3]

                print(questions_lengths)
                # [9]
                """
                _, loss = sess.run(
                    [train_op, cost],
                    {input_data: questions_batch,
                     targets: answers_batch,
                     lr: learning_rate,

                     answer_length: answers_lengths,
                     questions_length: questions_lengths,

                     keep_prob: keep_probability,
                     })

                update_loss += loss
                end_time = time.time()
                batch_time = end_time - start_time

                if batch_i % display_step == 0:
                    print('Epoch {:>3}/{} Batch {:>4}/{} - Loss: {:>6.3f}, Seconds: {:>4.2f}'
                          .format(epoch_i,
                                  epochs,
                                  batch_i,
                                  len(train_questions) // batch_size,
                                  update_loss / display_step,
                                  batch_time * display_step))
                    update_loss = 0

                if batch_i % update_check == 0 and batch_i > 0:
                    total_valid_loss = 0
                    start_time = time.time()
                    for batch_ii, (questions_batch, answers_batch, answers_lengths, questions_lengths) in \
                            enumerate(batch_data(valid_questions, valid_answers, batch_size)):
                        valid_loss = sess.run(
                            cost, {
                                    input_data: questions_batch,
                                    targets: answers_batch,
                                    questions_length: questions_lengths,
                                    answer_length: answers_lengths,
                                   keep_prob: 1
                            })
                        total_valid_loss += valid_loss
                    end_time = time.time()
                    batch_time = end_time - start_time
                    avg_valid_loss = total_valid_loss / (len(valid_questions) / batch_size)
                    print('Valid Loss: {:>6.3f}, Seconds: {:>5.2f}'.format(avg_valid_loss, batch_time))

                    # Reduce learning rate, but not below its minimum value
                    learning_rate *= learning_rate_decay
                    if learning_rate < min_learning_rate:
                        learning_rate = min_learning_rate

                    summary_update_loss.append(avg_valid_loss)
                    if avg_valid_loss <= min(summary_update_loss):
                        print('New Record!')
                        stop_early = 0
                        saver = tf.train.Saver()
                        saver.save(sess, checkpoint)

                    else:
                        print("No Improvement.")
                        stop_early += 1
                        if stop_early == stop:
                            break

            if stop_early == stop:
                print("Stopping Training.")
                break


import string

data = """\
vJrwpWtwJgWrhcsFMMfFFhFp
jqHRNqRjqzjGDLGLrsFMfFZSrLrFZsSL
PmmdzqPrVvPwwTWBwg
wMqvLMZHhHMvwLHjbvcjnnSBnvTQFn
ttgJtRGJQctTZtZT
CrZsJsPPZsGzwwsLwLmpwMDw
"""

# data = open("2022/day03/data.txt").read()

priorities = []
lines = data.split()
# data grouped into lines of 3
for group in [lines[i : i + 3] for i in range(0, len(lines), 3)]:

    dupl_char = set.intersection(*[set(l) for l in group]).pop()

    if dupl_char in string.ascii_lowercase:
        priorities.append(ord(dupl_char) - ord("a") + 1)
    else:
        priorities.append(ord(dupl_char) - ord("A") + 1 + 26)

print(sum(priorities))


import socket
import pickle

mySocket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)

message = pickle.dumps("turttle")

mySocket.sendto(message, ("127.0.0.1", 4444))

response, _ = mySocket.recvfrom(50)

print(pickle.loads(response))

Computer Networks/Socket programming/lab 1/problem 4/client.py


class Solution:
    def maxArea(self, h: int, w: int, horizontalCuts: List[int], verticalCuts: List[int]) -> int:
        '''
        find max area of a cut approach. sort with h, w, 0, cuts => find the max difference between two cuts. area is the multiplication of the two

        time O(n log n) | space O(n)

        '''
        #sort 0, h, w cuts. inserting 0 to make sure subtracting works later on
        horizontalCuts = [0] + horizontalCuts + [h]
        verticalCuts = [0] + verticalCuts + [w]

        horizontalCuts.sort()
        verticalCuts.sort()

        #find the max difference between two consecutive cuts
        maxh = float("-inf")
        maxv = float("-inf")

        for i in range(1, len(horizontalCuts)):
            maxh = max(maxh, horizontalCuts[i] - horizontalCuts[i-1])

        for i in range(1, len(verticalCuts)):
            maxv = max(maxv, verticalCuts[i] - verticalCuts[i-1])

        # the mod is for very high numbers only
        return (maxh * maxv) % (10**9 + 7)


"""Robust metrics to evaluate performance of copy number estimates.
"""
from __future__ import absolute_import, division, print_function
from builtins import zip

import logging

import numpy as np
import pandas as pd
from scipy import stats

from .descriptives import (biweight_midvariance, interquartile_range,
                           median_absolute_deviation)

def do_metrics(cnarrs, segments=None, skip_low=False):
    """Compute coverage deviations and other metrics for self-evaluation."""
    # Catch if passed args are single CopyNumArrays instead of lists
    from .cnary import CopyNumArray as CNA
    if isinstance(cnarrs, CNA):
        cnarrs = [cnarrs]
    if isinstance(segments, CNA):
        segments = [segments]
    elif segments is None:
        segments = [None]
    else:
        segments = list(segments)
    if skip_low:
        cnarrs = (cna.drop_low_coverage() for cna in cnarrs)
    rows = ((cna.meta.get("filename", cna.sample_id),
             len(seg) if seg is not None else '-'
            ) + ests_of_scale(cna.residuals(seg))
            for cna, seg in zip_repeater(cnarrs, segments))
    colnames = ["sample", "segments", "stdev", "mad", "iqr", "bivar"]
    return pd.DataFrame.from_records(rows, columns=colnames)

def zip_repeater(iterable, repeatable):
    """Repeat a single segmentation to match the number of copy ratio inputs"""
    rpt_len = len(repeatable)
    if rpt_len == 1:
        rpt = repeatable[0]
        for it in iterable:
            yield it, rpt
    else:
        i = -1
        for i, (it, rpt) in enumerate(zip(iterable, repeatable)):
            yield it, rpt
        # Require lengths to match
        if i + 1 != rpt_len:
            raise ValueError("""Number of unsegmented and segmented input files
                             did not match (%d vs. %d)""" % (i, rpt_len))

def ests_of_scale(deviations):
    """Estimators of scale: standard deviation, MAD, biweight midvariance.

    Calculates all of these values for an array of deviations and returns them
    as a tuple.
    """
    std = np.std(deviations, dtype=np.float64)
    mad = median_absolute_deviation(deviations)
    iqr = interquartile_range(deviations)
    biw = biweight_midvariance(deviations)
    return (std, mad, iqr, biw)

def segment_mean(cnarr, skip_low=False):
    """Weighted average of bin log2 values."""
    if skip_low:
        cnarr = cnarr.drop_low_coverage()
    if len(cnarr) == 0:
        return np.nan
    if 'weight' in cnarr:
        return np.average(cnarr['log2'], weights=cnarr['weight'])
    return cnarr['log2'].mean()

# Intervals

def confidence_interval_bootstrap(bins, alpha, bootstraps=100, smoothed=True):
    """Confidence interval for segment mean log2 value, estimated by bootstrap."""
    if not 0 < alpha < 1:
        raise ValueError("alpha must be between 0 and 1; got %s" % alpha)
    if bootstraps <= 2 / alpha:
        new_boots = int(np.ceil(2 / alpha))
        logging.warn("%d bootstraps not enough to estimate CI alpha level %f; "
                     "increasing to %d", bootstraps, alpha, new_boots)
        bootstraps = new_boots
    # Bootstrap for CI
    k = len(bins)
    if k < 2:
        return np.array([bins["log2"][0], bins["log2"][0]])

    np.random.seed(0xA5EED)
    rand_indices = np.random.randint(0, k, (bootstraps, k))
    samples = [bins.data.take(idx) for idx in rand_indices]
    if smoothed:
        # samples = _smooth_samples(bins, samples, alpha)
        pass
    # Recalculate segment means
    bootstrap_dist = np.array([segment_mean(samp) for samp in samples])
    alphas = np.array([alpha / 2, 1 - alpha / 2])
    if not smoothed:
        # alphas = _bca_correct_alpha(bins, bootstrap_dist, alphas)
        pass
    ci = np.percentile(bootstrap_dist, list(100 * alphas))
    return ci

def _smooth_samples(bins, samples, alpha):
    k = len(bins)
    # Essentially, resample from a kernel density estimate of the data
    # instead of the original data.
    # Estimate KDE bandwidth (Polansky 1995)
    resids = bins['log2'] - bins['log2'].mean()
    s_hat = 1/k * (resids**2).sum()  # sigma^2 = E[X-theta]^2
    y_hat = 1/k * abs((resids**3).sum())  # gamma = E[X-theta]^3
    z = stats.norm.ppf(alpha / 2)  # or alpha?
    bw = k**(-1/4) * np.sqrt(y_hat*(z**2 + 2) / (3*s_hat*z))
    # NB: or, Silverman's Rule for KDE bandwidth (roughly):
    # std = interquartile_range(bins['log2']) / 1.34
    # bw = std * (k*3/4) ** (-1/5)
    if bw > 0:
        samples = [samp.assign(log2=lambda x:
                                x['log2'] + bw * np.random.randn(k))
                    for samp in samples]
        logging.debug("Smoothing worked for this segment (bw=%s)", bw)
    else:
        logging.debug("Smoothing not needed for this segment (bw=%s)", bw)
    return samples

def _bca_correct_alpha(bins, bootstrap_dist, alphas):
    # BCa correction (Efron 1987, "Better Bootstrap Confidence Intervals")
    # http://www.tandfonline.com/doi/abs/10.1080/01621459.1987.10478410
    # Ported from R package "bootstrap" function "bcanon"
    n_boots = len(bootstrap_dist)
    orig_mean = segment_mean(bins)
    logging.warn("boot samples less: %s / %s",
                 (bootstrap_dist < orig_mean).sum(),
                 n_boots)
    n_boots_below = (bootstrap_dist < orig_mean).sum()
    if n_boots_below == 0:
        logging.warn("boots mean %s, orig mean %s",
                     bootstrap_dist.mean(), orig_mean)
    else:
        logging.warn("boot samples less: %s / %s",
                     n_boots_below, n_boots)
    z0 = stats.norm.ppf((bootstrap_dist < orig_mean).sum() / n_boots)
    zalpha = stats.norm.ppf(alphas)
    # Jackknife influence values
    u = np.array([segment_mean(bins.concat([bins[:i], bins[i+1:]]))
                    for i in range(len(bins))])
    uu = u.mean() - u
    acc = (u**3).sum() / (6 * (uu**2).sum()**1.5)
    alphas = stats.norm.cdf(z0 + (z0 + zalpha)
                                    / (1 - acc * (z0 + zalpha)))
    logging.warn("New alphas: %s -- via z0=%s, za=%s, acc=%s",
                    alphas, z0, zalpha, acc)
    if not 0 < alphas[0] < 1 and 0 < alphas[1] < 1:
        raise ValueError("CI alphas should be in (0,1); got %s" % alphas)
    return alphas

def prediction_interval(bins, alpha):
    """Prediction interval, estimated by percentiles."""
    pct_lo = 100 * alpha / 2
    pct_hi = 100 * (1 - alpha / 2)
    # ENH: weighted percentile
    return np.percentile(bins['log2'], [pct_lo, pct_hi])


######## Evaluation pipeline for perch data on pointrcnn model #########
# Author: Beiming Li

import _init_path
import os
import numpy as np
import torch
from torch.utils.data import DataLoader
from lib.datasets.perch_dataset import PerchDataset
import argparse

from lib.net.point_rcnn import PointRCNN
import tools.train_utils.train_utils as train_utils
from lib.utils.bbox_transform import decode_bbox_target
from tools.kitti_object_eval_python.evaluate import evaluate as kitti_evaluate

from lib.config import cfg, cfg_from_file, save_config_to_file, cfg_from_list
import lib.utils.kitti_utils as kitti_utils
import lib.utils.iou3d.iou3d_utils as iou3d_utils
from datetime import datetime
import logging
import re
import glob
import time
from tensorboardX import SummaryWriter
import tqdm

import matplotlib.pyplot as plt

np.random.seed(1024)  # set the same seed

parser = argparse.ArgumentParser(description="arg parser")

parser = argparse.ArgumentParser(description="arg parser")
parser.add_argument('--cfg_file', type=str, default='cfgs/default.yml', help='specify the config for evaluation')
parser.add_argument("--eval_mode", type=str, default='rpn', required=True, help="specify the evaluation mode")

parser.add_argument('--eval_all', action='store_true', default=False, help='whether to evaluate all checkpoints')
parser.add_argument('--test', action='store_true', default=False, help='evaluate without ground truth')
parser.add_argument("--ckpt", type=str, default=None, help="specify a checkpoint to be evaluated")
parser.add_argument("--rpn_ckpt", type=str, default=None, help="specify the checkpoint of rpn if trained separated")
parser.add_argument("--rcnn_ckpt", type=str, default=None, help="specify the checkpoint of rcnn if trained separated")

parser.add_argument('--batch_size', type=int, default=1, help='batch size for evaluation')
parser.add_argument('--workers', type=int, default=4, help='number of workers for dataloader')
parser.add_argument("--extra_tag", type=str, default='default', help="extra tag for multiple evaluation")
parser.add_argument('--output_dir', type=str, default=None, help='specify an output directory if needed')
parser.add_argument("--ckpt_dir", type=str, default=None, help="specify a ckpt directory to be evaluated if needed")

parser.add_argument('--save_result', action='store_true', default=False, help='save evaluation results to files')
parser.add_argument('--save_rpn_feature', action='store_true', default=False,
                    help='save features for separately rcnn training and evaluation')

parser.add_argument('--random_select', action='store_true', default=True, help='sample to the same number of points')
parser.add_argument('--start_epoch', default=0, type=int, help='ignore the checkpoint smaller than this epoch')
parser.add_argument("--rcnn_eval_roi_dir", type=str, default=None,
                    help='specify the saved rois for rcnn evaluation when using rcnn_offline mode')
parser.add_argument("--rcnn_eval_feature_dir", type=str, default=None,
                    help='specify the saved features for rcnn evaluation when using rcnn_offline mode')
parser.add_argument('--set', dest='set_cfgs', default=None, nargs=argparse.REMAINDER,
                    help='set extra config keys if needed')
args = parser.parse_args()

def create_dataloader():
    mode = 'TEST' if args.test else 'EVAL'
    DATA_PATH = os.path.join('..', 'data')

    # create dataloader
    test_set = PerchDataset(root_dir=DATA_PATH, mode=mode)

    test_loader = DataLoader(test_set, batch_size=args.batch_size, shuffle=False, pin_memory=True,
                             num_workers=args.workers, collate_fn=test_set.collate_batch)

    return test_loader

def load_ckpt_based_on_args(model):
    train_utils.load_checkpoint(model, filename=args.ckpt)

def eval_one_epoch(model, dataloader):
    if cfg.RPN.ENABLED and not cfg.RCNN.ENABLED:
        ret_dict = eval_one_epoch_rpn(model, dataloader)
    elif not cfg.RPN.ENABLED and cfg.RCNN.ENABLED:
        ret_dict = eval_one_epoch_rcnn(model, dataloader)
    elif cfg.RPN.ENABLED and cfg.RCNN.ENABLED:
        ret_dict = eval_one_epoch_joint(model, dataloader)
    else:
        raise NotImplementedError
    return ret_dict

def eval_one_epoch_rpn(model, dataloader):
    np.random.seed(1024)
    mode = 'TEST' if args.test else 'EVAL'

    thresh_list = [0.1, 0.3, 0.5, 0.7, 0.9]
    total_recalled_bbox_list, total_gt_bbox = [0] * 5, 0

    for data in dataloader:
        pts_rect, pts_features, pts_input, gt_boxes3d = data['pts_rect'], data['pts_features'], data['pts_input'], data['gt_boxes3d']
        inputs = torch.from_numpy(pts_input).cuda(non_blocking=True).float()
        input_data = {'pts_input': inputs}

        # model inference
        ret_dict = model(input_data)
        rpn_cls, rpn_reg = ret_dict['rpn_cls'], ret_dict['rpn_reg']
        backbone_xyz, backbone_features = ret_dict['backbone_xyz'], ret_dict['backbone_features']

        rpn_scores_raw = rpn_cls[:, :, 0]
        rpn_scores = torch.sigmoid(rpn_scores_raw)
        seg_result = (rpn_scores > cfg.RPN.SCORE_THRESH).long()

        # proposal layer
        rois, roi_scores_raw = model.rpn.proposal_layer(rpn_scores_raw, rpn_reg, backbone_xyz)  # (B, M, 7)
        batch_size = rois.shape[0]

        # calculate recall and save results to file
        for bs_idx in range(batch_size):
            cur_scores_raw = roi_scores_raw[bs_idx]  # (N)
            cur_boxes3d = rois[bs_idx]  # (N, 7)
            cur_seg_result = seg_result[bs_idx]
            cur_pts_rect = pts_rect[bs_idx]
            visualize_data(pts_input[bs_idx], gt_boxes3d[bs_idx], cur_boxes3d[cur_scores_raw > 0].cpu())

    return ret_dict

def eval_one_epoch_joint(model, dataloader):
    np.random.seed(666)
    MEAN_SIZE = torch.from_numpy(cfg.CLS_MEAN_SIZE[0]).cuda()
    mode = 'TEST' if args.test else 'EVAL'

    model.eval()

    thresh_list = [0.1, 0.3, 0.5, 0.7, 0.9]
    total_recalled_bbox_list, total_gt_bbox = [0] * 5, 0
    total_roi_recalled_bbox_list = [0] * 5
    cnt = final_total = total_cls_acc = total_cls_acc_refined = total_rpn_iou = 0

    progress_bar = tqdm.tqdm(total=len(dataloader), leave=True, desc='eval')

    for data in dataloader:
        pts_rect, pts_features, pts_input = data['pts_rect'], data['pts_features'], data['pts_input']
        batch_size = len(pts_input)
        inputs = torch.from_numpy(pts_input).cuda(non_blocking=True).float()
        input_data = {'pts_input': inputs}
        # model inference
        ret_dict = model(input_data)

        roi_scores_raw = ret_dict['roi_scores_raw']  # (B, M)
        roi_boxes3d = ret_dict['rois']  # (B, M, 7)
        seg_result = ret_dict['seg_result'].long()  # (B, N)

        rcnn_cls = ret_dict['rcnn_cls'].view(batch_size, -1, ret_dict['rcnn_cls'].shape[1])
        rcnn_reg = ret_dict['rcnn_reg'].view(batch_size, -1, ret_dict['rcnn_reg'].shape[1])  # (B, M, C)

        # bounding box regression
        anchor_size = MEAN_SIZE
        if cfg.RCNN.SIZE_RES_ON_ROI:
            assert False

        pred_boxes3d = decode_bbox_target(roi_boxes3d.view(-1, 7), rcnn_reg.view(-1, rcnn_reg.shape[-1]),
                                            anchor_size=anchor_size,
                                            loc_scope=cfg.RCNN.LOC_SCOPE,
                                            loc_bin_size=cfg.RCNN.LOC_BIN_SIZE,
                                            num_head_bin=cfg.RCNN.NUM_HEAD_BIN,
                                            get_xz_fine=True, get_y_by_bin=cfg.RCNN.LOC_Y_BY_BIN,
                                            loc_y_scope=cfg.RCNN.LOC_Y_SCOPE, loc_y_bin_size=cfg.RCNN.LOC_Y_BIN_SIZE,
                                            get_ry_fine=True).view(batch_size, -1, 7)
        # print(pred_boxes3d, rcnn_cls)
        # scoring
        if rcnn_cls.shape[2] == 1:
            raw_scores = rcnn_cls  # (B, M, 1)

            norm_scores = torch.sigmoid(raw_scores)
            pred_classes = (norm_scores > cfg.RCNN.SCORE_THRESH).long()

        # evaluation
        recalled_num = gt_num = rpn_iou = 0
        if not args.test:
            if not cfg.RPN.FIXED:
                rpn_cls_label, rpn_reg_label = data['rpn_cls_label'], data['rpn_reg_label']
                rpn_cls_label = torch.from_numpy(rpn_cls_label).cuda(non_blocking=True).long()

            gt_boxes3d = data['gt_boxes3d']

            for k in range(batch_size):
                # calculate recall
                cur_gt_boxes3d = gt_boxes3d[k]
                tmp_idx = cur_gt_boxes3d.__len__() - 1

                while tmp_idx >= 0 and cur_gt_boxes3d[tmp_idx].sum() == 0:
                    tmp_idx -= 1

                if tmp_idx >= 0:
                    cur_gt_boxes3d = cur_gt_boxes3d[:tmp_idx + 1]

                    cur_gt_boxes3d = torch.from_numpy(cur_gt_boxes3d).cuda(non_blocking=True).float()
                    iou3d = iou3d_utils.boxes_iou3d_gpu(pred_boxes3d[k], cur_gt_boxes3d)
                    gt_max_iou, _ = iou3d.max(dim=0)
                    refined_iou, _ = iou3d.max(dim=1)

                    for idx, thresh in enumerate(thresh_list):
                        total_recalled_bbox_list[idx] += (gt_max_iou > thresh).sum().item()
                    recalled_num += (gt_max_iou > 0.7).sum().item()
                    gt_num += cur_gt_boxes3d.shape[0]
                    total_gt_bbox += cur_gt_boxes3d.shape[0]

                    # original recall
                    iou3d_in = iou3d_utils.boxes_iou3d_gpu(roi_boxes3d[k], cur_gt_boxes3d)
                    gt_max_iou_in, _ = iou3d_in.max(dim=0)

                    for idx, thresh in enumerate(thresh_list):
                        total_roi_recalled_bbox_list[idx] += (gt_max_iou_in > thresh).sum().item()

                if not cfg.RPN.FIXED:
                    fg_mask = rpn_cls_label > 0
                    correct = ((seg_result == rpn_cls_label) & fg_mask).sum().float()
                    union = fg_mask.sum().float() + (seg_result > 0).sum().float() - correct
                    rpn_iou = correct / torch.clamp(union, min=1.0)
                    total_rpn_iou += rpn_iou.item()

        disp_dict = {'mode': mode, 'recall': '%d/%d' % (total_recalled_bbox_list[3], total_gt_bbox)}
        progress_bar.set_postfix(disp_dict)
        progress_bar.update()

        # scores thresh
        inds = norm_scores > cfg.RCNN.SCORE_THRESH

        for k in range(batch_size):
            cur_inds = inds[k].view(-1)
            if cur_inds.sum() == 0:
                continue

            pred_boxes3d_selected = pred_boxes3d[k, cur_inds]
            raw_scores_selected = raw_scores[k, cur_inds]
            norm_scores_selected = norm_scores[k, cur_inds]

            # NMS thresh
            # rotated nms
            boxes_bev_selected = kitti_utils.boxes3d_to_bev_torch(pred_boxes3d_selected)
            keep_idx = iou3d_utils.nms_gpu(boxes_bev_selected, raw_scores_selected, cfg.RCNN.NMS_THRESH).view(-1)
            pred_boxes3d_selected = pred_boxes3d_selected[keep_idx]
            scores_selected = raw_scores_selected[keep_idx]
            pred_boxes3d_selected, scores_selected = pred_boxes3d_selected.cpu().numpy(), scores_selected.cpu().numpy()

            if (gt_max_iou > thresh).sum().item() != 0:
                visualize_data(pts_input[k], gt_boxes3d[k], pred_boxes3d_selected)

    progress_bar.close()
    return ret_dict

def visualize_data(pts_input, gt_boxes3d, pred_boxes3d):
    print("visualization")
    fig = plt.figure()
    ax = plt.axes(projection='3d')
    ax.set_xlim(-100, 25)
    ax.set_ylim(-100, 50)
    ax.set_zlim(-100, 100)

    x, y, z = pts_input.T
    ax.scatter3D(x, y, z, s=0.5)

    x_gt, y_gt, z_gt = gt_boxes3d[:, 0], gt_boxes3d[:, 1], gt_boxes3d[:, 2]
    ax.scatter3D(x_gt, y_gt, z_gt, s=50, c='r')

    x_pred, y_pred, z_pred = pred_boxes3d[:, 0], pred_boxes3d[:, 1], pred_boxes3d[:, 2]
    ax.scatter3D(x_pred, y_pred, z_pred, s=100, c='g')
    plt.show()

if __name__ == "__main__":
    # merge config and log to file
    if args.cfg_file is not None:
        cfg_from_file(args.cfg_file)
    if args.set_cfgs is not None:
        cfg_from_list(args.set_cfgs)
    cfg.TAG = os.path.splitext(os.path.basename(args.cfg_file))[0]

    if args.eval_mode == 'rpn':
        cfg.RPN.ENABLED = True
        cfg.RCNN.ENABLED = False
    elif args.eval_mode == 'rcnn':
        cfg.RCNN.ENABLED = True
        cfg.RPN.ENABLED = cfg.RPN.FIXED = True
    else:
        raise NotImplementedError

    with torch.no_grad():
        # create dataloader & network
        test_loader = create_dataloader()
        model = PointRCNN(num_classes=2, use_xyz=True, mode='TEST')
        model.cuda()

        # load checkpoint
        load_ckpt_based_on_args(model)

        # start evaluation
        eval_one_epoch(model, test_loader)


#! /usr/bin/env python

try:
    from urllib.error import HTTPError
    from urllib.parse import quote, urlencode
    from urllib.request import urlopen
except ImportError:
    from urllib import urlencode
    from urllib2 import quote, urlopen, HTTPError

import argparse
import time
import pandas as pd
import json
import xml.etree.ElementTree as ET

# use this url to save cid list in eutils server
NAMES_LISTKEY_API = "https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/name/cids/JSON?list_return=listkey"

def xml2df(root):

    cols = 'ID,Name,PharmaActions,InChIKey'
    df = pd.DataFrame(columns=cols.split(","))
    for i, drug in enumerate(root):

        info = {}
        id = drug.find("./*[@Name='CID']")
        info['ID'] = id.text if id is not None else None
        name = drug.find("./*[@Name='MeSHHeadingList']/*[@Name='string']")
        info['Name'] = name.text if name is not None else None
        info['PharmaActions'] = ";".join(x.text for x in drug.findall("./*[@Name='PharmActionList']/*[@Name='string']"))
        inchikey = drug.find("./*[@Name='InChIKey']")
        info['InChIKey'] = inchikey.text if inchikey is not None else None
        df.loc[i] = pd.Series(info)

    return df

def get_id(name):

    namespace = 'name'
    post_body = urlencode([(namespace, name)]).encode('utf8')
    esummary = None
    try:
        response = urlopen(NAMES_LISTKEY_API, post_body)
        print("successfully get list key result")
        # Construct esummary retrieve url
        lsit_key_result = json.loads(response.read())
        esummary = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi" \
                   "?db=pccompound&WebEnv=" + lsit_key_result['IdentifierList']['EntrezWebEnv']\
                   + "&query_key=" + str(lsit_key_result['IdentifierList']['EntrezQueryKey'])

    except HTTPError as e:
        print("Fail to retrieve messages for {0!r}, caused by {1!r}".format(name, e))

    try:
        time.sleep(5)
        if esummary is not None:
            summary_response = urlopen(esummary)
            print("successfully get summary result")
            # Parsing the downloaded esummary xml string
            xml_result = summary_response.read().decode('utf-8')
            root = ET.fromstring(xml_result)
            for i, drug in enumerate(root):
                if i == 0:
                    id = drug.find("./*[@Name='CID']")
                    return id.text
    except HTTPError as e:
        print("Fail to retrieve summaries for {0!r}, caused by {1!r}".format(name, e))

if __name__ == "__main__":

    argparser = argparse.ArgumentParser()
    argparser.add_argument('name_ls_df', help = 'the data frame saving names to be searched')
    argparser.add_argument('result_df', help = 'directory to save final results')
    # argparser.add_argument('xml_file', help = "directory to save requested xml results from pubchem server")
    args = argparser.parse_args()

    names = pd.read_csv(args.name_ls_df)
    names = names[names['pert_type'] == 'trt_cp']
    names = names[~names['pert_iname'].str.startswith('BRD')]
    names_list = set(names['pert_iname'].astype(str))
    print(len(names_list))
    result = pd.DataFrame(columns = ['name', 'cid'])

    # construct apiurl and add cids into POST body
    for i, name in enumerate(names_list):
        cid = get_id(name)
        result.loc[i] = pd.Series({'name': name, 'cid': cid})
        if i%100 == 0:
            result.to_csv(args.result_df)
    result.to_csv(args.result_df)


import bisect
from collections import defaultdict

class MajorityChecker(object):

    def __init__(self, arr):
        """
        :type arr: List[int]
        """
        self.d = defaultdict(list)
        for i, j in enumerate(arr):
            self.d[j].append(i)

    def query(self, left, right, threshold):
        """
        :type left: int
        :type right: int
        :type threshold: int
        :rtype: int
        """
        for k, v in self.d.items():
            if len(v) < threshold:
                continue
            left_index = bisect.bisect_left(v, left)
            right_index = bisect.bisect_right(v, right)
            if right_index - left_index >= threshold:
                return k
        return -1

# Your MajorityChecker object will be instantiated and called as such:
# obj = MajorityChecker(arr)
# param_1 = obj.query(left,right,threshold)

python/Binary Search/Online Majority Element In Subarray/Online Majority Element In Subarray.py


from django.contrib import messages
from django.contrib.auth.decorators import login_required

from django.shortcuts import render

# Create your views here.

from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import render

# Create your views here.
from product.models import Comment, CommentForm

def index(request):

    return HttpResponse("Product Page")

@login_required(login_url='/login')
def add(request, id):
    url = request.META.get('HTTP_REFERER')
    if request.method == 'POST':
        form = CommentForm(request.POST)
        if form.is_valid():
            current_user = request.user

            data = Comment()
            data.user_id = current_user.id
            data.product_id = id
            data.subject = request.user.username
            data.comment = form.cleaned_data['comment']
            #data.rate = form.cleaned_data['rate']
            data.ip = request.META.get('REMOTE_ADDR')
            data.save()
            messages.success(request, "Yorumunuz Başarılı bir şekilde eklendi")
            #url = request.META.get('HTTP_REFERER')
            return HttpResponseRedirect(url)
        else:
            messages.warning(request, "Yorumunuz Eklenemedi :( ")
    return HttpResponseRedirect(url)


# ACM Contest and Blackout

def two_cheapest(graph: dict) -> tuple:
    first = second = 0
    connected = set()

    return first, second

n, m = list(map(int, input().split()))
graph = [[10 ** 9 for _ in range(n)] for _ in range(n)]
for _ in range(m):
    start, end, cost = list(map(int, input().split()))
    graph[start - 1][end - 1] = cost
    graph[end - 1][start - 1] = cost

seen = [False for _ in range(n)]
min_edge_from = [10 ** 9 for _ in range(n)]
min_edge_outer = [-1 for _ in range(n)]
min_edge_from[0] = 0
helper = []
first = second = 0
for _ in range(n):
    vert = -1
    for j in range(n):
        if not seen[j] and (min_edge_from[j] < min_edge_from[vert] or vert < 0):
            vert = j
    seen[vert] = True

    for k in range(n):
        if graph[vert][k] < min_edge_from[k]:
            min_edge_from[k] = graph[vert][k]
            min_edge_outer[k] = vert
    print(_, seen, min_edge_from, min_edge_outer)

print(sum(min_edge_from))


class Solution:
    def format(self, words, a, b, maxWidth, p, isLast):
        n = b - a
        if n == 1 or isLast:
            used = p[b] - p[a] + n - 1
            return " ".join(words[a:b]) + " " * (maxWidth - used)
        spaces = maxWidth - (p[b] - p[a])
        gap = spaces // (n - 1)
        k = spaces % (n - 1)
        res = []
        for i in range(a, b):
            res.append(words[i])
            if i == b - 1:
                break
            if k > 0:
                res.append(" " * (gap + 1))
                k -= 1
            else:
                res.append(" " * gap)
        return "".join(res)

    def justify(self, words, maxWidth, p, i, n):
        if i >= n:
            return []
        j = i
        l = 0
        while j < n and l + len(words[j]) <= maxWidth:
            l += len(words[j]) + 1
            j += 1
        rem = self.justify(words, maxWidth, p, j, n)
        return [self.format(words, i, j, maxWidth, p, len(rem) == 0)] + rem

    def fullJustify(self, words: List[str], maxWidth: int) -> List[str]:
        p = [0]
        for w in words:
            p.append(p[-1] + len(w))
        return self.justify(words, maxWidth, p, 0, len(words))


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import collections

def pretty_print_node_path(path):
  if not path:
    return "root object"
  else:
    return "root." + ".".join([p.name for p in path])

class CyclicDependencyError(Exception):

  def __init__(self, leftover_dependency_map):
    """Creates a CyclicDependencyException."""
    # Leftover edges that were not able to be topologically sorted.
    self.leftover_dependency_map = leftover_dependency_map
    super(CyclicDependencyError, self).__init__()

def order_by_dependency(dependency_map):
  """Topologically sorts the keys of a map so that dependencies appear first.

  Uses Kahn's algorithm:
  https://en.wikipedia.org/wiki/Topological_sorting#Kahn's_algorithm

  Args:
    dependency_map: a dict mapping values to a list of dependencies (other keys
      in the map). All keys and dependencies must be hashable types.

  Returns:
    A sorted array of keys from dependency_map.

  Raises:
    CyclicDependencyError: if there is a cycle in the graph.
    ValueError: If there are values in the dependency map that are not keys in
      the map.
  """
  # Maps trackables -> trackables that depend on them. These are the edges used
  # in Kahn's algorithm.
  reverse_dependency_map = collections.defaultdict(set)
  for x, deps in dependency_map.items():
    for dep in deps:
      reverse_dependency_map[dep].add(x)

  # Validate that all values in the dependency map are also keys.
  unknown_keys = reverse_dependency_map.keys() - dependency_map.keys()
  if unknown_keys:
    raise ValueError("Found values in the dependency map which are not keys: "
                     f"{unknown_keys}")

  # Generate the list sorted by objects without dependencies -> dependencies.
  # The returned list will reverse this.
  reversed_dependency_arr = []

  # Prefill `to_visit` with all nodes that do not have other objects depending
  # on them.
  to_visit = [x for x in dependency_map if x not in reverse_dependency_map]

  while to_visit:
    x = to_visit.pop(0)
    reversed_dependency_arr.append(x)
    for dep in set(dependency_map[x]):
      edges = reverse_dependency_map[dep]
      edges.remove(x)
      if not edges:
        to_visit.append(dep)
        reverse_dependency_map.pop(dep)

  if reverse_dependency_map:
    leftover_dependency_map = collections.defaultdict(list)
    for dep, xs in reverse_dependency_map.items():
      for x in xs:
        leftover_dependency_map[x].append(dep)
    raise CyclicDependencyError(leftover_dependency_map)

  return reversed(reversed_dependency_arr)

_ESCAPE_CHAR = "."  # For avoiding conflicts with user-specified names.

# Keyword for identifying that the next bit of a checkpoint variable name is a
# slot name. Checkpoint names for slot variables look like:
#
#   <path to variable>/<_OPTIMIZER_SLOTS_NAME>/<path to optimizer>/<slot name>
#
# Where <path to variable> is a full path from the checkpoint root to the
# variable being slotted for.
_OPTIMIZER_SLOTS_NAME = _ESCAPE_CHAR + "OPTIMIZER_SLOT"
# Keyword for separating the path to an object from the name of an
# attribute in checkpoint names. Used like:
#   <path to variable>/<_OBJECT_ATTRIBUTES_NAME>/<name of attribute>
OBJECT_ATTRIBUTES_NAME = _ESCAPE_CHAR + "ATTRIBUTES"

# A constant string that is used to reference the save and restore functions of
#  Trackable objects that define `_serialize_to_tensors` and
# `_restore_from_tensors`. This is written as the key in the
# `SavedObject.saveable_objects<string, SaveableObject>` map in the SavedModel.
SERIALIZE_TO_TENSORS_NAME = _ESCAPE_CHAR + "TENSORS"

def escape_local_name(name):
  # We need to support slashes in local names for compatibility, since this
  # naming scheme is being patched in to things like Layer.add_variable where
  # slashes were previously accepted. We also want to use slashes to indicate
  # edges traversed to reach the variable, so we escape forward slashes in
  # names.
  return (name.replace(_ESCAPE_CHAR, _ESCAPE_CHAR + _ESCAPE_CHAR).replace(
      r"/", _ESCAPE_CHAR + "S"))

def object_path_to_string(node_path_arr):
  """Converts a list of nodes to a string."""
  return "/".join(
      (escape_local_name(trackable.name) for trackable in node_path_arr))

def checkpoint_key(object_path, local_name):
  """Returns the checkpoint key for a local attribute of an object."""
  key_suffix = escape_local_name(local_name)
  if local_name == SERIALIZE_TO_TENSORS_NAME:
    # In the case that Trackable uses the _serialize_to_tensor API for defining
    # tensors to save to the checkpoint, the suffix should be the key(s)
    # returned by `_serialize_to_tensor`. The suffix used here is empty.
    key_suffix = ""

  return f"{object_path}/{OBJECT_ATTRIBUTES_NAME}/{key_suffix}"

def extract_object_name(key):
  """Substrings the checkpoint key to the start of "/.ATTRIBUTES"."""
  search_key = "/" + OBJECT_ATTRIBUTES_NAME
  return key[:key.index(search_key)]

def extract_local_name(key, prefix=None):
  """Returns the substring after the "/.ATTIBUTES/" in the checkpoint key."""
  # "local name" refers to the the keys of `Trackable._serialize_to_tensors.`
  prefix = prefix or ""
  search_key = OBJECT_ATTRIBUTES_NAME + "/" + prefix
  # If checkpoint is saved from TF1, return key as is.
  try:
    return key[key.index(search_key) + len(search_key):]
  except ValueError:
    return key

def slot_variable_key(variable_path, optimizer_path, slot_name):
  """Returns checkpoint key for a slot variable."""
  # Name slot variables:
  #
  #   <variable name>/<_OPTIMIZER_SLOTS_NAME>/<optimizer path>/<slot name>
  #
  # where <variable name> is exactly the checkpoint name used for the original
  # variable, including the path from the checkpoint root and the local name in
  # the object which owns it. Note that we only save slot variables if the
  # variable it's slotting for is also being saved.

  return (f"{variable_path}/{_OPTIMIZER_SLOTS_NAME}/{optimizer_path}/"
          f"{escape_local_name(slot_name)}")


import tkinter as tk
from tkinter import *
from tkinter import ttk
from tkinter.ttk import *
from googletrans import LANGUAGES
from translate import Translator

root = tk.Tk()

src_text = Text(root, font="arial 10", height=11, wrap=WORD, padx=5, pady=5, width=60)
src_text.place(x=50, y=120)

des_text = Text(root, font="arial 10", height=11, wrap=WORD, padx=5, pady=5, width=60)
des_text.place(x=600, y=120)

class Lang_Translator(Frame):
	def __init__(self, master=None):
		super().__init__(master)
		self.pack()
		self.label()

	src_text = Text(root)
	des_text = Text(root)

	def label(self):
		Label(root,text="Language Translator", font="italic 20").pack(side="top")
		Label(root,text="Select Language", font="arial 15").place(x=30, y=60)
		Label(root,text="Select Language", font="arial 15").place(x=600, y=60)

lang = list(LANGUAGES.values())
src_lang = ttk.Combobox(root, values=lang, width=30)
src_lang.place(x=200, y=63)
src_lang.set("english")

des_lang = ttk.Combobox(root, values=lang, width=30)
des_lang.place(x=770, y=63)
des_lang.set("marathi")

def trans():
	translator2 = Translator(to_lang = des_lang.get())
	translated = translator2.translate(src_text.get(0.0, END))
	des_text.insert(0.0, translated)

def clear_all() :
    des_text.delete(0.0, END)
    src_text.delete(0.0, END)

button_1 = Button(root, text="Translate" , command=trans).place(x=510, y=160)
button_2 = Button(root, text="Clear", command=clear_all).place(x=510, y=220)

app = Lang_Translator(master=root)
app.master.title('Translator')
app.master.geometry('1080x350')
app.mainloop()


"""
E2 Manager (Control, E2 Setup, indication)
"""
import os
import sys
import logging
import time, threading
import json
from ricxappframe.xapp_frame import RMRXapp
from ricsdl.syncstorage import SyncStorage
from ricsdl.exceptions import RejectedByBackend, NotConnected, BackendError
from ricxappframe.rmr import rmr
from influxdb import InfluxDBClient
import tracemalloc
import gc

os.environ['DBAAS_SERVICE_PORT'] = '6379'
os.environ['DBAAS_SERVICE_HOST'] = '10.0.2.15'#'10.244.0.18' #'140.123.102.112'
os.environ['RMR_SEED_RT'] = 'test_route_thrd.rt'
influxdbIP = '10.98.0.0'
influxdbPort = '8086'

count=0
# Constants used in the examples below.
#MY_NS = 'E2setup'
MY_GRP_NS = 'my_group_ns'
MY_LOCK_NS = 'my_group_ns'

set_list = list()
RAN_fun_id_list = list()
Temporary_storage_E2setup = list()
Temporary_storage_Report = list()
Temporary_storage_Insert = list()

def createTimer():
    t = threading.Timer(2, send_broadcast)
    t.start()

def send_broadcast():
    global q
    createTimer()
    size = q.qsize()
    if size > 0:
        pass

# database
def _try_func_return(func):
    try:
        return func()
    except RejectedByBackend as exp:
        #print(f'SDL function {func.__name__} failed: {str(exp)}')
        #raise
        pass
    except (NotConnected, BackendError) as exp:
        pass
        #print(f'SDL function {func.__name__} failed for a temporal error: {str(exp)}')

# initial E2 Manager
def post_init(_self):
    print("***********************************")
    print("E2 Manager born up!!!\n")
    print("E2 Manager start to work~~YA\n")
    print("***********************************\n")
    #createTimerForInflux()

    mysdl = _try_func_return(SyncStorage)
    is_active = mysdl.is_active()
    assert is_active is True
    #print("***********************************")
    #print("E2 Manager store route table from DB\n")
    my_ret_dict = _try_func_return(lambda: mysdl.get('Routing', {str('table')}))
    if(len(my_ret_dict)!=0):
        f = open("test_route_thrd.rt", 'w+')
        f.write('newrt|start\n')
        data = eval(my_ret_dict['table'].decode())
        for key in data.keys():
            ip = data[key]['IP']
            port = str(data[key]['port'])
            messagetype = str(data[key]['massage_type'])+port
            f.write('rte|'+messagetype+'|'+ip+':'+port+'\n')
        f.write('newrt|end\n')
        f.seek(0)
        f.close()
        #print("Write in routing table succussful!!!")

# saving logging
def save_list():
    file_name = 'E2data.txt'
    f = open(file_name, "w+")
    for item in set_list:
        f.write(item)
    f.close()

def get_Requestor_ID(jpay):
    try:
        xApp_port = jpay['RIC Request ID']['RIC Requestor ID']
        return xApp_port
    except:
        #print("There's no RIC ID element")
        return None

""" Procedure Function Section """
# ************************************************

def error_empty_tm(self, data, sbuf):
    #print("****Error occurred****\n")
    #print("The payload is", data, "\n")
    val = json.dumps({
        "RAN Function ID": 1,
        "Cause": "Unspecified",
        "Procedure Code": 2
    }).encode()
    #entry(self, sbuf, val, 7404000)
    self.rmr_send(val, int('7404000'))
# Check Error Indication(xApp)
def error_empty_xapp(self, data, xApp_port):
    #print("****Error occurred****\n")
    #print("The payload is empty\n")

    val = json.dumps(data).encode()

    self.rmr_send(val, int("740"+str(xApp_port)))

# E2 Setup Failure
def e2_setup_failure(self, sbuf, SocketFD):
    #print("E2 Setup Failure!!\n")
    #print("Send 204 back to E2 Termination\n")
    val = json.dumps({
        "Cause": "Unspecified",
        "Procedure Code": 1,
        "Type of Message": 2,
        "SocketFD": SocketFD
    }).encode()
    #entry(self, sbuf, val, 2044000)
    self.rmr_send(val, int('2004000'))

# **************************************************

# Get SocketFD
def get_SocketFD(jpay):
    try:
        SocketFD = jpay['SocketFD']
        return SocketFD
    except:
        #print("There's no SocketFD element")
        return None

# Check Failure
def failure_check(self, sbuf, data, procedure, SocketFD):

    if procedure == "setup":
        if data['E2 Node ID'] == "" or data['RAN Function ID'] == "" or data['RAN Function Definition'] == "" or data['RAN Function Revision'] == "":
            e2_setup_failure(self, sbuf, SocketFD)
            return False

        for i in range(len(RAN_fun_id_list)):
            if data['RAN Function ID'] == RAN_fun_id_list[i]:
                e2_setup_failure(self, sbuf, SocketFD)
                return False

        return True

    elif procedure == "control":
        check_SocketFD = -1

        if data['RAN Function ID'] == "" or data['RIC Control Header'] == "" or data['RIC Control Message'] == "":
            error_empty_xapp(self, data, get_Requestor_ID(data))
            return False, check_SocketFD
        """else:
            return True, check_SocketFD"""

        # check SockFD's correctness
        mysdl = _try_func_return(SyncStorage)
        is_active = mysdl.is_active()
        assert is_active is True

        xApp_ID = int(data['RIC Request ID']['RIC Requestor ID'])
        group_index = str((xApp_ID - 8000)//66)
        warn_msg = 'This port is irregularity.'
        white_text = eval(_try_func_return(lambda: mysdl.get('White', {str('text')}))['text'].decode())
        try:
            white_text = white_text[group_index]
            #print('group_index:', group_index, ' white_text: ', white_text)
            if(xApp_ID not in white_text['listport']):
                #print('warn_msg 1: ', warn_msg) # This port is not in the group
                return False, check_SocketFD
            if(data['SocketFD'] != white_text['SocketFD']): # The SocketFD is wrong
                check_SocketFD = white_text['SocketFD']
        except:
            #print('warn_msg 2: ', warn_msg) # This port is not in the white text table
            return False, check_SocketFD
        return True, check_SocketFD

# write in local routing table
def write_routing_table(NameSpace, Key):
    mysdl = _try_func_return(SyncStorage)
    is_active = mysdl.is_active()
    assert is_active is True

    #print("Write xApp information into routing table!!!\n")

    f = open("test_route_thrd.rt", 'w+')
    f.write('newrt|start\n')
    data = eval(_try_func_return(lambda: mysdl.get(
        NameSpace, {str(Key)}))[Key].decode())
    for key in data.keys():
        ip = data[key]['IP']
        port = str(data[key]['port'])
        messagetype = str(data[key]['massage_type'])+port
        f.write('rte|'+messagetype+'|'+ip+':'+port+'\n')
    f.write('newrt|end\n')
    f.seek(0)
    # txt=f.read()
    # #print(txt)
    f.close()
    #print("Write in routing table succussful!!!")
    #print("Update after 60s.\n")

def Request_Report(self, RIC_ID, SocketFD, xappIP):
    # xApp needs to get report data from E2 manager.
    # connect to influxdb
    client = InfluxDBClient(influxdbIP, influxdbPort, 'root', '', 'reportdb')
    tables = 'report_' + str(RIC_ID) + str(SocketFD)

    # get data from influxdb
    result = client.query('select * from ' + tables)
    ##print("influxdb data:", list(result.get_points()))
    report_data = list(result.get_points())
    # send to xApp
    val = json.dumps(
        {"ACK": RIC_ID, "Report Data": report_data}).encode()

    try:
        #print("870 Time:",time.ctime())
        #print("sdl ip:",RIC_ID)
        hostname = xappIP
        response = os.system("ping -c 1 " + hostname)
        #and then check the response...
        if response == 0:
            #print("UP Time:",time.ctime())
            self.rmr_send(val, int("871"+str(RIC_ID)))
        else:
            pass
            #print('down Time',time.ctime())
    except:
        #print("870 else")
        pass

    #self.rmr_send(val, int('871'+str(RIC_ID)))

# Send to E2 termination
def Send2E2T(self, summary, sbuf):
    mysdl = _try_func_return(SyncStorage)
    is_active = mysdl.is_active()
    assert is_active is True

    jpay = json.loads(summary['payload'])
    message = str(summary['message type'])
    message = int(message[0:3])
    SocketFD = get_SocketFD(jpay)

    # Control Request
    if message == 610:
        """
        mysdl = _try_func_return(SyncStorage)
        is_active = mysdl.is_active()
        assert is_active is True
        """
        if len(str(SocketFD))<5:
            pass
        else:
            #self.logger.info("E2 Manager 610 handler called!")
            print("Control Request: ", time.time())
            #print("Got the 610 Control Request!!\n")
            check, check_SocketFD = failure_check(self, sbuf, jpay, "control", SocketFD)
            if(check_SocketFD != -1):
                jpay['SocketFD'] = check_SocketFD
            if check:
                jpay['SocketFD']= int(_try_func_return(lambda: mysdl.get(str(jpay['SocketFD']), {str('SocketFD')}))['SocketFD'].decode())
                #print("610 payload:", jpay, "\n")
                val = json.dumps(jpay).encode()
                self.rmr_send(val, int('6004000'))
                #entry(self, sbuf, val, 6004000)
                #print("Send it\n")

    self.rmr_free(sbuf)
"""
def report_thread(self, xappID, xApp_port)
    t = threading.Timer(1, send_broadcast)
    t.start()
"""
"""
def report_thread_function(self, xappID, xApp_port, val):
    try:
        #print("Time:",time.ctime())
        #print("sdl ip:",xappID)
        hostname = xappID
        response = os.system("ping -c 1 " + hostname)

        #and then check the response...
        if response == 0:
            #print("UP Time:",time.ctime())
            self.rmr_send(val, int("700"+str(xApp_port)))
        else:
            pass
            #print('down Time',time.ctime())

    except:
        #print("else")
        pass

    #global q
    #createTimer()
    #size = q.qsize()
    #if size > 0:
    #    new = RMRXapp(default_handler=Routing, rmr_port=4601, post_init=send, use_fake_sdl=False)
    #    new.run()
"""

global eport_time_before
eport_time_before = 0
sum_report_time=0
report_time=0
# Send to xApp
def Send2xApp(self, summary, sbuf):
    global xApp_port,sum_report_time,report_time
    """global count"""
    mysdl = _try_func_return(SyncStorage)
    is_active = mysdl.is_active()
    assert is_active is True

    #tracemalloc.start()

    jpay = summary['payload']
    message = str(summary['message type'])
    message = int(message[0:3])
    print("700 jpay:",jpay)
    data = jpay.decode('utf-8')

    if data != "":
        #print("Data is OK\n")
        payload = data[data.index("{") + 1:data.rindex("}")]
        payload = "{" + payload + "}"
        #print("data:", payload, "type:", type(payload))
        #print("##message:",message)
        #print("\njpay",jpay)

        dataSize = sys.getsizeof(jpay)
        jpay = json.loads(payload)

        xApp_port = get_Requestor_ID(jpay)

        ##print("jpay['Procedure Code']:",jpay['Procedure Code'])
        ##print("jpay['RIC Indication Message']:",jpay['RIC Indication Message)'])

        #dataSize = sys.getsizeof(jpay)
        #jpay = json.loads(payload)

        #xApp_port = get_Requestor_ID(jpay)

        # Control Response
        if message == 601:
            #self.logger.info("E2 Manager 601 Response!")
            print("Receive Control ACK: ", time.time())
            #print("Got the 601 Control Response (ACK)!!\n")
            #print("601 Payload:", jpay, "\n")
            jpay = jpay['RIC Control Status']
            if jpay == 0:
                jpay='Success'
            else:
                jpay='False'
            """
            xApp_ID = int(jpay['RIC Request ID']['RIC Requestor ID'])
            group_index = str((xApp_ID - 8000)//66)
            white_text = eval(_try_func_return(lambda: mysdl.get('White', {str('text')}))['text'].decode())
            #RequestFD = white_text[group_index]['SocketFD']
            jpay['SocketFD']= white_text[group_index]['SocketFD']
            """
            val = json.dumps(jpay).encode()
            #print("601 Payload val#@:",val)

            messagetype = int("611"+str(xApp_port))
            self.rmr_send(val, messagetype)
            print("Send Control ACK: ", time.time())

        # Control Failure
        if message == 604:
            self.logger.info("E2 Manager 604 Control Failure!")
            #print("Got the 604 Control Failure!!\n")
            #print("604 Received:", jpay, "\n")

            #print("Send 614 Control Failure to xApp")
            val = json.dumps(jpay).encode()
            self.rmr_send(val, int("614"+str(xApp_port)))

        # Indication Report
        if message == 700:
#            global report_time_before
#            try:
#                report_time_after = time.time()
#                #print("report_time_after is OK !!!")
#                report_time = report_time_after - report_time_before
#                #print("report_time is OK !!!")
#                report_time_before = report_time_after
#                #print("report_time_before is OK !!!")
#            except:
#                print("Enter except")
#                report_time_before = time.time()
#                report_time = 0
            #if count==250:
            #os.execl(sys.executable, sys.executable, *sys.argv)
#            self.logger.info("\n\nGot 700 Indication Report! \nTime: " + str(report_time))
#            print("Reporting period: ", report_time)
            #print("Got the 700 Indication Report!!\n")
            #print("Receive report: ", time.time())
            Receive_report = time.time()
            #print("700 Received:", jpay, "\n")
            #print('Size of integer:', dataSize)

            """val = json.dumps({
                "RIC Requestor ID": jpay['RIC Request ID']['RIC Requestor ID'],
                "paysize": dataSize
            }).encode()

            self.rmr_send(val, 67006000)
            Temporary_storage_Report.append(jpay)"""
            #report_influxdb = dbreport(jpay, message)
            store_thread = threading.Thread(target=dbreport, args=(jpay, message,))
            store_thread.start()
            #print("Send 700 Indication Report to xApp")
            InicationMs=jpay['RIC Indication Message']
            #InicationMs.update({'Time':time.time()})
            val = json.dumps(InicationMs).encode()

            #my_ret_dict = _try_func_return(lambda: mysdl.get(str('connecter'), {str(xApp_port)}))
            ##xappID = list(_try_func_return(lambda: mysdl.get_members('connecter', str(jpay['RIC Request ID']['RIC Requestor ID']))))
            #xappID = self.sdl_get('connecter', str(jpay['RIC Request ID']['RIC Requestor ID'])) # '10.0.2.15'

            #report_thread(self, xappID, xApp_port)
            """
            report_thread = threading.Timer(1, report_thread_function)
            report_thread.start()
            """
#            report_thread = threading.Thread(target=report_thread_function, args=(self, xappID, xApp_port, val))
#            report_thread.start()
            self.rmr_send(val, int("700"+str(xApp_port)))

            report_time = report_time + 1
            sum_report_time = sum_report_time+time.time()-Receive_report
            #print("Send report: ", time.time())
            print("average Receive_report->Send report: ", sum_report_time/report_time)
            del val
            gc.collect()
            #collect_stats = gc.get_stats()
            #print("collect stats = ", collect_stats)

            store_thread.join()
            """
            try:
                #print("Time:",time.ctime())
                #print("sdl ip:",xappID)
                hostname = xappID
                response = os.system("ping -c 1 " + hostname)
                #and then check the response...
                if response == 0:
                    #print("UP Time:",time.ctime())
                    self.rmr_send(val, int("700"+str(xApp_port)))
                else:
                    pass
                    #print('down Time',time.ctime())

            except:
                #print("else")
                pass
            """
            #self.rmr_send(val, int("700"+str(xApp_port)))
            #print("@@@count:",count)
            """count +=1"""
        # Indication Insert
        if message == 710:
            self.logger.info("Got 710 Indication Insert!")
            #print("Got 710 Indication Insert!!\n")
            #print("710 Received:", jpay, "\n")

            """val = json.dumps({
                "RIC Requestor ID": jpay['RIC Request ID']['RIC Requestor ID'],
                "paysize": dataSize
            }).encode()

            self.rmr_send(val, 67106000)
            Temporary_storage_Insert.append(jpay)"""
            #report_influxdb = dbreport(jpay, message)

            #print("Send 710 Indication Insert to xApp")
            val = json.dumps(jpay).encode()
            self.rmr_send(val, int("710"+str(xApp_port)))

        # Indication Insert
        if message == 720:
            self.logger.info("Got 720 E2 Node Disconnected!")
            #print("Got 720 E2 Node Disconnected!!\n")
            #print("720 Received:", jpay, "\n")
            val = json.dumps(jpay).encode()
            self.rmr_send(val, int("720"+str(xApp_port)))

        # Error Indication
        if message == 740:
            self.logger.info("Got 740 Error Indication !")
            #print("Got 740 Error Indication !!\n")
            #print("740 Received:", jpay, "\n")

            #print("Send 740 Error Indication to xApp")
            val = json.dumps(jpay).encode()

            self.rmr_send(val, int("710"+str(xApp_port)))

        # Report Query
        if message == 870:
            self.logger.info("Got 870 Report Query!")
            #print("Got 870 Report Query!!\n")
            #print("870 Received:", jpay, "\n")
            #SocketFD = jpay['SocketFD']
            xApp_port_SPC = jpay['RIC Requestor ID']#xApp_ID
            xappIP = self.sdl_get('connecter', str(jpay['RIC Requestor ID']))

            """
            group_index = str((xApp_port_SPC - 8000)//66)
            white_text = eval(_try_func_return(lambda: mysdl.get('White', {str('text')}))['text'].decode())
            SocketFD = white_text[group_index]['SocketFD']
            """

            #query_report_thread = threading.Thread(target=query_report_thread_function, args=(self, xApp_port_SPC, jpay))
            #query_report_thread.start()

            try:
                SocketFD = _try_func_return(lambda: mysdl.get(jpay['SocketFD'], {str('SocketFD')}))['SocketFD'].decode()
                Request_Report(self, xApp_port_SPC, SocketFD, xappIP)
            except:
                pass
                #print("\n")

        self.rmr_free(sbuf)

    else:
        error_empty_tm(self, "empty", sbuf)
    """
    snapshot = tracemalloc.take_snapshot()
    top_stats = snapshot.statistics('lineno')

    for stat in top_stats:
        print(stat)

    tracemalloc.stop()
    """

# Default Setting
def e2_process(self, summary, sbuf):
    jpay = summary['payload']
    message = str(summary['message type'])
    message = int(message[0:3])
    data = jpay.decode('utf-8')

    if data != "":
        #print("Data is OK\n")
        payload = data[data.index("{") + 1:data.rindex("}")]
        payload = "{" + payload + "}"
        dataSize = sys.getsizeof(jpay)
        jpay = json.loads(payload)
        SocketFD = get_SocketFD(jpay)

        # Recieve E2 setup request --> Response 201
        if message == 200:
            mysdl = _try_func_return(SyncStorage)
            is_active = mysdl.is_active()
            assert is_active is True

            #self.logger.info("Got 200 E2 Setup Request!")
            #print("Got 200 E2 Setup Request!!")
            print("200 Received:", jpay, "\n")
            #print("payload length:", len(data))
            #print('size of integer:', sys.getsizeof(jpay))
            dataSize = sys.getsizeof(data)

            #check = failure_check(self, sbuf, jpay, "setup", SocketFD)
            check = True
            if check:
                val = json.dumps({
                    "E2 Node ID": jpay['E2 Node ID'].replace(' ',''),
                    "paysize": dataSize,
                    "SocketFD":jpay['SocketFD']
                }).encode()
                #print("Send 6200 to RIC Redis Manager\n")
                self.rmr_send(val, 62006000)
                Temporary_storage_E2setup.append(jpay)
                #RAN_fun_id_list.append(jpay['RAN Function ID'])

        # Error indication
        elif message == 740:
            self.logger.info("Got 740 Error Indication!")
            #print("Got 740 Error Indication!\n")
            #print("740 Received:", jpay, "\n")

        # Service Update
        elif message == 800:
            self.logger.info("Got 800 Service Update!")
            #print("Got 800 Service Update!\n")
            #print("800 Received:", jpay, "\n")

            try:
                mode = jpay["RAN Functions Added"]
            except:
                try:
                    mode = jpay["RAN Functions Modified"]
                except:
                    mode = jpay["RAN Functions Deleted"]

            #print("Mode", mode, "\n")

            val = json.dumps(jpay).encode()
            #entry(self, sbuf, val, 8014000)
            self.rmr_send(val, int('8014000'))
        #Restart E2 manager
        elif message == 100:
            self.logger.info("Received the 1000 Routing Restart!!")
            #print("Received the 1000 Routing Restart!!\n")
            #print("1000 Received:", jpay, "\n")

            NS = jpay['Routing Table']['Namespace']
            key1 = jpay['Routing Table']['Key']

            write_routing_table(NS, key1)

        else:
            self.logger.info("Default Callback!")
            #print("Default handler received: {0}".format(summary))

        self.rmr_free(sbuf)

    else:
        #print("Received: {0}".format(summary))
        error_empty_tm(self, "empty", sbuf)
        self.rmr_free(sbuf)

#Send to E2 Termination
def entry(self, sbuf, data, message):

    # Init rmr
    mrc = rmr.rmr_init(b"9998", rmr.RMR_MAX_RCV_BYTES, 0x00)
    while rmr.rmr_ready(mrc) == 0:
        time.sleep(1)
        #print("Not yet ready")
    rmr.rmr_set_stimeout(mrc, 2)
    sbuf = rmr.rmr_alloc_msg(mrc, 1024)
    val=data

    try:
        val = data.encode('utf-8')

    except:
        val = data

    #print("val control request:",val)
    rmr.set_payload_and_length(val, sbuf)
    rmr.generate_and_set_transaction_id(sbuf)
    sbuf.contents.state = 0
    sbuf.contents.mtype = message
    #print("Send summary: {}".format(rmr.message_summary(sbuf)))
    sbuf = rmr.rmr_send_msg(mrc, sbuf)
    #print("Post send summary: {}".format(rmr.message_summary(sbuf)))
    #print("\n")

# Store the data into database
def db(self, summary, sbuf):
    mysdl = _try_func_return(SyncStorage)
    is_active = mysdl.is_active()
    assert is_active is True

    jpay = json.loads(summary['payload'])
    message = summary['message type']
    #print("Got the 6201 message!!!!!!!!!\n")
    #print("6201 Received:", jpay)

    # Wrong message
    if((message % 10) == 4):
        if(jpay['NAK'] == -2):
            #print("There is something wrong. Please check the RMR message type.")
            os._exit(0)
        elif(jpay['NAK'] == -1):
            #print("The memoory is full. Please wait.")
            os._exit(0)

    MY_NS = jpay['namespace']
    #print("MY_NS:", MY_NS)
    #_try_func_return(lambda: mysdl.set(str(jpay["E2 Node ID"].encode()), {'SocketFD': str(jpay["SocketFD"].encode())}))

    # Store data in redis
    try:
        tmpdata = Temporary_storage_E2setup.pop(0)
    except:
        #print("The Temporary_storage_E2setup is empty")
        return
    _try_func_return(lambda: mysdl.set(str(tmpdata["E2 Node ID"]).replace(' ','') , {'SocketFD': str(tmpdata["SocketFD"]).encode()}))
    #print("Send 201 back to E2 Termination\n")

    val = json.dumps({
        "Global RIC ID": 1,
        "RAN Function ID": tmpdata['RAN Function ID'],
        "Procedure Code": 1,
        "Type of Message": 1,
        "SocketFD":  tmpdata['SocketFD']
    }).encode()
    #print("to E2_agent val:",val)
    #entry(self, sbuf, val, 2014000)
    self.rmr_send(val, int('2014000'))
    print("to E2_agent val:",val)

    storedata = {}
    for col in jpay["columns"]:
        try:
            storedata.update({str(col): tmpdata[col]})
        except:
            pass
    storedata.update({'xAppID': 99999})
    storedata.update({"SocketFD":storedata['E2 Node ID'].replace(' ','')})
    store_data = str(storedata)
    byte_send_data = store_data.encode()
    _try_func_return(lambda: mysdl.set(
        MY_NS, {str(storedata['E2 Node ID'].replace(' ','')): byte_send_data}))
    #print("Store in Redis\n")

    # Get data in redis
    my_ret_dict = _try_func_return(lambda: mysdl.get(
        MY_NS, {str(storedata['E2 Node ID'].replace(' ',''))}))
    #print(MY_NS, " redis_get:", str(storedata['E2 Node ID'].replace(' ','')), "->", my_ret_dict)

    self.rmr_free(sbuf)

store_time=0
sum_time=0
#Report (database)
def dbreport(jpay, message):
    global client, store_time, sum_time
    start_ = time.time()
    print("**** Enter the dbreport ****")
    if(message == 700):
        #print("700 enter dbreport:", jpay)
        MY_NS = 'report'
        store_columns = ['RIC Request ID', 'RIC Instance ID', 'RAN Function ID', 'RIC Action ID', 'RIC Indication Type', 'RIC Indication Header', 'RIC Indication Message']
    elif(message == 710):
        #print("710 enter dbreport:", jpay)
        MY_NS = 'insert'
        store_columns = ['RIC Request ID', 'RIC Instance ID', 'RAN Function ID', 'RIC Action ID', 'RIC Indication Type', 'RIC Indication Header', 'RIC Indication Message', 'Call process ID']
    #print("MY_NS:", MY_NS)

    storedata = {}
    for col in store_columns:
        try:
            storedata.update({str(col): jpay[col]})
        except:
            pass

    store_data = str(storedata)
    #print("Store Data", storedata)

    # please decision that these still send to xApp? orange words under this line.
    """
    val = json.dumps({"ACK": storedata['RIC Request ID']['RIC Requestor ID'],
                      "RIC Requestor ID": storedata['RIC Request ID']['RIC Requestor ID'],
                      "namespace": MY_NS}).encode()
    self.rmr_send(val, 67006000)
    """

    # store data in influxdb

    #client = InfluxDBClient(influxdbIP, influxdbPort, 'root', '', 'reportdb')
    #print("database: ", client.get_list_database())  # 顯示所有數據庫名稱
    # client.create_database('testdb') # 創建數據庫
    # #print(client.get_list_database()) # 顯示所有數據庫名稱
    """
    retention_policy = 'exist_Time'
    try:
        client.create_retention_policy(retention_policy, '1h', 3, default=True)
    except:
        #print("Error:not successful create new retention policy.")
    """
    FD = jpay['SocketFD']
    tables = 'report_'+str(storedata['RIC Request ID']['RIC Requestor ID'])# + str(FD)

    # This block is for test
    #latency : ms, reliability : percentage, throughput : Gbps
    # procedure": "Indication Report"

    #print("storedata['RIC Request ID']['RIC Requestor ID']:",storedata['RIC Request ID']['RIC Requestor ID'])
    #print("type storedata['RIC Request ID']['RIC Requestor ID']:",storedata['RIC Request ID']['RIC Requestor ID'])
    json_body = []

    json_body.append(str(tables)+",procedure="+str(MY_NS)+" RIC_Requestor_ID="+str(storedata['RIC Request ID']['RIC Requestor ID'])+",RIC_Instance_ID="+str(storedata['RIC Request ID']['RIC Instance ID'])+",RAN_Function_ID="+str(storedata['RAN Function ID'])+",RIC_Action_ID="+str(storedata['RIC Action ID'])+",RIC_Indication_Type="+str(storedata['RIC Indication Type'])+",RIC_Indication_Header="+str(storedata['RIC Indication Header'])+",RIC_Indication_Message="+str(storedata['RIC Indication Message']))
    """
    json_body.append("{measurement},procedure={procedure} RIC_Requestor_ID={RIC_Requestor_ID},RIC_Instance_ID={RIC_Instance_ID},RAN_Function_ID={RAN_Function_ID},RIC_Action_ID={RIC_Action_ID},RIC_Indication_Type={RIC_Indication_Type},RIC_Indication_Header={RIC_Indication_Header},RIC_Indication_Message={RIC_Indication_Message}"
                .format(measurement='tables',
                        procedure='MY_NS',
                        RIC_Requestor_ID=int(storedata['RIC Request ID']['RIC Requestor ID']),
                        RIC_Instance_ID=int(storedata['RIC Request ID']['RIC Instance ID']),
                        RAN_Function_ID=int(storedata['RAN Function ID']),
                        RIC_Action_ID=int(storedata['RIC Action ID']),
                        RIC_Indication_Type=int(storedata['RIC Indication Type']),
                        RIC_Indication_Header=int(storedata['RIC Indication Header']),
                        RIC_Indication_Message=str(storedata['RIC Indication Message'])))
    """
    client.write_points(json_body, batch_size=5000, protocol='line',retention_policy=retention_policy)

    end_ = time.time()
    store_time = store_time + 1
    sum_time = sum_time + end_- start_
    print("average storedb time = ", sum_time/store_time)
    #print("-----------------store in influxdb------------------")
    #result = client.query('select * from ' + tables)
    #print("influxdb data:", list(result.get_points()))
    """
    # please decision that these still send to xApp? orange words under this line.
    if message == 6701:
        #print("Send 700 Indication Report to xApp")
        self.rmr_send(val, int("700"+str(xApp_port)))

    elif message == 6711:
        #print("Send 710 Indication Insert to xApp")
        self.rmr_send(val, int("710"+str(xApp_port)))
    return list(result.get_points())
    """

client = InfluxDBClient(influxdbIP, influxdbPort, 'root', '', 'reportdb')
client.create_database('reportdb')  # 創建數據庫
retention_policy = 'exist_Time'
try:
    client.create_retention_policy(retention_policy, '1h', 3, default=True)
except:
    pass
    #print("Error:not successful create new retention policy.")

xapp = RMRXapp(default_handler=e2_process, rmr_port=4200,post_init=post_init, use_fake_sdl=False)

#--------------------------------------------
xapp.register_callback(Send2E2T, 6104200)  # control request
xapp.register_callback(Send2xApp, 6014200)  # control response
xapp.register_callback(Send2xApp, 6044200)  # control failure
#--------------------------------------------
xapp.register_callback(Send2xApp, 7004200)  # report
xapp.register_callback(Send2xApp, 7104200)  # insert
#--------------------------------------------
xapp.register_callback(Send2xApp, 7004200)  # report
xapp.register_callback(Send2xApp, 7204200)  # disconnected
xapp.register_callback(Send2xApp, 8704200)  # Report Query
#--------------------------------------------
xapp.register_callback(db, 62014200)  # db
xapp.register_callback(db, 62044200)  # db
xapp.register_callback(db, 66014200)  # db
xapp.register_callback(db, 66044200)  # db
#--------------------------------------------
xapp.register_callback(dbreport, 67014200)  # dbreport
xapp.register_callback(dbreport, 67044200)  # dbreport
xapp.register_callback(dbreport, 67114200)  # dbreport
xapp.register_callback(dbreport, 67144200)  # dbreport

xapp.run()  # will not thread by default


import numpy as np
import keras
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation

from matplotlib import pyplot as plt
import csv
from itertools import islice

K = 15

Learn_size = 210
test_size = 40
predict_size = 40

data = []
num_examples = 0
ile = 0

X_train = []
y_train = []

X_test = []
y_test = []

licznik = 0

# load train data
for i in range(K):
    points = []
    with open('dane/dane%d.txt' %i) as file:
        reader = csv.reader(file, delimiter=',', skipinitialspace=True)
        for row in islice(reader,0,Learn_size):
            X_train.append([float(row[0]), float(row[1])])
            y_train.append(i)
            licznik = licznik + 1
    data.append(np.asarray(points))

# load test data

for i in range(K):
    with open('dane/dane%d.txt' %i) as file:
        reader = csv.reader(file, delimiter=',', skipinitialspace=True)
        for row in islice(reader,Learn_size,Learn_size + test_size):
            X_test.append([float(row[0]), float(row[1])])
            y_test.append(i)
            licznik = licznik + 1

X = []
Y = []

# load data to predict
for i in range(K):
    with open('dane/dane%d.txt' %i) as file:
        reader = csv.reader(file, delimiter=',', skipinitialspace=True)
        for row in islice(reader,Learn_size + test_size ,Learn_size + test_size +  predict_size ):
            X.append([float(row[0]), float(row[1])])
            licznik = licznik + 1

X_train /= np.std(X_train, axis = 0)
X_test /= np.std(X_test, axis = 0)
X /= np.std(X, axis = 0)

X_test = np.asarray(X_test)
X_train = np.asarray(X_train)
X = np.asarray(X)

y_test = np.asarray(y_test)
y_train = np.asarray(y_train)

model = Sequential()

model.add(Dense(32, activation='relu', input_dim=2))
model.add(Dense(64, activation='relu'))
model.add(Dense(K, activation='softmax'))

model.compile(optimizer='adam',
              loss='categorical_crossentropy',
              metrics=['accuracy'])

Y_train = keras.utils.to_categorical(y_train, K)
Y_test = keras.utils.to_categorical(y_test, K)

model.fit(X_train, Y_train, epochs=16, batch_size=16)

score = model.evaluate(X_test,Y_test, batch_size=32)

print(score)

separated = []

for i in range(K * predict_size):
    value = np.array(X[i]).reshape((1, 2))
    result = model.predict(value)
    predicted_cluster = np.argmax(result)
    temp = [predicted_cluster,X[i]]
    separated.append(temp)

#print(separated)
new_group = []
all = []

for k in range(K):
    new_group = []
    zliczanie = 0
    zaden = True
    for i in range(K * predict_size):
        if separated[i][0] == k:
            new_group.append(X[i])
            zliczanie = zliczanie + 1
            zaden = False
    print(" DLa klastra :", k, " znalazlem :", zliczanie, " punktow.")
    if zaden:
        new_group.append([0,0])
    all.append(np.asarray(new_group))

all = np.asarray(all)

#print(all)

for i in range(K):
    plt.plot(all[i][:][:,0],all[i][:][:,1], '.')

plt.show()