import datetime
import logging
from enum import Enum
from typing import List, Optional

from hikaru.model.rel_1_26 import Pod, PodList
from robusta.api import (
    ActionException,
    BaseBlock,
    ErrorCodes,
    KubernetesResourceEvent,
    MarkdownBlock,
    action,
    build_selector_query,
    get_crash_report_blocks,
    get_image_pull_backoff_blocks,
    get_job_all_pods,
    get_pending_pod_blocks,
    parse_kubernetes_datetime_to_ms,
)

class PodIssue(str, Enum):
    ImagePullBackoff = "ImagePullBackoff"
    Pending = "Pending"
    CrashloopBackoff = "CrashloopBackoff"
    Crashing = "Crashing"
    NoneDetected = "NoneDetected"

supported_resources = ["Deployment", "DaemonSet", "ReplicaSet", "Pod", "StatefulSet", "Job"]

@action
def pod_issue_investigator(event: KubernetesResourceEvent):
    """
    Enriches alert with a finding that investigates the pods issues
    Note:
        The only supported resources for investigation are "Deployment", "DaemonSet", "ReplicaSet", "Pod", "StatefulSet", "Job"
    """
    resource = event.get_resource()
    if resource.kind not in supported_resources:
        raise ActionException(
            ErrorCodes.RESOURCE_NOT_SUPPORTED, f"Pod investigator is not supported for resource {resource.kind}"
        )

    if resource.kind == "Job":
        job_pods = get_job_all_pods(resource)
        pods = job_pods if job_pods else []
    elif resource.kind == "Pod":
        pods = [resource]
    else:
        # if the kind is Deployment", "DaemonSet", "ReplicaSet", "StatefulSet"
        selector = build_selector_query(resource.spec.selector)
        pods = PodList.listNamespacedPod(namespace=resource.metadata.namespace, label_selector=selector).obj.items

    pods_with_issues = [pod for pod in pods if detect_pod_issue(pod) != PodIssue.NoneDetected]
    if not pods_with_issues:
        logging.info(f"No pod issues discovered for {resource.kind} {resource.metadata.name}")
        return
    # Investigate first issue found
    pod_issue = detect_pod_issue(pods_with_issues[0])
    report_pod_issue(event, pods_with_issues, pod_issue)

def detect_pod_issue(pod: Pod) -> PodIssue:
    if has_image_pull_issue(pod):
        return PodIssue.ImagePullBackoff
    elif is_crashlooping(pod):
        return PodIssue.CrashloopBackoff
    elif had_recent_crash(pod):
        return PodIssue.Crashing
    elif is_pod_pending(pod):
        return PodIssue.Pending
    return PodIssue.NoneDetected

def is_pod_pending(pod: Pod) -> bool:
    return pod.status.phase.lower() == "pending"

def is_crashlooping(pod: Pod) -> bool:
    all_statuses = pod.status.containerStatuses + pod.status.initContainerStatuses
    crashlooping_containers = [
        container_status
        for container_status in all_statuses
        if container_status.state.waiting is not None
        and container_status.restartCount > 1
        and "CrashloopBackOff" in container_status.state.waiting.reason
    ]
    return len(crashlooping_containers) > 0

def timestamp_in_last_15_minutes(timestamp: str) -> bool:
    # most relevant alerts are fired on issues that occurred in the last 15 minutes
    time_ms = parse_kubernetes_datetime_to_ms(timestamp)
    timestamp_happened = datetime.datetime.fromtimestamp(time_ms / 1000)
    now = datetime.datetime.now()
    last_15_minutes = datetime.timedelta(minutes=15)
    return now - timestamp_happened < last_15_minutes

def had_recent_crash(pod: Pod) -> bool:
    # is a pod is crashlooping but currently running it won't have the state.waiting.reason crashloop backoff
    all_statuses = pod.status.containerStatuses + pod.status.initContainerStatuses
    crashing_containers = [
        container_status
        for container_status in all_statuses
        if container_status.lastState.terminated is not None
        and container_status.lastState.terminated.reason == "Error"
        and timestamp_in_last_15_minutes(container_status.lastState.terminated.finishedAt)
    ]
    return len(crashing_containers) > 0

def has_image_pull_issue(pod: Pod) -> bool:
    all_statuses = pod.status.containerStatuses + pod.status.initContainerStatuses
    image_pull_statuses = [
        container_status
        for container_status in all_statuses
        if container_status.state.waiting is not None
        and container_status.state.waiting.reason in ["ImagePullBackOff", "ErrImagePull"]
    ]
    return len(image_pull_statuses) > 0

def report_pod_issue(event: KubernetesResourceEvent, pods: List[Pod], issue: PodIssue):
    # find pods with issues
    pods_with_issue = [pod for pod in pods if detect_pod_issue(pod) == issue]
    pod_names = [pod.metadata.name for pod in pods_with_issue]
    expected_pods = get_expected_replicas(event)
    message_string = f"{len(pod_names)}/{expected_pods} pod(s) are in {issue} state. "
    resource = event.get_resource()
    if resource.kind == "Job":
        message_string = f"{len(pod_names)} pod(s) are in {issue} state. "

    # no need to report here if len(pods) != expected_pods since there are mismatch enrichers

    blocks: List[BaseBlock] = [MarkdownBlock(message_string)]
    # get blocks from specific pod issue
    additional_blocks = get_pod_issue_blocks(pods_with_issue[0])

    if additional_blocks:
        blocks.append(MarkdownBlock(f"\n\n*{pod_names[0]}* was picked for investigation\n"))
        blocks.extend(additional_blocks)
        event.add_enrichment(blocks)

def get_expected_replicas(event: KubernetesResourceEvent) -> int:
    resource = event.get_resource()
    kind = resource.kind
    try:
        if kind == "Deployment" or kind == "StatefulSet":
            return resource.spec.replicas if resource.spec.replicas is not None else 1
        elif kind == "DaemonSet":
            return 0 if not resource.status.desired_number_scheduled else resource.status.desired_number_scheduled
        elif kind == "Pod":
            return 1
        return 0
    except Exception:
        logging.error(f"Failed to extract total pods from {resource}", exc_info=True)
    return 1

def get_pod_issue_blocks(pod: Pod) -> Optional[List[BaseBlock]]:
    if has_image_pull_issue(pod):
        return get_image_pull_backoff_blocks(pod)
    elif is_pod_pending(pod):
        return get_pending_pod_blocks(pod)
    elif is_crashlooping(pod):
        return get_crash_report_blocks(pod)
    elif had_recent_crash(pod):
        return get_crash_report_blocks(pod)
    return None


import tensorflow as tf

x_data = [[1,2,1,1], [2,1,3,2], [3,1,3,4], [4,1,5,5], [1,7,5,5], [1,2,5,6], [1,6,6,6], [1,7,7,7]] # vectors
y_data = [[0,0,1], [0,0,1], [0,0,1], [0,1,0], [0,1,0], [0,1,0], [1,0,0], [1,0,0]] # one hot encoding
X = tf.placeholder(tf.float32, shape=[None, 4])
Y = tf.placeholder(tf.float32, shape=[None, 3])
W = tf.Variable(tf.random_normal([4, 3]))
b = tf.Variable(tf.random_normal([3]))

model_LC = tf.add(tf.matmul(X,W),b)
model = tf.nn.softmax(model_LC)
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=model_LC, labels=Y))
train = tf.train.GradientDescentOptimizer(0.1).minimize(cost)

with tf.Session() as sess:
	sess.run(tf.global_variables_initializer())
	# Training
	for step in range(2001):
		c, _ = sess.run([cost, train], feed_dict={X: x_data, Y: y_data})
		print(step, c)
	# Testing
	test1 = sess.run(model, feed_dict={X: [[1,11,7,9]]})
	print(test1, sess.run(tf.argmax(test1, 1)))


"""Rock component."""
import warnings
import numpy as np
import matplotlib.pyplot as plt

from .base_spatial import SpatialComponent
from .decorators import apply_to_each_input, state_check, ndim_check
from .plot_utils import show_cube_static, show_cube_interactive
from .utils import rolling_window, get_single_path
from .parse_utils import read_ecl_bin

class Rock(SpatialComponent):
    """Rock component of geological model."""

    def _load_ecl_binary(self, path_to_results, attrs, basename, logger=None):
        path = get_single_path(path_to_results, basename + '.INIT', logger)
        if path is None:
            return
        sections = read_ecl_bin(path, attrs, logger=logger)
        for k in ['PORO', 'PERMX', 'PERMY', 'PERMZ']:
            if (k in attrs) and (k in sections):
                setattr(self, k, sections[k])

    @apply_to_each_input
    def _to_spatial(self, attr, dimens, inplace):
        """Spatial order 'F' transformations."""
        return self.reshape(attr=attr, newshape=dimens, order='F', inplace=inplace)

    def _make_data_dump(self, attr, fmt=None, float_dtype=None, **kwargs):
        """Prepare data for dump."""
        if fmt.upper() != 'HDF5':
            return super()._make_data_dump(attr, fmt=fmt, **kwargs)
        data = self.ravel(attr=attr, inplace=False)
        return data if float_dtype is None else data.astype(float_dtype)

    @apply_to_each_input
    def pad_na(self, attr, actnum, fill_na=0., inplace=True):
        """Add dummy cells into the rock vector in the positions of non-active cells if necessary.

        Parameters
        ----------
        attr: str, array-like
            Attributes to be padded with non-active cells.
        actnum: array-like of type bool
            Vector representing a mask of active and non-active cells.
        fill_na: float
            Value to be used as filler.
        inplace: bool
            Modify сomponent inplace.

        Returns
        -------
        output : component if inplace else padded attribute.
        """
        data = getattr(self, attr)
        if np.prod(data.shape) == actnum.size:
            return self if inplace else data
        if data.ndim > 1:
            raise ValueError('Data should be raveled before padding.')
        padded_data = np.full(shape=(actnum.size,), fill_value=fill_na, dtype=float)
        padded_data[actnum.ravel(order='F')] = data
        if inplace:
            setattr(self, attr, padded_data)
            return self
        return padded_data

    @apply_to_each_input
    def strip_na(self, attr, actnum, inplace=True):
        """Remove non-active cells from the rock vector.

        Parameters
        ----------
        attr: str, array-like
            Attributes to be stripped
        actnum: array-like of type bool
            Vector representing mask of active and non-active cells.
        inplace: bool
            Modify сomponent inplace.

        Returns
        -------
        output : component if inplace else stripped attribute.
        """
        if self.state.spatial and inplace:
            raise ValueError('Inplace is not allowed in spatial state.')
        data = self.ravel(attr, inplace=False)
        if data.size == np.sum(actnum):
            return self if inplace else data
        stripped_data = data[actnum.ravel(order='F')]
        if inplace:
            setattr(self, attr, stripped_data)
            return self
        return stripped_data

    def show_histogram(self, attr, actnum=None, **kwargs):
        """Show properties distribution.

        Parameters
        ----------
        attr : str
            Attribute to compute the histogram.
        actnum : array, optional
            Actnum array. If None, all cell are active.
        kwargs : misc
            Any additional named arguments to ``plt.hist``.

        Returns
        -------
        plot : Histogram plot.
        """
        data = getattr(self, attr)
        if actnum is not None:
            data = data * actnum
        plt.hist(data.ravel(), **kwargs)
        plt.show()
        return self

    @state_check(lambda state: state.spatial)
    @ndim_check(3)
    def show_cube(self, attr, x=None, y=None, z=None, actnum=None, figsize=None, **kwargs):
        """Visualize slices of 3D array. If no slice is specified, all 3 slices
        will be shown with interactive slider widgets.

        Parameters
        ----------
        attr : str
            Attribute to show.
        x : int or None, optional
            Slice along x-axis to show.
        y : int or None, optional
            Slice along y-axis to show.
        z : int or None, optional
            Slice along z-axis to show.
        actnum : array, optional
            Actnum array. If None, all cell are active.
        figsize : array-like, optional
            Output plot size.
        kwargs : dict, optional
            Additional keyword arguments for plot.
        """
        data = getattr(self, attr)
        if actnum is not None:
            data = data * actnum
        if np.all([x is None, y is None, z is None]):
            show_cube_interactive(data, figsize=figsize, **kwargs)
        else:
            show_cube_static(data, x=x, y=y, z=z, figsize=figsize, **kwargs)
        return self

    @state_check(lambda state: state.spatial)
    @apply_to_each_input
    def upscale(self, attr, factors, volumes, weights=None):
        """Upscale properties.

        Parameters
        ----------
        attr : str, list of str
            Attributes to be upscaled.
        factors : tuple, int
            Scale factors along each axis. If int, factors are the same for each axis.
        volumes : ndarray, same shape as `attr`
            Cell volumes.
        weights : ndarray, same shape as `attr`, optional
            Cell weights.

        Returns
        -------
        out : ndarray
            Upscaled properties.
        """
        factors = np.atleast_1d(factors)
        if factors.size == 1:
            factors = np.repeat(factors, 3)
        data = getattr(self, attr)
        binned_weights = rolling_window(volumes, factors)
        if weights is not None:
            binned_weights *= rolling_window(weights, factors)
        out = rolling_window(data, factors) * binned_weights
        binned_weights = np.sum(binned_weights, axis=(-3, -2, -1))
        zero_mask = np.isclose(binned_weights, 0)
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            out = np.sum(out, axis=(-3, -2, -1)) / binned_weights
        out[zero_mask] = 0
        return out


# BOJ 1058 친구
# 그래프 탐색
import sys
from collections import deque

input = sys.stdin.readline

def finding_two_friend_bfs(start_node):
    visit = [0]*N
    q = deque([start_node])
    visit[start_node] = 1
    while q:
        node = q.popleft()
        for child in graph[node]:
            if visit[child] == 0:
                q.append(child)
                visit[child] = visit[node] + 1
    cnt = -1
    for i in visit:
        if 0 < i <= 3:
            cnt += 1
    return cnt

N = int(input())
graph = [[] for _ in range(N)]
for parents_idx in range(N):
    info = input().rstrip()
    for child_idx in range(len(info)):
        if info[child_idx] == 'Y':
            graph[parents_idx].append(child_idx)

answer = 0
for person in range(N):
    answer = max(answer, finding_two_friend_bfs(person))
print(answer)


from __future__ import absolute_import

import unittest

from cbus.common import add_cbus_checksum

class CommonTest(unittest.TestCase):

    def test_cbus_checksum(self):
        """Test that adding a cbus checksum works for every byte value."""
        for x in range(256):
            b = bytes([x])
            c = bytes([x, ((x ^ 0xff) + 1) & 0xff])
            self.assertEqual(c, add_cbus_checksum(b), f'bad checksum for {x}')

    def test_empty_checksum(self):
        self.assertEqual(b'\0', add_cbus_checksum(b''))

if __name__ == '__main__':
    unittest.main()


import logging
import random
from pathlib import Path
from typing import Callable

from core.exceptions import MessageTooLongException, BuildingAudioFileTimedOutExeption, UnableToBuildAudioFileException
from core.tts.tts_controller import TTSController
from common.audio_player import AudioPlayer
from common.configuration import Configuration
from common.command_management.invoked_command import InvokedCommand
from common.command_management.invoked_command_handler import InvokedCommandHandler
from common.exceptions import NoVoiceChannelAvailableException, UnableToConnectToVoiceChannelException
from common.logging import Logging
from common.module.module import Cog
from common.message_parser import MessageParser

from discord import app_commands, Interaction, Member
from discord.app_commands import describe
from discord.ext.commands import Bot

## Config & logging
CONFIG_OPTIONS = Configuration.load_config()
LOGGER = Logging.initialize_logging(logging.getLogger(__name__))

class SpeechCog(Cog):
    SAY_COMMAND_NAME = "say"

    def __init__(self, bot: Bot, *args, **kwargs):
        super().__init__(bot, *args, **kwargs)

        self.bot = bot
        self.audio_player_cog: AudioPlayer = kwargs.get('dependencies', {}).get('AudioPlayer')
        assert(self.audio_player_cog is not None)
        self.invoked_command_handler: InvokedCommandHandler = kwargs.get('dependencies', {}).get('InvokedCommandHandler')
        assert(self.invoked_command_handler is not None)
        self.message_parser: MessageParser = kwargs.get('dependencies', {}).get('MessageParser')
        assert(self.message_parser is not None)
        self.tts_controller: TTSController = kwargs.get('dependencies', {}).get('TTSController')
        assert(self.tts_controller is not None)

        self.channel_timeout_phrases = CONFIG_OPTIONS.get('channel_timeout_phrases', [])
        self.audio_player_cog.channel_timeout_handler = self.play_random_channel_timeout_message

        ## Commands
        self.add_command(app_commands.Command(
            name=SpeechCog.SAY_COMMAND_NAME,
            description=self.say_command.__doc__,
            callback=self.say_command
        ))

    ## Methods

    async def play_random_channel_timeout_message(self, server_state, callback):
        '''Channel timeout logic, picks an appropriate sign-off message and plays it'''

        try:
            if (len(self.channel_timeout_phrases) > 0):
                message = random.choice(self.channel_timeout_phrases)
                file_path = await self.build_audio_file(message, True)

                await self.audio_player_cog._play_audio_via_server_state(server_state, file_path, callback)
        except Exception as e:
            LOGGER.exception("Exception during channel sign-off")
            await callback()

    async def build_audio_file(self, text: str, ignore_char_limit = False, interaction: Interaction = None) -> Path:
        '''Turns a string of text into a wav file for later playing. Returns a filepath pointing to that file.'''

        ## Make sure the message isn't too long
        if(not self.tts_controller.check_length(text) and not ignore_char_limit):
            raise MessageTooLongException(f"Message is {len(text)} characters long when it should be less than {self.tts_controller.char_limit}")

        ## Parse down the message before sending it to the TTS service
        if (interaction is not None):
            text = self.message_parser.parse_message(text, interaction.data)

        ## Build the audio file for speaking
        return await self.tts_controller.save(text, ignore_char_limit)

    async def say(
            self,
            text: str,
            author: Member,
            target_member: Member = None,
            ignore_char_limit = False,
            interaction: Interaction = None,
            callback: Callable = None
    ) -> InvokedCommand:
        '''Internal say method, for use with presets and anything else that generates phrases on the fly'''

        async def audio_player_callback():
            self.tts_controller.delete(wav_path)
            if (callback is None):
                return

            await callback()

        try:
            wav_path = await self.build_audio_file(text, ignore_char_limit, interaction)

        except BuildingAudioFileTimedOutExeption as e:
            LOGGER.exception(f"Timed out building audio for message: '{text}'")
            return InvokedCommand(False, e, f"Sorry <@{author.id}>, it took too long to generate speech for that.")

        except MessageTooLongException as e:
            LOGGER.warn(f"Unable to build too long message. Message was {len(text)} characters long (out of {self.tts_controller.char_limit})")
            ## todo: Specify how many characters need to be removed?
            return InvokedCommand(False, e, f"Wow <@{author.id}>, that's waaay too much. You've gotta keep messages shorter than {self.tts_controller.char_limit} characters.")

        except UnableToBuildAudioFileException as e:
            LOGGER.exception(f"Unable to build .wav file for message: '{text}'")
            return InvokedCommand(False, e, f"Sorry <@{author.id}>, I can't say that right now.")

        try:
            await self.audio_player_cog.play_audio(wav_path, author, target_member or author, interaction, audio_player_callback)

        except NoVoiceChannelAvailableException as e:
            LOGGER.error("No voice channel available", exc_info=e)
            if (e.target_member.id == author.id):
                return InvokedCommand(False, e, f"Sorry <@{author.id}>, you're not in a voice channel.")
            else:
                return InvokedCommand(False, e, f"Sorry <@{author.id}>, that person isn't in a voice channel.")

        except UnableToConnectToVoiceChannelException as e:
            ## Logging handled in AudioPlayer

            error_values = []
            if (not e.can_connect):
                error_values.append("connect to")
            if (not e.can_speak):
                error_values.append("speak in")

            return InvokedCommand(False, e, f"Sorry <@{author.id}>, I'm not able to {' or '.join(error_values)} that channel. Check the permissions and try again later.")

        except FileNotFoundError as e:
            LOGGER.error("FileNotFound when invoking `play_audio`", exc_info=e)
            return InvokedCommand(False, e, f"Sorry <@{author.id}>, I can't say that right now.")

        return InvokedCommand(True)

    ## Commands

    @describe(text="The text that Hawking will speak")
    @describe(user="The user that will be spoken to")
    async def say_command(self, interaction: Interaction, text: str, user: Member = None):
        """Speaks your text aloud"""

        mention = self.invoked_command_handler.get_first_mention(interaction)
        invoked_command = lambda: self.say(text, interaction.user, user or mention or None, False, interaction)

        await self.invoked_command_handler.invoke_command(interaction, invoked_command, ephemeral=False)


# BOJ 1929 소수 구하기
# 에라토스테네스의 체
from sys import stdin
import math

l = [0, -1] + [0]*999999
M, N = map(int, stdin.readline().split())
for i in range(2, int(math.sqrt(1000000))):
    if l[i] == -1:
        continue
    j = 2
    while i * j <= 1000000:
        l[i*j] = -1
        j += 1
for i in range(M, N+1):
    if l[i] == 0:
        print(i)


import datetime as datetime
import requests
import json
import pandas as pd

current_d = datetime.datetime.today().strftime('%Y-%m-%d')

def get_stock(ticker, start_date='2015-01-01', end_date=current_d):
	data = requests.get("https://fmpcloud.io/api/v3/historical-price-full/" + ticker + "?from=" + start_date + "&to=" + end_date + "2&apikey=demo")
	data = json.loads(data.content)
	data = data['historical']

	dates = []  # List where we are going to save the dates values
	close = []  # List where we are going to save the close values
	adjClose = []

	# ForLoop for saving the data into the lists
	for item in data:
	    dates.append(pd.to_datetime(item['date']))  # getting the date as a pandas time-series
	    close.append(float(item['close']))  # getting the close price as a float
	    adjClose.append(float(item['adjClose']))  # getting the close price as a float

	# Reversing the content of the list, because we don't want the last close as our first value
	close.reverse()
	dates.reverse()
	adjClose.reverse()

	# Creating a pandas DataFrame with the values
	df = pd.DataFrame(list(zip(close, adjClose)), index=dates, columns=['Close', 'adjClose'])

	return df


import xmlrpc.client
from xmlrpc.server import SimpleXMLRPCServer
import socket

max_node_num = 10000
myname = socket.getfqdn(socket.gethostname())
myip = socket.gethostbyname(myname)
# 默认DNSip，可初始化修改
DNSip = "192.168.43.195"
# 默认软件使用端口
DNSport = "1234"
port = "1234"
table = {}
id = None
socket.setsockopt(1)

def get_myID():
    return id

class RouteServer(object):
    def __init__(self, id):
        self.id = id

    def help_find(self, targetid):
        # 协助寻找节点
        # 表中没有更近的节点就返回自身
        if min(map(lambda x: abs(x - targetid), table)) >= abs(self.id - targetid):
            return [abs(self.id - targetid), self.id]
        # 否则进行递归
        else:
            for i in table.keys():
                if (abs(i - targetid) < abs(self.id - targetid)):
                    c = xmlrpc.client.ServerProxy("http://" + i + ":" + port + "/")
                    return c.help_find(targetid)

    def update(self, id, ip):
        l = max_node_num / 2
        n = 1
        while n < l:
            min = abs(table.key()[0] - 1)
            minid = 0
            for i in table.key():
                if min > abs(i - (self.id - n) % max_node_num):
                    min = abs(i - (self.id - n) % max_node_num)
                    minid = i
            if abs(id - (self.id - n)) < min:
                table.pop(minid)
                table[id] = ip
        n = 1
        while n < l:
            min = abs(table.key()[0] - 1)
            minid = 0
            for i in table.key():
                if min > abs(i - (self.id + n) % max_node_num):
                    min = abs(i - (self.id + n) % max_node_num)
                    minid = i
            if abs(id - (self.id - n)) < min:
                table.pop(minid)
                table[id] = ip
        return 0

    def ping_response(self, ip):
        # 回应ping操作
        return

    def get_blocklist(self):
        # 返回id表块中的文件名
        list = []
        folder = './'
        for root, dirs, files in os.walk(folder):
            for f in files:
                if 'ID_block' in f:
                    list.append(f[:f.find('ID_blcok')])
        return list

class RouteClient(object):
    def __init__(self, dnsip):
        DNSc = xmlrpc.client.ServerProxy("http://" + dnsip + ":" + port + "/")
        self.id = DNSc.create_id(myip)
        id = self.id
        table[id] = myip
        t = DNSc.download_all()
        s = t.keys().sort()
        for i in range(len(s)):
            if (s[i] == self.id):
                pos = i
                break
        l = max_node_num / 2
        n = 1
        while n < l:
            # 向左遍历
            # 找到离理想节点最近的节点
            mindist = max_node_num
            mini = max_node_num
            for i in range(pos + 1):
                if (abs(s[pos - i] - (self.id - n) % max_node_num)) < mindist:
                    mini = pos - i
                    mindist = abs(s[pos - i] - (self.id - n) % max_node_num)
                else:
                    if mindist != max_node_num:
                        break
            exit_code = os.system("ping " + t[s[mini]])
            if not exit_code:
                # 获取长连接信息————插入
                n = n * 2
                table[s[mini]] = t[s[mini]]
                # 通知更新二分表————插入
                c = xmlrpc.client.ServerProxy("http://" + t[s[mini]] + ":" + port + "/")
                c.update(self.id, myip)
            else:
                # 插入服务器连接
                DNSc.dele(ID)
                s.remove(s[mini])

        while n < l:
            # 向右遍历
            # 找到离理想节点最近的节点
            mindist = max_node_num
            mini = max_node_num
            for i in range(pos + 1):
                if (abs(s[pos + i] - (self.id + n) % max_node_num)) < mindist:
                    mini = pos - i
                    mindist = abs(s[pos + i] - (self.id + n) % max_node_num)
                else:
                    if mindist != max_node_num:
                        break
            exit_code = os.system("ping " + t[s[mini]])
            if not exit_code:
                # 获取长连接信息————插入
                n = n * 2
                table[s[mini]] = t[s[mini]]
                # 通知更新二分表————插入
                c = xmlrpc.client.ServerProxy("http://" + t[s[mini]] + ":" + port + "/")
                c.update(self.id, myip)
            else:
                # 插入服务器连接
                DNSc.dele(ID)
                s.remove(s[mini])

    def get_mylrID(myID):
        L = []
        for key in table.keys():
            L.append(key)
        Ls = L.sort()
        pos = 1
        for i in range(len(Ls)):
            if Ls[i] == myID:
                pos = i
                break
        return Ls[pos - 1], Ls[pos + 1]

    def ping(self, ip):
        # 检查节点是否在线
        c = xmlrpc.client.ServerProxy("http://" + ip + ":" + port + "/")
        return c.ping_response(targetid)

    def findID(self, targetid):
        # 按id寻找节点
        dis = abs(targetid - self.id)
        ip = myip
        for i in table.keys():
            # 对表中节点遍历操作
            if (abs(i - targetid) < abs(self.id - targetid)):
                c = xmlrpc.client.ServerProxy("http://" + i + ":" + port + "/")
                # 用远程调用递归查询
                res = c.help_find(targetid)
                if (dis > res[0]):
                    dis = res[0]
                    ip = res[1]
        # 若没有更近的最近的就是自己
        return ip

    def get_blocknamedir(self):
        l, r = self.get_mylrID()
        blocknamedir = {}
        id = l
        s = xmlrpc.client.ServerProxy("http://" + id + ":" + port + "/")
        blocknamedir['leftID'] = s.get_blocklist()
        id = r
        s = xmlrpc.client.ServerProxy("http://" + id + ":" + port + "/")
        blocknamedir['rightID'] = s.get_blocklist()
        id = self.id
        s = xmlrpc.client.ServerProxy("http://" + id + ":" + port + "/")
        blocknamedir['myID'] = s.get_blocklist()

if __name__ == "__main__":
    port = "1234"
    c = Client(DNSip)
    server = Server(id)
    s = SimpleXMLRPCServer(("0.0.0.0", int(port)))
    print("start service...")
    s.register_instance(server)
    s.serve_forever()


#!/usr/bin/env python
"""Visualizes achieved energy resolutions.
"""

# python-2 compatibility
from __future__ import division        # 1/2 = 0.5, not 0
from __future__ import print_function  # print() syntax from python-3

import os
import pickle
import fnmatch
import ROOT

# for keeping drawed ROOT objects in memory
saves = []

def main():
    """Steering function.
    """
    ROOT.gROOT.SetBatch(True)
    ROOT.gStyle.SetOptStat(0)
    ROOT.TH1.AddDirectory(False)

    # keep current working directory clean from .d and .so files;
    # CPU-intensive part is written in C++ (python is too slow)
    ROOT.gSystem.SetBuildDir('output', True);
    ROOT.gROOT.LoadMacro('draw_results_helper.cc+')

    # ntuples to process
    ntuples = fnmatch.filter(os.listdir('input'), '*.root')
    ntuples = sorted('input/' + f for f in ntuples)

    # MVAs to process
    mvas = [f[f.rfind('/') + 1:].replace('.root', '') for f in ntuples]

    # text in legends
    txts = [f[f.rfind('gun_') + 4:].replace('.root', '') for f in ntuples]
    txts = [t + ', no correction' for t in txts] + txts[:]

    # make output directories
    for d in ['output', 'output/cache', 'output/plots_results', 'output/plots_results/fits']:
        if not os.access(d, os.X_OK):
            os.mkdir(d)

    eregions = [(0, 1), (1, 2), (2, 10), (10, 20), (20, 100), (100, 1000)]

    for det in ['EB', 'EE']:
        for mva in mvas:
            r  = [make_graphs(f, det, '',                eregions, blockSize=3000) for f in ntuples]
            r += [make_graphs(f, det, 'mva_mean_' + mva, eregions, blockSize=3000) for f in ntuples]

            # repack graphs into per-parameter tuples
            r = list(zip(*r))

            # text in captions
            cap = 'trained on {0}, {1}'.format(mva[mva.rfind('gun_') + 4:], det)

            # mean vs E
            title = 'mean_vs_e_{0}_{1}'.format(mva, det)
            combine(r[0], txts, title,           cap, 'E^{gen}', 'Mean_{E^{rec}/E^{gen}}')
            combine(r[0], txts, title + '_zoom', cap, 'E^{gen}', 'Mean_{E^{rec}/E^{gen}}', False, 20)

            # sigma vs E
            title = 'sigma_vs_e_{0}_{1}'.format(mva, det)
            combineR(r[1], txts, title,           cap, 'E^{gen}', '#sigma_{E^{rec}/E^{gen}}/mean')
            combineR(r[1], txts, title + '_zoom', cap, 'E^{gen}', '#sigma_{E^{rec}/E^{gen}}/mean', False, 20)

            # mean vs pT
            title = 'mean_vs_pt_{0}_{1}'.format(mva, det)
            combine(r[2], txts, title,           cap, 'p_{T}^{gen}', 'Mean_{E^{rec}/E^{gen}}')
            combine(r[2], txts, title + '_zoom', cap, 'p_{T}^{gen}', 'Mean_{E^{rec}/E^{gen}}', False, 20)

            # sigma vs pT
            title = 'sigma_vs_pt_{0}_{1}'.format(mva, det)
            combineR(r[3], txts, title,           cap, 'p_{T}^{gen}', '#sigma_{E^{rec}/E^{gen}}/mean')
            combineR(r[3], txts, title + '_zoom', cap, 'p_{T}^{gen}', '#sigma_{E^{rec}/E^{gen}}/mean', False, 20)

            for (i, (e1, e2)) in enumerate(eregions):
                # mean vs nVtx (nopu results excluded)
                try:
                    title = 'mean_vs_nvtx_{0}_{1}_E{2}-{3}'.format(mva, det, e1, e2)
                    combine([x[i] for x in r[6]], txts, title, cap, 'nVtx', 'Mean_{E^{rec}/E^{gen}}', True)
                except:
                    print('Plot excluded: {0}'.format(title))

                # sigma vs nVtx
                try:
                    title = 'sigma_vs_nvtx_{0}_{1}_E{2}-{3}'.format(mva, det, e1, e2)
                    combineR([x[i] for x in r[7]], txts, title, cap, 'nVtx', '#sigma_{E^{rec}/E^{gen}}/mean', True)
                except:
                    print('Plot excluded: {0}'.format(title))

                cap = 'trained on {0}, {1} < E^{{gen}} < {2} GeV/c'.format(mva[mva.rfind('gun_') + 4:], e1, e2)

                # mean and sigma vs eta
                if det == 'EB': # not necessary to repeat for EE
                    title = 'mean_vs_eta_{0}_E{1}-{2}'.format(mva, e1, e2)
                    combine([x[i] for x in r[4]], txts, title, cap, '#eta^{gen}', 'Mean_{E^{rec}/E^{gen}}')

                    # sigma vs eta
                    title = 'sigma_vs_eta_{0}_E{1}-{2}'.format(mva, e1, e2)
                    combineR([x[i] for x in r[5]], txts, title, cap, '#eta^{gen}', '#sigma_{E^{rec}/E^{gen}}/mean')

def make_graphs(infile, det, mva_branch, eregions, blockSize):
    """Fills, fits and visualizes distributions of Etrue/Erec.

    Results are cached into file.
    """
    # return cached results, if any
    fname = os.path.basename(infile).replace('.root', '')
    fmt = 'output/cache/draw_results_{0}_{1}_{2}_{3}.pkl'
    cachefile = fmt.format(fname, det, mva_branch, blockSize)
    if os.access(cachefile, os.R_OK):
        with open(cachefile, 'rb') as f:
            return pickle.load(f)

    # fill necessary arrays of points in C++
    friend = 'output/friend_{0}.root'.format(fname)
    ROOT.fill_arrays(infile, friend, mva_branch, True if det == 'EE' else False)

    # resolution vs mcE
    title = 'mcE_{0}_{1}_{2}'.format(fname, det, mva_branch)
    ROOT.fit_slices(0, blockSize, title, 'E^{gen}')
    grMeanE = ROOT.grMean.Clone()
    grSigmaE = ROOT.grSigma.Clone()

    # resolution vs mcPt
    title = 'mcPt_{0}_{1}_{2}'.format(fname, det, mva_branch)
    ROOT.fit_slices(1, blockSize, title, 'p_{T}^{gen}')
    grMeanPt = ROOT.grMean.Clone()
    grSigmaPt = ROOT.grSigma.Clone()

    # resolution vs mcEta in energy ranges
    grMeanEta = []
    grSigmaEta = []
    for (e1, e2) in eregions:
        title = 'mcEta_{0}_{1}_{2}_E{3}-{4}'.format(fname, det, mva_branch, e1, e2)
        ROOT.fit_slices(2, blockSize, title, '#eta^{gen}', e1, e2)
        grMeanEta.append(ROOT.grMean.Clone())
        grSigmaEta.append(ROOT.grSigma.Clone())

    # resolution vs nVtx in energy ranges
    grMeanVtx = []
    grSigmaVtx = []
    for (e1, e2) in eregions:
        title = 'nVtx_{0}_{1}_{2}_E{3}-{4}'.format(fname, det, mva_branch, e1, e2)
        ROOT.fit_slices(3, blockSize, title, 'nVtx', e1, e2)
        grMeanVtx.append(ROOT.grMean.Clone())
        grSigmaVtx.append(ROOT.grSigma.Clone())

    result = (grMeanE, grSigmaE, grMeanPt, grSigmaPt, grMeanEta, grSigmaEta,
              grMeanVtx, grSigmaVtx)

    # save cache
    with open(cachefile, 'wb') as f:
        pickle.dump(result, f)

    return result

def combine(grs, txts, cname, title, xtitle, ytitle, skipNoPU=False, xmax=-1, ymax=-1):
    """Visualization of several graphs on single canvas.
    """
    c = ROOT.TCanvas(cname, cname, 700, 700)
    saves.append(c)

    c.SetLeftMargin(0.14)
    c.SetRightMargin(0.08)
    c.SetTopMargin(0.06)
    c.SetBottomMargin(0.1)
    c.SetGridx()
    c.SetGridy()

    # draw dummy histogram
    xmin = min(gr.GetX()[i] for gr in grs for i in range(gr.GetN()))
    ymin = min(gr.GetY()[i] for gr in grs for i in range(gr.GetN()))
    if xmax < 0:
        xmax = max(gr.GetX()[i] for gr in grs for i in range(gr.GetN()))
    if ymax < 0:
        ymax = max(gr.GetY()[i] for gr in grs for i in range(gr.GetN()))

    frame = c.DrawFrame(xmin, ymin * 0.9, xmax, ymax * 1.1)
    frame.SetTitle(title)
    frame.SetXTitle(xtitle)
    frame.SetYTitle(ytitle)
    frame.SetTitleOffset(1.2, 'X')
    frame.SetTitleOffset(1.95, 'Y')
    frame.Draw()

    # number of corrected and uncorrected graphs
    ncorr = len(grs)//2

    # legend
    leg = ROOT.TLegend(0.58, 0.12, 0.89, 0.12 + 0.033 * len(txts))

    clrs = [ROOT.kOrange] * ncorr + [8, ROOT.kBlack, ROOT.kBlue, ROOT.kRed]

    for (gr, clr, txt) in zip(grs, clrs, txts):
        if skipNoPU and txt.startswith('nopu'):
           continue

        gr.SetMarkerStyle(20)
        gr.SetMarkerSize(0.4)
        gr.SetMarkerColor(clr)
        gr.SetLineColor(clr)

        gr.Draw('PZL')
        leg.AddEntry(gr, txt, 'p')

    leg.SetFillColor(0)
    leg.Draw('same')

    saves.append((frame, grs, leg))

    c.Update()
    c.SaveAs('output/plots_results/{0}.png'.format(c.GetTitle()))

def combineR(grs, txts, cname, title, xtitle, ytitle, skipNoPU=False, xmax=-1, ymax=-1):
    """Visualization of graphs along with corrected/uncorrected ratios.
    """
    c = ROOT.TCanvas(cname, cname, 700, 700)
    saves.append(c)

    pad1 = ROOT.TPad('top', 'top', 0, 0.3, 1, 1)
    pad2 = ROOT.TPad('bottom', 'bottom', 0, 0, 1, 0.3)
    pad1.Draw()
    pad2.Draw()
    saves.append(pad1)
    saves.append(pad2)

    # top pad
    pad1.cd()
    pad1.SetTopMargin(0.07)
    pad1.SetBottomMargin(0)
    pad1.SetLeftMargin(0.095)
    pad1.SetRightMargin(0.02)
    pad1.SetGridx()
    pad1.SetGridy()

    # draw dummy histogram
    xmin = min(gr.GetX()[i] for gr in grs for i in range(gr.GetN()))
    ymin = min(gr.GetY()[i] for gr in grs for i in range(gr.GetN()))
    if xmax < 0:
        xmax = max(gr.GetX()[i] for gr in grs for i in range(gr.GetN()))
    if ymax < 0:
        ymax = max(gr.GetY()[i] for gr in grs for i in range(gr.GetN()))

    frame = pad1.DrawFrame(xmin, ymin * 0.9, xmax, ymax * 1.1)
    frame.SetTitle(title)
    frame.SetXTitle(xtitle)
    frame.SetYTitle(ytitle)
    frame.SetTitleOffset(1.2, 'X')
    frame.SetTitleOffset(1.35, 'Y')
    frame.Draw()

    # number of corrected and uncorrected graphs
    ncorr = len(grs)//2

    # legend
    leg = ROOT.TLegend(0.58, 0.58, 0.95, 0.58 + 0.04 * len(txts))

    clrs = [ROOT.kOrange] * ncorr + [8, ROOT.kBlack, ROOT.kBlue, ROOT.kRed]

    for (gr, clr, txt) in zip(grs, clrs, txts):
        if skipNoPU and txt.startswith('nopu'):
           continue

        gr.SetMarkerStyle(20)
        gr.SetMarkerSize(0.4)
        gr.SetMarkerColor(clr)
        gr.SetLineColor(clr)

        gr.Draw('PZL')
        leg.AddEntry(gr, txt, 'p')

    leg.SetFillColor(0)
    leg.Draw('same')

    saves.append((frame, grs, leg))

    # bottom pad
    pad2.cd()
    pad2.SetTopMargin(0)
    pad2.SetBottomMargin(0.25)
    pad2.SetLeftMargin(0.095)
    pad2.SetRightMargin(0.02)
    pad2.SetGridx()
    pad2.SetGridy()

    # make corrected/uncorrected ratios
    rats = [ROOT.TGraphErrors() for _ in range(ncorr)]

    for n in range(ncorr):
        gru = grs[n]         # uncorrected
        grc = grs[ncorr + n] # corrected

        for i in range(grc.GetN()):
            x  = grc.GetX()[i]
            y1 = grc.GetY()[i]
            ex  = grc.GetEX()[i]
            ey1 = grc.GetEY()[i]

            # NOTE: gru may have different x and ex, but we ignore this
            # difference. Instead, y for gru is estimated from linear
            # extrapolation between corresponding nearest points.

            # find nearest points
            for j in range(gru.GetN() - 1):
                x1 = gru.GetX()[j]
                x2 = gru.GetX()[j + 1]

                if  x1 <= x < x2:
                    a = (gru.GetY()[j] - gru.GetY()[j + 1]) / (x1 - x2)
                    b = gru.GetY()[j] - a * x1
                    y2 = a * x + b

                    ea = (gru.GetEY()[j] - gru.GetEY()[j + 1]) / (x1 - x2)
                    eb = gru.GetEY()[j] - ea * x1
                    ey2 = ea * x + eb

                    break
            else:
                if x < gru.GetX()[0]:
                    y2 = gru.GetY()[0]
                    ey2 = gru.GetEY()[0]
                else:
                    y2 = gru.GetY()[gru.GetN() - 1]
                    ey2 = gru.GetEY()[gru.GetN() - 1]

            rats[n].SetPoint(i, x, y1/y2)
            rats[n].SetPointError(i, ex, y1/y2 * ((ey1/y1)**2 + (ey2/y2)**2)**0.5)

    frame = pad2.DrawFrame(xmin, 0.1, xmax, 1.25)
    frame.SetXTitle(xtitle)
    frame.SetYTitle('Corr/uncorr')
    frame.SetTitleOffset(1.2, 'X')
    frame.SetTitleOffset(0.58, 'Y')
    frame.SetTitleSize(0.08, 'XY')
    frame.SetLabelSize(0.08, 'XY')
    frame.Draw()

    # draw corrected/uncorrected ratios
    for (gr, clr, txt) in zip(rats, clrs[ncorr:], txts[ncorr:]):
        if skipNoPU and txt.startswith('nopu'):
           continue

        gr.SetMarkerStyle(20)
        gr.SetMarkerSize(0.4)
        gr.SetMarkerColor(clr)
        gr.SetLineColor(clr)
        gr.Draw('PZL')

    saves.append((frame, rats))

    c.Update()
    c.SaveAs('output/plots_results/{0}.png'.format(c.GetTitle()))

if __name__ == '__main__':
    main()


import torch
from newspaper import Article
from sentence_transformers import SentenceTransformer, util
from sklearn.decomposition import PCA
import matplotlib.pyplot as plt
from typing import Dict, List, Tuple
from torch import Tensor
from context_concepts import contexts
import numpy as np
from mpl_toolkits.mplot3d import Axes3D

def get_corpus_from_links(urls: List[str]
                          ) -> Tuple[List[str], List[str], Tensor]:
    # create and embed corpus
    corpus, titles = [], []
    for url in urls:
        article = Article(url)
        try:
            article.download()
            article.parse()
        except:
            continue
        corpus.append(article.text)
        titles.append(article.title)
    corpus_embeddings = model.encode(corpus, convert_to_tensor=True)
    return titles, corpus, corpus_embeddings

def filter(corpus_embeddings: Tensor,
           query_embedding: Tensor,
           top_k: int=5
           ) -> List[int]:
    cos_scores = util.pytorch_cos_sim(query_embedding, corpus_embeddings)[0]
    cos_scores = cos_scores.cpu()
    top_results = np.argpartition(-cos_scores, range(top_k))[0:top_k]
    return top_results.tolist()

def new_order(l: List[int], mirror: List[int]) -> List[int]:
    return [l[x] for x in mirror]

def post_filter(corpus_embeddings: Tensor,
                topic_embedding: Tensor,
                curr_top_indices: List[int],
                top_k: int = 5
                ) -> List[int]:
    new_embeddings = None
    for item_index in top_indices:
        if new_embeddings == None:
            new_embeddings = corpus_embeddings[int(item_index)].reshape(1, 768)
        else:
            new_embeddings = torch.cat((new_embeddings, corpus_embeddings[int(item_index)].reshape(1, 768)), dim=0)

    new_scores = util.pytorch_cos_sim(topic_embedding, new_embeddings)[0]
    new_scores = new_scores.cpu()
    top_results = np.argpartition(-new_scores, range(top_k))[0:top_k]
    order = new_order(curr_top_indices, top_results)
    return order

def similarity_matrix(encoded_topics: List[Tensor]
                      ) -> List[List[float]]:
    topic_length = len(encoded_topics)
    m = [[0] * topic_length for x in range(topic_length)]
    for i, row in enumerate(encoded_topics):
        for j, col in enumerate(encoded_topics):
            m[i][j] = round(float(dist(row, col)), 4)
    return m

def dist(x, y):
    return util.pytorch_cos_sim(x,y)[0]
    # dist = lambda x, y: util.pytorch_cos_sim(x,y)[0]

def dist_w(x, y):
    return util.pytorch_cos_sim(en(x),en(y))[0]
    # dist_w = lambda x, y: util.pytorch_cos_sim(en(x),en(y))[0]
def en(x):
    return model.encode(x)
    # en = lambda x: model.encode(x)

model = SentenceTransformer('distilbert-base-nli-stsb-mean-tokens')#distilbert-base-nli-mean-tokens')
"""
Experiment 001

Setting:
Taking a toy corpus of 10 corona articles. All articles talk about
masks and some talk about masks w.r.t. kids and their environment (school).
We assume a mother (role) that wants to know if her child should wear a mask (e.g. in
school (location)). The location will be context modelled here (i.e. encoded through a wiki article)
and and used in a post-processing manner.

Hypothesis:
The hypothesis is that having this knowledge about the mother and the setting
she is interested in results in better information.

Result:
We can see that when we query we retrieve as results the ranking [5,4,0,9,8]. This shows that
the general face mask articles are ranked 1 and 2. The third article is a kids article, however
the two best articles which explicitly talk about what the mother wants to know are ranked low.
After querying against that ranking with the school we retrieve a new ranking [0, 8, 9, 5, 4]
which, without a doubt, would perfectly meet the mother's concern.

The PCA visualizes that the documents indeed are very close to each other. It shows that in this
semantic space only minor differences are responsible for worse or better results.
"""
# link collection
urls = ["https://www.nytimes.com/2020/08/06/us/crayola-masks-kids.html",
		"https://www.nytimes.com/article/face-shield-mask-california-coronavirus.html",
		"https://www.nytimes.com/2020/07/27/health/coronavirus-mask-protection.html",
		"https://www.nytimes.com/interactive/2020/health/coronavirus-best-face-masks.html",
		"https://www.nytimes.com/2020/05/24/health/coronavirus-face-shields.html",
		"https://www.nytimes.com/2020/07/27/business/fashion-masks-coronavirus.html",
		"https://www.nytimes.com/2020/08/14/parenting/schools-reopening-south-korea.html",
		"https://www.nytimes.com/2020/06/23/well/family/children-masks-coronavirus.html",
		"https://www.nytimes.com/2020/04/09/parenting/coronavirus-kids-masks.html"]
# filter corpus with query and retrieve top k indices
query_embedding = model.encode("should my child wear a face mask at school", convert_to_tensor=True)
titles, corpus, corpus_embeddings = get_corpus_from_links(urls)
top_indices = filter(corpus_embeddings, query_embedding, 5)
# query corpus with the topic
school_text = "primary school"#A primary school, junior school, elementary school or grade school is a school for children from about four to eleven years old, in which they receive primary or elementary education. It can refer to both the physical structure (buildings) and the organisation. Typically it comes after preschool, and before secondary school."
school_embedding = model.encode(school_text, convert_to_tensor=True)
top_indices = post_filter(corpus_embeddings, school_embedding, top_indices)
# visualisation
pca = PCA(n_components=3)
embeddings = corpus_embeddings + query_embedding + school_embedding
emb = torch.cat((corpus_embeddings, query_embedding.reshape(1, 768), school_embedding.reshape(1, 768)), dim=0)
pc = pca.fit_transform(emb.cpu())
x, y, z = pc[:,0], pc[:,1], pc[:,2]
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
for i in range(12):
    if i == 10: # this is the query
        ax.scatter(x[i], y[i], z[i], c='g', marker='o')
    elif i == 11: # this is the school embedding
        ax.scatter(x[i], y[i], z[i], c='b', marker='o')
    else:
        ax.scatter(x[i], y[i], z[i], c='r', marker='o')
plt.show()
s=0
"""
Experiment 002

Setting:
We now look at the context "theme" and one of its topics "mask".
As we are in latent high dimensional space we cannot make sense of representation of encoded words.
Moreover, making sense of encoded sentences or documents cannot be made sense of even more.
It is crucial to verify if encoded information are precise and if words, descriptions and texts that
are supposedly close to each are so indeed.

Hypothesis:
The hypothesis is that encoded wikipedia articles comprise important knowledge and are a better representation
of a certain concept than just a word alone.

Results:
- The two texts describing the topic in the wiki article are VERY close to each other (0.8398)
  - This implies that the underlying topic, despite talking about different parts of it, is encoded in the vectors
- Two first section seems to be closer to the other words
  - This implies that choosing the right context to embed is of importance w.r.t. accuracy
- The custom, supposedly very precise, description of a the corona face mask, is very close to all concepts
  - This implies that embedding the text might be reasonable
- The custom, precise description is very close to the wikipedia articles, not as close to general masks
  - This implies that these sections to cover semantic knowledge about the circumstances without being trained on COVID
- Facemask and Face mask alone have almost no similary to the wiki articles
  - This implies that embedding the word alone may not be precise enough

Matrix:
[1.0, 0.8398, 0.1179, 0.1427, 0.3443, 0.1397, 0.6104]
[0.8398, 1.0, 0.0, 0.0514, 0.3142, 0.0476, 0.5565]
[0.1179, 0.0, 1.0, 0.9056, 0.781, 0.8695, 0.4708]
[0.1427, 0.0514, 0.9056, 1.0, 0.8334, 0.9466, 0.4629]
[0.3443, 0.3142, 0.781, 0.8334, 1.0, 0.8444, 0.5279]
[0.1397, 0.0476, 0.8695, 0.9466, 0.8444, 1.0, 0.4656]
[0.6104, 0.5565, 0.4708, 0.4629, 0.5279, 0.4656, 1.0]
"""
fm1 = en(contexts["theme"]["face mask"]["text"][0])
fm2 = en(contexts["theme"]["face mask"]["text"][1])
fm3 = en("facemask")
fm4 = en("face mask")
fm5 = en("surgical mask")
fm6 = en("mask")
fm7 = en("mouth-nose-covering used to prevent viral spread")
all_words1 = [fm1, fm2, fm3, fm4, fm5, fm6, fm7]
sim_matrix1 = similarity_matrix(all_words1)
print(*sim_matrix1, sep="\n")

"""
Experiment 003

Setting:
We now look at the context "theme" and one of its topics "social distancing".

Hypothesis:
Same as before.

Results:
We can see similar results as above.
The encoded topics are all quite similar to each other.
The first encoded topic is the best performing w.r.t. the other topics.
Encoded Topic 2 and 3 have almost 0 similarity with the hardcoded encoded words.
  - This implies that there are definitely bad candidate sentences.
  - This implies that there is a need for evaluating the best performing textual representation
If we only look at the synonyms we see that all encodings are very close to each other
with a score of 0.7 - 0.9.
  - This implies that encoding a very particular word may not be that important.
Matrix:
[1.0000, 0.6898, 0.6943, 0.7919, 0.3188, 0.2143, 0.1944, 0.2310, 0.2633, 0.3341, 0.3174, 0.3163]
[0.6898, 1.0000, 0.5752, 0.7837, 0.0723, 0.0529, 0.0004, 0.0436, 0.0559, 0.0942, 0.0621, 0.0201]
[0.6943, 0.5752, 1.0000, 0.5847, 0.0905, 0.0552, 0.0821, 0.0431, 0.0429, 0.1190, 0.1124, 0.1262]
[0.7919, 0.7837, 0.5847, 1.0000, 0.3147, 0.1441, 0.2077, 0.2037, 0.2474, 0.3431, 0.3187, 0.1973]
[0.3188, 0.0723, 0.0905, 0.3147, 1.0000, 0.8677, 0.7550, 0.7949, 0.7975, 0.8075, 0.8187, 0.7211]
[0.2143, 0.0529, 0.0552, 0.1441, 0.8677, 1.0000, 0.7859, 0.8514, 0.7194, 0.8115, 0.8280, 0.7862]
[0.1944, 0.0004, 0.0821, 0.2077, 0.7550, 0.7859, 1.0000, 0.8705, 0.6632, 0.8490, 0.8751, 0.7405]
[0.2310, 0.0436, 0.0431, 0.2037, 0.7949, 0.8514, 0.8705, 1.0000, 0.7904, 0.9105, 0.9346, 0.8691]
[0.2633, 0.0559, 0.0429, 0.2474, 0.7975, 0.7194, 0.6632, 0.7904, 1.0000, 0.7319, 0.7447, 0.6668]
[0.3341, 0.0942, 0.1190, 0.3431, 0.8075, 0.8115, 0.8490, 0.9105, 0.7319, 1.0000, 0.9917, 0.8196]
[0.3174, 0.0621, 0.1124, 0.3187, 0.8187, 0.8280, 0.8751, 0.9346, 0.7447, 0.9917, 1.0000, 0.8333]
[0.3163, 0.0201, 0.1262, 0.1973, 0.7211, 0.7862, 0.7405, 0.8691, 0.6668, 0.8196, 0.8333, 1.0000]
"""
sd1 = en(contexts["theme"]["social distancing"]["text"][0])
sd2 = en(contexts["theme"]["social distancing"]["text"][1])
sd3 = en(contexts["theme"]["social distancing"]["text"][2])
sd4 = en(contexts["theme"]["social distancing"]["text"][3])
sd5 = en("social distancing")
sd6 = en("social distance")
sd7 = en("not going to public areas")
sd8 = en("avoiding physical contact")
sd9 = en("cutting oneself off")
sd10 = en("avoiding contacts with people")
sd11 = en("avoiding contacts with individuals")
sd12 = en("avoiding crowded places")
sd13 = en("A drastic measure of social distancing avoiding physical contact with other people and crowded places.")
all_words2 = [eval(f"sd{i}") for i in range(1,13)]
sim_matrix2 = similarity_matrix(all_words2)
print(*sim_matrix2, sep="\n")

"""
General Notes:
- it is of very importance to find out how to embed the topics the best way
	- embedding wikipedia articles yields different results. some are really good (w.r.t the solely embedded terms) and some are really bad. Wikipedia articles might be too general
	for the model to capture the notion / sense behind it
	- embedding described topics by hand with precise and many synomys might be good
	- embedding the most general concept that describes the topic might be the best
- different models yield different vector spaces
- Pre-trained models only know general concepts. Corona specific words are unknown and have no sense in the vector space
	- Possibly map specific terms to general terms
"""

"""
Experiment 004
"""

# Create Corpus

context = r"""Extractive Question Answering is the task of extracting an answer from a text given a question. An example of a question answering dataset is the SQuAD dataset, which is entirely based on that task. If you would like to fine-tune a model on a SQuAD task, you may leverage the examples/question-answering/run_squad.py script"""


#!/usr/bin/python3
import rospy
import moveit_commander
import sys
import geometry_msgs.msg
def move_ur_to_target():
    # Initialize MoveIt
    moveit_commander.roscpp_initialize(sys.argv)
    rospy.init_node('move_ur_node', anonymous=True)

    # Instantiate a MoveGroupCommander for the UR arm group
    arm_group = moveit_commander.MoveGroupCommander("manipulator_left")

    # Set the target position and orientation
    target_pose = geometry_msgs.msg.Pose()
    target_pose.position.x = 1.2  # Set your desired target position
    target_pose.position.y = 0.3
    target_pose.position.z = 0.2
    # Set your desired target orientation if needed

    # Plan and execute the motion
    arm_group.set_pose_target(target_pose)
    arm_group.go(wait=True)

    # Shutdown MoveIt
    moveit_commander.roscpp_shutdown()

if __name__ == '__main__':
    try:
        move_ur_to_target()
    except rospy.ROSInterruptException:
        pass


import sandbox
import shapeGenerator

from panda3d.core import PerlinNoise2, PNMImage, PointLight, Shader, Texture, Vec3, Vec4
from direct.filter.CommonFilters import CommonFilters

#sandbox.base.render.setShaderAuto()

planet = shapeGenerator.Sphere(1, 64, 'planet')
planet.reparentTo(sandbox.base.render)

planet.setPos(5, 20, 0)

mesh = shapeGenerator.Sphere(1, 64, 'star')
mesh.reparentTo(sandbox.base.render)

mesh.setPos(-5, 20, 0)

noise = PerlinNoise2(64, 64)
texture = Texture('noise')
texture.setup2dTexture()
img = PNMImage(1024, 1024)
for y in range(1024):
    for x in range(1024):
        img.setXel(x, y, noise.noise(x, y))
texture.load(img)
mesh.setTexture(texture, 1)

mesh.setShaderInput('time', 0)
#shaders = Shader.load(Shader.SLGLSL, 'vortexVertex.glsl', 'starFrag2.glsl')
shaders = Shader.load(Shader.SLGLSL, 'sphereVertex.glsl', 'starFrag2.glsl')
mesh.setShader(shaders)

light = mesh.attachNewNode(PointLight("sun"))
light.node().setColor(Vec4(1, 1, 1, 1))
sandbox.base.render.setLight(light)

def time_update(task):
    mesh.setShaderInput('time', task.time)
    mesh.setH(-task.time*10)
    return task.cont

sandbox.base.taskMgr.add(time_update, 'task')

#filters = CommonFilters(base.win, base.cam)
#filters.setBloom()

sandbox.run()


from .import views
from django.urls import path

urlpatterns = [
    path('',views.base,name='base'),
    path('add_emp',views.add_emp,name='add_emp'),
    path('show_emp',views.show_emp,name='show_emp'),
    path('editpage/<int:pk>',views.editpage,name='editpage'),
    path('edit_emp/<int:pk>',views.edit_emp,name='edit_emp'),
]


import contextlib
import os
import pickle
import sys
import time
from collections import Counter
from pathlib import Path
from typing import Dict, List, Optional, Union

import joblib
import numpy as np
import pandas as pd
import rdkit.Chem as Chem
import rdkit.Chem.AllChem as AllChem
from joblib import Parallel, delayed
from rdchiral.main import rdchiralReactants, rdchiralReaction, rdchiralRun
from rdkit import DataStructs
from rxnebm.proposer.RetroSim.retrosim.data.get_data import (
    get_data_df, split_data_df)
from rxnebm.proposer.RetroSim.retrosim.utils.generate_retro_templates import \
    process_an_example
from tqdm import tqdm

@contextlib.contextmanager
def tqdm_joblib(tqdm_object):
    # https://stackoverflow.com/questions/24983493/tracking-progress-of-joblib-parallel-execution
    """Context manager to patch joblib to report into tqdm progress bar given as argument"""
    class TqdmBatchCompletionCallback(joblib.parallel.BatchCompletionCallBack):
        def __init__(self, *args, **kwargs):
            super().__init__(*args, **kwargs)

        def __call__(self, *args, **kwargs):
            tqdm_object.update(n=self.batch_size)
            return super().__call__(*args, **kwargs)

    old_batch_callback = joblib.parallel.BatchCompletionCallBack
    joblib.parallel.BatchCompletionCallBack = TqdmBatchCompletionCallback
    try:
        yield tqdm_object
    finally:
        joblib.parallel.BatchCompletionCallBack = old_batch_callback
        tqdm_object.close()

# wrappers for multiprocessing
def mol_from_smiles(smiles):
    mol = Chem.MolFromSmiles(smiles)
    return mol

def mol_to_smiles(mol, isomericSmiles=True):
    smi = Chem.MolToSmiles(mol, isomericSmiles)
    return smi

def similarity_metric(fp, list_fps):
    result = DataStructs.BulkTanimotoSimilarity(fp, list_fps)
    return result

def rdchiralreactant_dist(smi):
    return rdchiralReactants(smi)

def rdchiralreaction_dist(template):
    return rdchiralReaction(template)

def rdchiralrun_dist(rxn, rct, combine_enantiomers):
    return rdchiralRun(rxn, rct, combine_enantiomers=combine_enantiomers)

class Retrosim:
    '''
    Wrapper over retrosim for preparing training corpus + fingerprints,
    and generating one-step precursor proposals for EBM re-ranking task
    Called by rxnebm.proposer.retrosim_proposer (self.build_model & self.propose methods)

    Parameters
    ----------
    topk : int (Default = 200)
        for each product, how many of all the total proposals generated by Retrosim to be extracted
        the original Retrosim paper uses 50 (testlimit variable)
    max_prec: int (Default = 200)
        for each product, how many similar products Retrosim should consider in the product similarity search
        the original Retrosim paper uses 100 (max_prec variable)
    similarity_type : Optional[str] (Default = 'Tanimoto')
        ['Tanimoto', 'Dice', 'TverskyA', 'TverskyB']
        metric to use for similarity search of product fingerprints
    fp_type : Optional[str] (Default = 'Morgan2Feat')
        ['Morgan2noFeat', 'Morgan3noFeat', 'Morgan2Feat', 'Morgan3Feat']
        fingerprint type to generate for each product molecule
    input_data_folder : Optional[Union[str, bytes, os.PathLike]] (Default = None)
        path to the folder containing the train/valid/test reaction SMILES strings
        if None, this defaults to:   path/to/rxn/ebm/data/cleaned_data/
    input_data_file_prefix : Optional[str] (Default = '50k_clean_rxnsmi_noreagent_allmapped')
        prefix of the 3 pickle files containing the train/valid/test reaction SMILES strings
    output_folder : Optional[Union[str, bytes, os.PathLike]] (Default = None)
        path to the folder that will contain the CSV file(s) containing Retrosim's proposals
        if None, this defaults to the same folder as input_data_folder
    parallelize : Optional[bool] (Default = False)
        whether to parallelize the proposal generation step, using all available cores
    '''
    def __init__(self,
                topk: int = 200,
                max_prec: int = 200,
                similarity_type: Optional[str] = 'Tanimoto',
                fp_type: Optional[str] = 'Morgan2Feat',
                input_data_folder: Optional[Union[str, bytes, os.PathLike]] = None,
                input_data_file_prefix: Optional[str] = '50k_clean_rxnsmi_noreagent_allmapped',
                output_folder: Optional[Union[str, bytes, os.PathLike]] = None,
                parallelize: Optional[bool] = True):
        self.topk = topk
        self.max_prec = max_prec

        if similarity_type == 'Tanimoto':
            self.similarity_metric = lambda x, y: DataStructs.BulkTanimotoSimilarity(x, y)
        elif similarity_type == 'Dice':
            self.similarity_metric = lambda x, y: DataStructs.BulkDiceSimilarity(x, y)
        elif similarity_type == 'TverskyA': # weighted towards punishing only A
            self.similarity_metric = lambda x, y: DataStructs.BulkTverskySimilarity(x, y, 1.5, 1.0)
        elif similarity_type == 'TverskyB': # weighted towards punishing only B
            self.similarity_metric = lambda x, y: DataStructs.BulkTverskySimilarity(x, y, 1.0, 1.5)
        else:
            raise ValueError('Unknown similarity type')

        if fp_type == 'Morgan2Feat':
            self.getfp = lambda smi: AllChem.GetMorganFingerprint(Chem.MolFromSmiles(smi), 2, useFeatures=True)
        elif fp_type == 'Morgan2noFeat':
            self.getfp = lambda smi: AllChem.GetMorganFingerprint(Chem.MolFromSmiles(smi), 2, useFeatures=False)
        elif fp_type == 'Morgan3Feat':
            self.getfp = lambda smi: AllChem.GetMorganFingerprint(Chem.MolFromSmiles(smi), 3, useFeatures=True)
        elif fp_type == 'Morgan3noFeat':
            self.getfp = lambda smi: AllChem.GetMorganFingerprint(Chem.MolFromSmiles(smi), 3, useFeatures=False)
        else:
            raise ValueError('Unknown fingerprint type')

        if input_data_folder is None:
            self.input_folder = Path(__file__).resolve().parents[1] / 'data/cleaned_data/'
        else:
            self.input_folder = Path(input_data_folder)
        self.input_prefix = input_data_file_prefix

        if output_folder is None:
            self.output_folder = self.input_folder
        else:
            self.output_folder = Path(output_folder)

        self.parallelize = parallelize

        self._prep_training_corpus()

    def _prep_training_corpus(self) -> None:
        '''
        Sets (but only with training data):
            self.clean_50k (Dict[str, List[str]]), self.clean_50k_remove_atom_map (Dict[str, List[str]]),
            self.prod_smiles (Dict[str, List[str]]), self.rcts_smiles (Dict[str, List[str]]), self.all_prod_smiles (List[str])
        '''
        print('Preparing training data for Retrosim...')
        clean_50k, clean_50k_remove_atom_map = {}, {}
        prod_smiles, rcts_smiles = {}, {}

        phase_marker = [] # to create 'dataset' column for dataframe
        all_prod_smiles, all_rxn_smiles = [], [] # for dataframe

        self.phases = ['train']
        for phase in self.phases:
            phase_rxn_smi_remove_atom_map, phase_prod_smis, phase_rcts_smis = [], [], []

            with open(self.input_folder / f'{self.input_prefix}_{phase}.pickle', 'rb') as handle:
                clean_50k[phase] = pickle.load(handle)
                phase_marker.extend([phase] * len(clean_50k[phase]))

            for rxn_smi in tqdm(clean_50k[phase]):
                all_rxn_smiles.append(rxn_smi)

                prod_smi = rxn_smi.split('>>')[-1]
                prod_mol = Chem.MolFromSmiles(prod_smi)
                [atom.ClearProp('molAtomMapNumber') for atom in prod_mol.GetAtoms()]
                prod_smi_remove_atom_map = Chem.MolToSmiles(prod_mol, True)
                prod_smi_remove_atom_map = Chem.MolToSmiles(Chem.MolFromSmiles(prod_smi_remove_atom_map), True)

                all_prod_smiles.append(prod_smi_remove_atom_map)
                phase_prod_smis.append(prod_smi_remove_atom_map)

                rcts_smi = rxn_smi.split('>>')[0]
                rcts_mol = Chem.MolFromSmiles(rcts_smi)
                [atom.ClearProp('molAtomMapNumber') for atom in rcts_mol.GetAtoms()]
                rcts_smi_remove_atom_map = Chem.MolToSmiles(rcts_mol, True)
                # Sometimes stereochem takes another canonicalization...
                rcts_smi_remove_atom_map = Chem.MolToSmiles(Chem.MolFromSmiles(rcts_smi_remove_atom_map), True)
                phase_rcts_smis.append(rcts_smi_remove_atom_map)

                rxn_smi_remove_atom_map = rcts_smi_remove_atom_map + '>>' + prod_smi_remove_atom_map
                phase_rxn_smi_remove_atom_map.append(rxn_smi_remove_atom_map)

            clean_50k_remove_atom_map[phase] = phase_rxn_smi_remove_atom_map
            prod_smiles[phase] = phase_prod_smis
            rcts_smiles[phase] = phase_rcts_smis

        data = pd.DataFrame({
            'prod_smiles': all_prod_smiles,
            'rxn_smiles': all_rxn_smiles,
            'dataset': phase_marker,
        })

        try:
            if prev_FP != self.getfp:
                raise NameError
        except NameError:
            all_fps = []
            for smi in tqdm(data['prod_smiles'], desc='Generating fingerprints'):
                all_fps.append(self.getfp(smi))
            data['prod_fp'] = all_fps
            prev_FP = self.getfp

        self.datasub = data.loc[data['dataset'] == 'train']
        fps = list(self.datasub['prod_fp'])
        print(f'Size of training corpus: {len(fps)}')

        try:
            with open(self.input_folder / 'jx_cache.pickle', 'rb') as handle:
                self.jx_cache = pickle.load(handle)
        except:
            print('Did not find jx_cache.pickle, initialising new jx_cache dictionary')
            self.jx_cache = {}

        self.clean_50k = clean_50k
        self.clean_50k_remove_atom_map = clean_50k_remove_atom_map
        self.prod_smiles = prod_smiles
        self.rcts_smiles = rcts_smiles
        self.all_prod_smiles = all_prod_smiles

    def prep_valid_and_test_data(self,
                                input_data_folder: Optional[Union[str, bytes, os.PathLike]] = None,
                                input_data_file_prefix: Optional[str] = None) -> None:
        '''
        Needs self._prep_training_corpus() to have executed first!
        Sets:
            self.clean_50k (Dict[str, List[str]]), self.clean_50k_remove_atom_map (Dict[str, List[str]]),
            self.prod_smiles (Dict[str, List[str]]), self.rcts_smiles (Dict[str, List[str]]), self.all_prod_smiles (List[str])
        '''
        # retrieve existing data prepared by self._prep_training_corpus()
        clean_50k, clean_50k_remove_atom_map = self.clean_50k, self.clean_50k_remove_atom_map
        prod_smiles, rcts_smiles = self.prod_smiles, self.rcts_smiles
        all_prod_smiles = self.all_prod_smiles # for self.propose_all()

        print('Preparing validation and testing data for Retrosim...')
        self.phases = ['valid', 'test']
        for phase in self.phases:
            phase_rxn_smi_remove_atom_map, phase_prod_smis, phase_rcts_smis = [], [], []

            if input_data_folder is None:
                input_data_folder = self.input_folder
            if input_data_file_prefix is None:
                input_data_file_prefix = self.input_prefix
            with open(input_data_folder / f'{input_data_file_prefix}_{phase}.pickle', 'rb') as handle:
                clean_50k[phase] = pickle.load(handle)

            for rxn_smi in tqdm(clean_50k[phase], desc=f'Processing {phase}'):
                prod_smi = rxn_smi.split('>>')[-1]
                prod_mol = Chem.MolFromSmiles(prod_smi)
                [atom.ClearProp('molAtomMapNumber') for atom in prod_mol.GetAtoms()]
                prod_smi_remove_atom_map = Chem.MolToSmiles(prod_mol, True)
                prod_smi_remove_atom_map = Chem.MolToSmiles(Chem.MolFromSmiles(prod_smi_remove_atom_map), True)

                all_prod_smiles.append(prod_smi_remove_atom_map)
                phase_prod_smis.append(prod_smi_remove_atom_map)

                rcts_smi = rxn_smi.split('>>')[0]
                rcts_mol = Chem.MolFromSmiles(rcts_smi)
                [atom.ClearProp('molAtomMapNumber') for atom in rcts_mol.GetAtoms()]
                rcts_smi_remove_atom_map = Chem.MolToSmiles(rcts_mol, True)
                # Sometimes stereochem takes another canonicalization...
                rcts_smi_remove_atom_map = Chem.MolToSmiles(Chem.MolFromSmiles(rcts_smi_remove_atom_map), True)
                phase_rcts_smis.append(rcts_smi_remove_atom_map)

                rxn_smi_remove_atom_map = rcts_smi_remove_atom_map + '>>' + prod_smi_remove_atom_map
                phase_rxn_smi_remove_atom_map.append(rxn_smi_remove_atom_map)

            clean_50k_remove_atom_map[phase] = phase_rxn_smi_remove_atom_map
            prod_smiles[phase] = phase_prod_smis
            rcts_smiles[phase] = phase_rcts_smis

        self.clean_50k = clean_50k
        self.clean_50k_remove_atom_map = clean_50k_remove_atom_map
        self.prod_smiles = prod_smiles
        self.rcts_smiles = rcts_smiles
        self.all_prod_smiles = all_prod_smiles

    def propose_one(self,
                    prod_smiles: str,
                    topk: int = 200,
                    max_prec: int = 200) -> List[str]:
        ex = mol_from_smiles(prod_smiles)
        rct = rdchiralreactant_dist(prod_smiles)
        fp = self.getfp(prod_smiles)

        sims = self.similarity_metric(fp, [fp_ for fp_ in self.datasub['prod_fp']])
        js = np.argsort(sims)[::-1]

        # Get probability of precursors
        probs = {}
        for j in js[:max_prec]:
            jx = self.datasub.index[j]
            if jx in self.jx_cache:
                (template, rcts_ref_fp) = self.jx_cache[jx]
            else:
                retro_canonical = process_an_example(self.datasub['rxn_smiles'][jx], super_general=True)
                if retro_canonical is None: # cannot get template, most likely due to 'could not find consistent tetrahedral centre'
                    continue
                template = '(' + retro_canonical.replace('>>', ')>>')
                rcts_ref_fp = self.getfp(self.datasub['rxn_smiles'][jx].split('>')[0])
                self.jx_cache[jx] = (template, rcts_ref_fp)

            rxn = rdchiralreaction_dist(template)
            try:
                outcomes = rdchiralrun_dist(rxn, rct, combine_enantiomers=False)
            except Exception as e:
                print(e)
                outcomes = []

            for precursors in outcomes:
                precursors_fp = self.getfp(precursors)
                precursors_sim = self.similarity_metric(precursors_fp, [rcts_ref_fp])[0]
                if precursors in probs:
                    probs[precursors] = max(probs[precursors], precursors_sim * sims[j])
                else:
                    probs[precursors] = precursors_sim * sims[j]

        mols = []
        found_rank = 9999
        for r, (prec, prob) in enumerate(sorted(probs.items(), key=lambda x:x[1], reverse=True)[:topk]):
            mols.append(mol_from_smiles(prec))

        proposed_precursors_smiles = [mol_to_smiles(x, True) for x in mols]
        return proposed_precursors_smiles

    def propose_one_helper(self,
                        prod_smiles: str,
                        results: dict,
                        topk: int = 200,
                        max_prec: int = 200) -> Dict[str, List[str]]:
        ''' wrapper over self.propose_one() to allow parallelization within self.propose_all()
        '''
        results[prod_smiles] = self.propose_one(prod_smiles, topk=topk, max_prec=max_prec)
        return results

    def propose_all(self) -> None:
        ''' iterates through all product smiles in dataset (self.all_prod_smiles)
        and proposes precursors for them based on self.max_prec and self.topk

        Sets self.all_proposed_smiles upon successful execution
        '''
        if (self.output_folder /
            f'retrosim_proposed_smiles_{self.topk}maxtest_{self.max_prec}maxprec.pickle'
            ).exists(): # file already exists
            with open(self.output_folder /
                f'retrosim_proposed_smiles_{self.topk}maxtest_{self.max_prec}maxprec.pickle', 'rb'
                ) as handle:
                self.all_proposed_smiles = pickle.load(handle)
            self._compile_into_csv()

        else:
            all_proposed_smiles = {}
            if self.parallelize:
                # TODO: see if this can also be checkpointed from inside self.propose_one_helper()
                try:
                    num_workers = len(os.sched_getaffinity(0))
                except AttributeError:
                    num_workers = os.cpu_count()
                print(f"Parallelizing over {num_workers} cores")
                results = {}
                with tqdm_joblib(tqdm(desc="Generating Retrosim's Proposals", total=len(self.all_prod_smiles))) as progress_bar:
                    output_dicts = Parallel(n_jobs=num_workers)(
                        delayed(self.propose_one_helper)(
                                prod_smi, results, self.topk, self.max_prec
                            )
                        for prod_smi in self.all_prod_smiles
                    )
                for output_dict in output_dicts:
                    all_proposed_smiles.update(output_dict)
            else:
                for i, prod_smi in enumerate(tqdm(self.all_prod_smiles, desc="Generating Retrosim's Proposals...")):
                    if prod_smi in all_proposed_smiles and len(all_proposed_smiles[prod_smi]) > 0:
                        continue # no need to repeat calculation

                    all_proposed_smiles[prod_smi] = self.propose_one(prod_smi, self.topk, self.max_prec)

                    if i % 4000 == 0: # checkpoint temporary files
                        with open(self.output_folder / f'retrosim_proposed_smiles_{self.topk}maxtest_{self.max_prec}maxprec_{i}.pickle', 'wb') as handle:
                            pickle.dump(all_proposed_smiles, handle, protocol=pickle.HIGHEST_PROTOCOL)

            with open(
                    self.output_folder /
                    f'retrosim_proposed_smiles_{self.topk}maxtest_{self.max_prec}maxprec.pickle', 'wb'
                ) as handle:
                pickle.dump(all_proposed_smiles, handle, protocol=pickle.HIGHEST_PROTOCOL)

            with open(self.input_folder / 'jx_cache.pickle', 'wb') as handle:
                pickle.dump(jx_cache, handle, protocol=pickle.HIGHEST_PROTOCOL)

            self.all_proposed_smiles = all_proposed_smiles
            self._compile_into_csv()

    def _compile_into_csv(self):
        '''
        Sets self.proposed_precursors
        Also runs self._calc_accs()
        '''
        if self.all_proposed_smiles is None:
            with open(
                    self.output_folder /
                    f'retrosim_proposed_smiles_{self.topk}maxtest_{self.max_prec}maxprec.pickle', 'rb'
                ) as handle:
                self.all_proposed_smiles = pickle.load(handle)

        proposed_precursors = {}
        self.phases = ['train', 'valid', 'test']
        for phase in self.phases:
            dup_count = 0
            phase_proposed_precursors = []
            for rxn_smi in self.clean_50k_remove_atom_map[phase]:
                prod_smi = rxn_smi.split('>>')[-1]

                precursors = self.all_proposed_smiles[prod_smi]

                # check for duplicates - but by design, retrosim shouldn't make any duplicate proposal
                seen = []
                for prec in precursors: # no need to canonicalize bcos retrosim already canonicalized
                    if prec not in seen:
                        seen.append(prec)
                    else:
                        dup_count += 1

                if len(precursors) < self.topk:
                    precursors.extend(['9999'] * (self.topk - len(precursors)))

                phase_proposed_precursors.append(precursors)
            proposed_precursors[phase] = phase_proposed_precursors
            dup_count /= len(self.clean_50k_remove_atom_map[phase])
            print(f'Avg # dups per product for {phase}: {dup_count}') # should be 0
        self.proposed_precursors = proposed_precursors
        print('Compiled proposed_precursors by rxn_smi!')

        self.ranks, self.accs = self._calc_accs()
        # repeat accuracy calculation after removing ground truth predictions
        _, _ = self._calc_accs()

        combined = {}
        for phase in self.phases:
            zipped = []
            for rxn_smi, prod_smi, rcts_smi, rank_of_true_precursor, proposed_rcts_smi in zip(
                self.clean_50k[phase],
                self.prod_smiles[phase],
                self.rcts_smiles[phase],
                self.ranks[phase],
                proposed_precursors[phase],
            ):
                result = []
                result.extend([rxn_smi, prod_smi, rcts_smi, rank_of_true_precursor])
                result.extend(proposed_rcts_smi)
                zipped.append(result)

            combined[phase] = zipped
        print('Zipped all info for each rxn_smi into a list for dataframe creation!')

        processed_dataframes = {}
        for phase in self.phases:
            temp_dataframe = pd.DataFrame(
                data={
                    'zipped': combined[phase]
                }
            )

            phase_dataframe = pd.DataFrame(
                temp_dataframe['zipped'].to_list(),
                index=temp_dataframe.index
            )
            if phase == 'train': # true precursor has been removed from the proposals, so whatever is left are negatives
                proposed_col_names = [f'neg_precursor_{i}' for i in range(1, self.topk+1)]
            else: # validation/testing, we don't assume true precursor is present & we also do not remove them if present
                proposed_col_names = [f'cand_precursor_{i}' for i in range(1, self.topk+1)]

            base_col_names = ['orig_rxn_smi', 'prod_smi', 'true_precursors', 'rank_of_true_precursor']
            base_col_names.extend(proposed_col_names)
            phase_dataframe.columns = base_col_names

            processed_dataframes[phase] = phase_dataframe
            print(f'Shape of {phase} dataframe: {phase_dataframe.shape}')

            phase_dataframe.to_csv(
                self.output_folder /
                f'retrosim_{self.topk}maxtest_{self.max_prec}maxprec_{phase}.csv',
                index=False
            )
        print(f'Saved all proposals as 3 dataframes in {self.output_folder}!')

    def _calc_accs(self):
        '''
        Returns:
            ranks, accs
        '''
        ranks = {}
        for phase in self.phases:
            phase_ranks = []
            if phase == 'train':
                for idx in tqdm(range(len(self.clean_50k[phase])), desc=phase):
                    true_precursors = self.rcts_smiles[phase][idx]
                    all_proposed_precursors = self.proposed_precursors[phase][idx]

                    found = False
                    for rank, proposal in enumerate(all_proposed_precursors): # ranks are 0-indexed
                        if true_precursors == proposal:
                            phase_ranks.append(rank)
                            # remove true precursor from proposals
                            all_proposed_precursors.pop(rank)
                            all_proposed_precursors.append('9999')
                            found = True
                            break

                    if not found:
                        phase_ranks.append(9999)
                    self.proposed_precursors[phase][idx] = all_proposed_precursors
            else:
                for idx in tqdm(range(len(self.clean_50k[phase])), desc=phase):
                    true_precursors = self.rcts_smiles[phase][idx]
                    all_proposed_precursors = self.proposed_precursors[phase][idx]

                    found = False
                    for rank, proposal in enumerate(all_proposed_precursors): # ranks are 0-indexed
                        if true_precursors == proposal:
                            phase_ranks.append(rank)
                            # do not pop true precursor from proposals!
                            found = True
                            break

                    if not found:
                        phase_ranks.append(9999)
                    self.proposed_precursors[phase][idx] = all_proposed_precursors
            ranks[phase] = phase_ranks

        accs = {}
        for phase in self.phases:
            phase_accs = []
            for n in [1, 3, 5, 10, 20, 50]:
                total = float(len(ranks[phase]))
                phase_accs.append(sum([r+1 <= n for r in ranks[phase]]) / total)
                print(f'{phase} Top-{n} accuracy: {phase_accs[-1]*100:.3f}%')

            print('\n')
            accs[phase] = phase_accs

        return ranks, accs

    def analyse_proposed(self):
        if self.all_proposed_smiles is None:
            with open(
                    self.output_folder /
                    f'retrosim_proposed_smiles_{self.topk}maxtest_{self.max_prec}maxprec.pickle', 'rb'
                ) as handle:
                self.all_proposed_smiles = pickle.load(handle)

        proposed_counter = Counter()
        total_proposed, min_proposed, max_proposed = 0, float('+inf'), float('-inf')
        key_count = 0
        for key, value in tqdm(self.all_proposed_smiles.items()):
            precursors_count = len(value)
            total_proposed += precursors_count
            if precursors_count > max_proposed:
                max_proposed = precursors_count
                prod_smi_max = key
            if precursors_count < min_proposed:
                min_proposed = precursors_count
                prod_smi_min = key

            proposed_counter[key] = precursors_count
            key_count += 1

        print(f'Average precursors proposed per prod_smi: {total_proposed / key_count}')
        print(f'Min precursors: {min_proposed} for {prod_smi_min}')
        print(f'Max precursors: {max_proposed} for {prod_smi_max})')

        print(f'\nMost common 20:')
        for i in proposed_counter.most_common(20):
            print(f'{i}')
        print(f'\nLeast common 20:')
        for i in proposed_counter.most_common()[-20:]:
            print(f'{i}')

if __name__ == '__main__':
    retrosim_model = Retrosim(topk=200, max_prec=200, similarity_type='Tanimoto',
                            fp_type='Morgan2Feat')
    retrosim_model.prep_valid_and_test_data(input_data_file_prefix='50k_clean_rxnsmi_noreagent_allmapped')
    retrosim_model.propose_all()
    retrosim_model.analyse_proposed()


def main():
    dados = input().split(" ")
    A, B, C, D = map(int, dados)

    if ((B > C) and (D > A) and ((C + D) > (A + B)) and (C > 0) and (D > 0) and (A % 2 == 0)):
        print('Valores aceitos')
    else:
        print('Valores nao aceitos')

if __name__ == '__main__':
        main()


from __future__ import (absolute_import, division, print_function)
import os
from addie.utilities.file_handler import FileHandler
from addie.autoNOM.step1_widgets_handler import Step1WidgetsHandler

class AutoPopulateWidgets(object):

    input_file_name = 'exp.ini'
    file_found_message = "Config file %s has been found!" % input_file_name
    file_not_found_message = "Config file %s has  not been found! " % input_file_name

    def __init__(self, main_window=None):
        self.main_window = main_window
#        self.main_window.autonom_ui = parent.ui

    def run(self):
        full_file_name = os.path.join(self.main_window.current_folder, self.input_file_name)
        if os.path.exists(full_file_name):
            self.main_window.autonom_ui.exp_ini_file_status.setText(self.file_found_message)
            o_retriever = RetrieveExpIniConfiguration(exp_ini_file_name=full_file_name)
            o_retriever.run()
            self.populate_widgets(o_retriever.exp_ini_dico)
            return

        self.main_window.autonom_ui.exp_ini_file_status.setText(self.file_not_found_message)

    def populate_widgets(self, widgets_dico):
        self.main_window.autonom_ui.diamond.setText(widgets_dico['Dia'])
        self.main_window.autonom_ui.diamond_background.setText(widgets_dico['DiaBg'])
        self.main_window.autonom_ui.vanadium.setText(widgets_dico['Vana'])
        self.main_window.autonom_ui.vanadium_background.setText(widgets_dico['VanaBg'])
        self.main_window.autonom_ui.sample_background.setText(widgets_dico['MTc'])

        o_gui = Step1WidgetsHandler(parent=self.main_window)

        try:
            _recali = True if (widgets_dico['recali'].strip() == 'yes') else False
        except:
            _recali = False
        finally:
            o_gui.set_recalibration(_recali)

        try:
            _renorm = True if (widgets_dico['renorm'].strip() == 'yes') else False
        except:
            _renorm = False
        finally:
            o_gui.set_renormalization(_renorm)

        try:
            _auto = True if (widgets_dico['autotemp'].strip() == 'yes') else False
        except:
            _auto = False
        finally:
            o_gui.set_autotemplate(_auto)

        self.main_window.autonom_ui.first_scan.setText(widgets_dico['scan1'])
        self.main_window.autonom_ui.last_scan.setText(widgets_dico['scanl'])

        if str(self.main_window.autonom_ui.frequency.currentText()) == '60':
            self.main_window.autonom_ui.frequency.setCurrentIndex(0)
        else:
            self.main_window.autonom_ui.frequency.setCurrentIndex(1)

        try:
            _comments = widgets_dico['#']
        except:
            _comments = ''
        finally:
            self.main_window.autonom_ui.comments.setText(_comments)

class RetrieveExpIniConfiguration(object):

    exp_ini_dico = {}

    def __init__(self, exp_ini_file_name=None):
        self.full_file_name = exp_ini_file_name

    def run(self):
        o_file = FileHandler(filename=self.full_file_name)
        o_file.retrieve_contain()
        _file_contrain = o_file.file_contain

        self.retrieve_settings(_file_contrain)

    def retrieve_settings(self, file_contain):
        _exp_ini_dico = {}
        file_contain = file_contain.split("\n")
        for _line in file_contain:
            _parsed_line = _line.split()
            if len(_parsed_line) > 1:
                _keyword = _parsed_line[0]
                if _keyword == '#':
                    _value = " ".join(_parsed_line[1:])
                else:
                    _value = _parsed_line[1]
                _exp_ini_dico[_keyword] = _value

        self.exp_ini_dico = _exp_ini_dico


import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import matplotlib
np.random.seed(2)  #设置随机种子
zhfont1 = matplotlib.font_manager.FontProperties(fname='C:\Windows\Fonts\simsun.ttc')
data = [[0.3,0.2,0.2,0.25],
        [0.6,0.5,0.5,0.6],
        [0.95,0.47,1.3,0.7],
        [1.2,0.9,2.0,0.4],
        [1.8,0.7,2.9,0.8]]

df = pd.DataFrame((data),columns=['LmsLoc', 'SmartPDR', 'MaLoc', 'WiLoc'])
df.boxplot() #也可用plot.box()
plt.ylabel('Time(s)')
plt.show()


# -*- coding: utf-8 -*-
"""
Created on Sun Jun  5 17:32:07 2022

@author: jhazelde
"""

import torch
import torchvision
import numpy as np
import matplotlib.pyplot as plt

import os
import yaml

from lfads import LFADS_Net
from utils import read_data, load_parameters, save_parameters

from config import CFG
from bnn import HH_Gap
from torchviz import make_dot

def shutoff_stuff():
    def shutoff_test():
        CFG.sim_t = 4000
        bnn = HH_Gap(1).cuda()
        w = torch.ones(()) * 1.0
        w = w.cuda()
        w.requires_grad = True
        optim = torch.optim.Adam([w], lr=0.1)
        I = torch.ones((1, CFG.sim_t, 1)).cuda()
        for epoch in range(100):
            optim.zero_grad()
            if epoch > 0:
                print(w.grad.cpu().item())
            z = I * w
            out = bnn(z).squeeze()
            plt.plot(out.detach().cpu())
            plt.title(f'{w.cpu().item()}')
            plt.show()
            loss = torch.mean(out)
            loss.backward()
     #       make_dot(loss).render("loss", format="png")

            optim.step()
            print(epoch, w.item(), w.grad.cpu().item())

    def shutoff_sweep():
        CFG.sim_t = 4000
        bnn = HH_Gap(1).cuda()
        CFG.plot = False
        ws = np.linspace(0.9000104069709778, 0.9000104069709778+0.0001, 20)
        I = torch.ones((1, CFG.sim_t, 1)).cuda()
        losses = np.zeros(len(ws))
        for idx, w in enumerate(ws):
            print(idx)
            z = I * w
            out = bnn(z).squeeze()
            # plt.plot(out.detach().cpu())
            # plt.title(f'{w.cpu().item()}')
            # plt.show()
            loss = torch.mean(out)
            losses[idx] = loss
            if idx > 0:
                print((losses[idx] - losses[idx-1]) / (ws[idx] - ws[idx-1]))
        plt.plot(ws, losses)
        plt.show()
    shutoff_sweep()

# plt.style.use('dark_background')

device = 'cuda' if torch.cuda.is_available() else 'cpu'; print('Using device: %s'%device)
Ncells = 50
if False and os.path.exists('./synth_data/chaotic_rnn_300'):
    data_dict = read_data('./synth_data/chaotic_rnn_300')
else:
    if not os.path.isdir('./synth_data'):
        os.mkdir('./synth_data/')

    from synth_data_chaotic_rnn import generate_data
    data_dict = generate_data(T= 0.3, dt_rnn= 0.01, dt_cal= 0.01,
                              Ninits= 400, Ntrial= 10, Ncells=Ncells, trainp= 0.8,
                              tau=0.025, gamma=1.5, maxRate=30, B=20,
                              seed=300, save=True)

train_data = torch.Tensor(data_dict['train_spikes']).to(device)
valid_data = torch.Tensor(data_dict['valid_spikes']).to(device)

print(train_data.shape)

train_truth = torch.Tensor(data_dict['train_rates']).to(device)
valid_truth = torch.tensor(data_dict['valid_rates']).to(device)

train_ds      = torch.utils.data.TensorDataset(train_data)
valid_ds      = torch.utils.data.TensorDataset(valid_data)

num_trials, num_steps, num_cells = train_data.shape
print(train_data.shape)

plt.figure(figsize = (12,12))
plt.imshow(data_dict['train_spikes'][0].T, cmap=plt.cm.Greys)
plt.xticks(np.linspace(0, 100, 6), ['%.1f'%i for i in np.linspace(0, 1, 6)])
plt.xlabel('Time (s)')
plt.ylabel('Cell #')
plt.colorbar(orientation='horizontal', label='# Spikes in 0.01 s time bin')
plt.title('Example trial')
plt.show()

plt.figure(figsize = (12,12))
plt.imshow(data_dict['train_rates'][0].T, cmap=plt.cm.plasma)
plt.xticks(np.linspace(0, 100, 6), ['%.1f'%i for i in np.linspace(0, 1, 6)])
plt.xlabel('Time (s)')
plt.ylabel('Cell #')
plt.colorbar(orientation='horizontal', label='Firing Rate (Hz)')
plt.title('Example trial')
plt.show()

T = 0.3

def fitting_code():
    def compare_rates(r, out, k, plot=False):
        true_rates = torch.mean(r, 1).cuda().float()
        true_rates = (true_rates - torch.mean(true_rates)) / torch.std(true_rates)

        # spike_duration = 10 # Assume spikes take 0.5 ms
        # pred_rates = torch.sum(out[0, :, :], 0).float() / spike_duration
        # pred_rates = pred_rates / T

        pred_rates = torch.sum(out[0, :, :], 0).float()
        pred_rates = (pred_rates - torch.mean(pred_rates)) / torch.std(pred_rates)

        loss_fun = torch.nn.MSELoss()
        loss = loss_fun(pred_rates, true_rates) / 1000

    #    loss = torch.mean(out**2)

        if plot:
            plt.figure(dpi=150)
            plt.plot(true_rates.cpu().detach().numpy(), zorder=5, color='black', linewidth=1)
            plt.plot(pred_rates.cpu().detach().numpy(), zorder=5, color='red', linewidth=1)
            plt.title(f'k={k:.5f}, loss={loss.cpu().item():.5f}')

        return loss

    r = torch.from_numpy(data_dict['train_rates']).cuda()
    interp = torchvision.transforms.Resize((CFG.sim_t, Ncells))
    r = interp(r)

    def k_grad_sweep():
        CFG.plot = False
        bnn = HH_Gap(Ncells)

        k = torch.ones(()).float() * 0.1
        k.requires_grad = True

        ks = np.zeros(100)
        losses = np.zeros(len(ks))
        optim = torch.optim.SGD([k], lr=0.01)
        grads = np.zeros(len(ks))

        for idx in range(len(ks)):
            optim.zero_grad()
            CFG.dt = 0.1
            CFG.sim_t = int((T * 1000) / CFG.dt)

            r_sub = r[:10, :, :]
            z = torch.clamp(k * r_sub, 0.0, 1.8)

            out = bnn(z)
            # plt.imshow(out[0, :, :].cpu().detach().numpy(), aspect='auto')
            # plt.show()

            plt.plot(out[-1, :, 0].cpu().detach().numpy())
            plt.show()

            loss = compare_rates(r_sub, out, k, True)
            plt.show()
            losses[idx] = loss.cpu().item()
            ks[idx] = k.cpu().item()

            loss.backward()
            grad = k.grad
            grads[idx] = grad.cpu().item()
            print(idx, losses[idx], k.item(), grad.item())
            optim.step()

        #plt.plot(ks, losses)
        #plt.savefig(f'loss_gradient_find_2.pdf')
        #plt.show()
    k_grad_sweep()

    def k_simple_sweep(a, b, N=10):
        CFG.plot = False
        bnn = HH_Gap(Ncells)

        ks = np.linspace(a, b, N)
        losses = np.zeros(len(ks))

        slopes = []
        for idx, k in enumerate(ks):
            CFG.dt = 0.1
            CFG.sim_t = int((T * 1000) / CFG.dt)
            z = torch.clamp(k * r, 0.0, 1.8)
            out = bnn(z)

            plt.plot(out[-1, :, 0].cpu().detach().numpy())
            plt.show()

            loss = compare_rates(r, out, k, True)
            plt.show()
            losses[idx] = loss.cpu().item()

            print(idx, losses[idx], k)
            if idx > 0:
                slope = (losses[idx] - losses[idx-1]) / (ks[idx] - ks[idx-1])
                print(f'slope k={k}, {slope:.3f}')
                slopes.append(slope)

        plt.plot(ks, losses)
        for i in range(len(slopes)):
            plt.text((ks[i] + ks[i+1]) * 0.5, (losses[i] + losses[i+1]) * 0.5, f'{slopes[i]:.1f}', size='small')

        plt.ylabel('Loss')
        plt.title('Noisy Loss Landscape Smoother')
        plt.xlabel('k')
        plt.savefig(f'loss_simple_noise_sweep_{a}_{b}_{N}.pdf')
        plt.show()
        return ks, losses

    k = 0.0001
    def compare_rates_old(r, out, k, plot=False):
        true_rates = torch.mean(r[0, :, :], 0).cuda().float()
        true_rates = (true_rates - torch.mean(true_rates)) / torch.std(true_rates)

        pred_rates = torch.sum(out[0, :, :], 0).float()
        pred_rates = (pred_rates - torch.mean(pred_rates)) / torch.std(pred_rates)

        plt.figure(dpi=150)
        plt.plot(true_rates.cpu().detach().numpy(), zorder=5, color='black', linewidth=1, label='True Rates')
        plt.plot(pred_rates.cpu().detach().numpy(), zorder=5, color='red', linewidth=1, label='BNN Output Rates')
        plt.title('Before Training')
        plt.xlabel('Cell #')
        plt.yticks([])
        plt.legend()

    CFG.dt = 0.1
    CFG.sim_t = int((T * 1000) / CFG.dt)
    bnn = HH_Gap(Ncells).cuda()
    z = torch.clamp(k * r, 0.0, 1.8)
    out = bnn(z)
    compare_rates_old(r, out, k, True)
    plt.show()
fitting_code()