# -*- coding: utf-8 -*-
"""Tests for the LifeScan OneTouch Ultra 2 driver."""

__author__ = 'Diego Elio Pettenò'
__email__ = 'flameeyes@flameeyes.eu'
__copyright__ = 'Copyright © 2013, Diego Elio Pettenò'
__license__ = 'MIT'

import os
import sys
import unittest

import mock

sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from glucometerutils.drivers import otultra2
from glucometerutils.support import lifescan
from glucometerutils import exceptions

class TestOTUltra2(unittest.TestCase):
    def setUp(self):
        self.addCleanup(mock.patch.stopall)

        mock_serial = mock.patch('serial.Serial').start()
        self.mock_readline = mock_serial.return_value.readline

        self.device = otultra2.Device('mockdevice')

    def _set_return_string(self, string):
        self.mock_readline.return_value = bytes(string, 'ascii')

    def test_checksum(self):
        checksum = otultra2._calculate_checksum(bytes('T', 'ascii'))
        self.assertEqual(0x0054, checksum)

        checksum = otultra2._calculate_checksum(
            bytes('T "SAT","08/03/13","22:12:00   "', 'ascii'))
        self.assertEqual(0x0608, checksum)

    def test_missing_checksum(self):
        self._set_return_string('INVALID')

        self.assertRaises(lifescan.MissingChecksum,
                          self.device.get_serial_number)

    def test_short_response(self):
        self._set_return_string('.\r')

        self.assertRaises(exceptions.InvalidResponse,
                          self.device.get_serial_number)

    def test_invalid_response(self):
        self._set_return_string('% 2500\r')

        self.assertRaises(exceptions.InvalidResponse,
                          self.device.get_serial_number)

    def test_invalid_serial_number(self):
        self._set_return_string('@ "12345678O" 0297\r')

        self.assertRaises(lifescan.InvalidSerialNumber,
                          self.device.get_serial_number)

    def test_invalid_checksum(self):
        self._set_return_string('% 1337\r')

        self.assertRaises(exceptions.InvalidChecksum,
                          self.device.get_serial_number)

    def test_broken_checksum(self):
        self._set_return_string('% 13AZ\r')

        self.assertRaises(lifescan.MissingChecksum,
                          self.device.get_serial_number)

if __name__ == '__main__':
    unittest.main()


from typing import Optional, List

from highcharts_core.options.series.base import SeriesBase
from highcharts_core.options.series.data.treegraph import TreegraphData
from highcharts_core.options.plot_options.treegraph import TreegraphOptions
from highcharts_core.utility_functions import mro__to_untrimmed_dict

class TreegraphSeries(SeriesBase, TreegraphOptions):
    """General options to apply to all :term:`Treegraph` series types.

    A treegraph visualizes a relationship between ancestors and descendants with a clear parent-child relationship,
    e.g. a family tree or a directory structure.

    .. figure:: ../../../_static/treegraph-example.png
      :alt: Treegraph Example Chart
      :align: center

    """

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    @property
    def data(self) -> Optional[List[TreegraphData]]:
        """Collection of data that represents the series. Defaults to
        :obj:`None <python:None>`.

        While the series type returns a collection of :class:`TreegraphData` instances,
        it accepts as input:

        .. tabs::

          .. tab:: 1D Array of Arrays

            A one-dimensional collection where each member of the collection is itself
            a collection of data points.

            .. note::

              If using the Array of Arrays pattern you *must* set
              :meth:`.keys <highcharts_core.options.series.treegraph.TreegraphSeries.keys>` to indicate
              which value in the inner array corresponds to
              :meth:`.id <highcharts_core.options.series.treegraph.TreegraphSeries.id>`,
              :meth:`.parent <highcharts_core.options.series.treegraph.TreegraphSeries.parent>`, or
              :meth:`.name <highcharts_core.options.series.treegraph.TreegraphSeries.name>`.

          .. tab:: Object Collection

            A one-dimensional collection of :class:`TreegraphData` objects or
            :class:`dict <python:dict>` instances coercable to :class:`TreegraphData`

        :rtype: :class:`list <python:list>` of :class:`TreegraphData` or
          :obj:`None <python:None>`
        """
        return self._data

    @data.setter
    def data(self, value):
        if not value:
            self._data = None
        else:
            self._data = TreegraphData.from_array(value)

    @classmethod
    def _get_kwargs_from_dict(cls, as_dict):
        kwargs = {
            'accessibility': as_dict.get('accessibility', None),
            'allow_point_select': as_dict.get('allowPointSelect', None),
            'animation': as_dict.get('animation', None),
            'class_name': as_dict.get('className', None),
            'clip': as_dict.get('clip', None),
            'color': as_dict.get('color', None),
            'cursor': as_dict.get('cursor', None),
            'custom': as_dict.get('custom', None),
            'dash_style': as_dict.get('dashStyle', None),
            'data_labels': as_dict.get('dataLabels', None),
            'description': as_dict.get('description', None),
            'enable_mouse_tracking': as_dict.get('enableMouseTracking', None),
            'events': as_dict.get('events', None),
            'include_in_data_export': as_dict.get('includeInDataExport', None),
            'keys': as_dict.get('keys', None),
            'label': as_dict.get('label', None),
            'legend_symbol': as_dict.get('legendSymbol', None),
            'linked_to': as_dict.get('linkedTo', None),
            'marker': as_dict.get('marker', None),
            'on_point': as_dict.get('onPoint', None),
            'opacity': as_dict.get('opacity', None),
            'point': as_dict.get('point', None),
            'point_description_formatter': as_dict.get('pointDescriptionFormatter', None),
            'selected': as_dict.get('selected', None),
            'show_checkbox': as_dict.get('showCheckbox', None),
            'show_in_legend': as_dict.get('showInLegend', None),
            'skip_keyboard_navigation': as_dict.get('skipKeyboardNavigation', None),
            'sonification': as_dict.get('sonification', None),
            'states': as_dict.get('states', None),
            'sticky_tracking': as_dict.get('stickyTracking', None),
            'tooltip': as_dict.get('tooltip', None),
            'turbo_threshold': as_dict.get('turboThreshold', None),
            'visible': as_dict.get('visible', None),

            'animation_limit': as_dict.get('animationLimit', None),
            'boost_blending': as_dict.get('boostBlending', None),
            'boost_threshold': as_dict.get('boostThreshold', None),
            'color_index': as_dict.get('colorIndex', None),
            'crisp': as_dict.get('crisp', None),
            'crop_threshold': as_dict.get('cropThreshold', None),
            'find_nearest_point_by': as_dict.get('findNearestPointBy', None),
            'get_extremes_from_all': as_dict.get('getExtremesFromAll', None),
            'relative_x_value': as_dict.get('relativeXValue', None),
            'soft_threshold': as_dict.get('softThreshold', None),
            'step': as_dict.get('step', None),

            'point_interval': as_dict.get('pointInterval', None),
            'point_interval_unit': as_dict.get('pointIntervalUnit', None),
            'point_start': as_dict.get('pointStart', None),
            'stacking': as_dict.get('stacking', None),

            'allow_traversing_tree': as_dict.get('allowTraversingTree', None),
            'collapse_button': as_dict.get('collapseButton', None),
            'color_by_point': as_dict.get('colorByPoint', None),
            'fill_space': as_dict.get('fillSpace', None),
            'link': as_dict.get('link', None),
            'reversed': as_dict.get('reversed', None),

            'data': as_dict.get('data', None),
            'id': as_dict.get('id', None),
            'index': as_dict.get('index', None),
            'legend_index': as_dict.get('legendIndex', None),
            'name': as_dict.get('name', None),

        }

        return kwargs

    def _to_untrimmed_dict(self, in_cls = None) -> dict:
        untrimmed = mro__to_untrimmed_dict(self, in_cls = in_cls) or {}

        return untrimmed


import torch
from typing import Tuple

def precompute_freqs_cis(dim: int, end: int, theta: float) -> torch.Tensor:
    freqs = 1.0 / (theta ** (torch.arange(0, dim, 2)[: (dim // 2)].float() / dim))
    t = torch.arange(end, device=freqs.device)  # type: ignore
    freqs = torch.outer(t, freqs).float()  # type: ignore
    return torch.polar(torch.ones_like(freqs), freqs)  # complex64

def apply_rotary_emb(
    xq: torch.Tensor,
    xk: torch.Tensor,
    freqs_cis: torch.Tensor,
) -> Tuple[torch.Tensor, torch.Tensor]:
    xq_ = torch.view_as_complex(xq.float().reshape(*xq.shape[:-1], -1, 2))
    xk_ = torch.view_as_complex(xk.float().reshape(*xk.shape[:-1], -1, 2))
    freqs_cis = freqs_cis[:, None, :]
    xq_out = torch.view_as_real(xq_ * freqs_cis).flatten(2)
    xk_out = torch.view_as_real(xk_ * freqs_cis).flatten(2)
    return xq_out.type_as(xq), xk_out.type_as(xk)


# -*- coding: utf-8 -*-
# @Time    : 2021/11/25 20:35
# @Author  : Meng Jianing
# @FileName: request.py
# @Software: PyCharm
# @Versions: v0.1
# @Github  ：https://github.com/NekoSilverFox
# --------------------------------------------

class Request:
    """请求"""

    num_request = 0  # 累计生成请求的数量
    num_cancel_request = 0

    def __init__(self, source, request_id):
        """初始化一个请求

        :param source: 生成这个请求的源
        :param request_id: 分配给这个请求的 ID（从 1 开始编号）
        """
        self.source = source        # 产生这个请求的源
        self.request_id_in_source = request_id        # 这个请求在它源中的ID
        Request.num_request += 1
        self.request_id_in_cmo = Request.num_request    # 在整个 CMO 中的唯一 ID

    def __str__(self):
        """ 返回说明当前请求(Request)的字符串

        :return: 说明当前请求(Request)的字符串
        """
        return format('\033[36;1m[Request]\033[0m Number in CMO: %s' % self.request_id_in_cmo, '<45') \
               + format('Source: %s' % self.source.id, '<20')\
               + format('Request ID in source: %s' % self.request_id_in_source, '<30')

    @staticmethod
    def reset():
        """ 将所有 static 的属性恢复到默认值，谨慎调用！

        :return: 无
        """
        Request.num_request = 0
        Request.num_cancel_request = 0


class Node:
    def __init__(self,key,value) :
        self.key = key
        self.value = value
        self.next = None

class Seperatechaining :
    def __init__(self,capacity) :
        self.capacity =capacity
        self.size = 0
        self.table = [None] * capacity

    def hashh(self,key) :
        capacity = 10
        value = 0
        for i in key :
            value = (value + ord(i)+ 23) % capacity
        return value

    def insert(self,key,value) :
        index = self.hashh(key)
        newnode = Node(key ,value)
        if self.table[index] is None :
            self.table[index] = newnode
            print(f"Inserted {newnode.value}")
        else:
            current =self.table[index]
            while current :
                if current.key == key :
                    current.value = value
                    return
                current = current.next

            newnode.next = self.table[index]
            self.table[index] = newnode
            print(f"Inserted okeyy {value}")
            self.size +=1

    def remove(self,key) :
        index = self.hashh(key)
        current = self.table[index]
        previous = None
        if self.table[index] is not None :
            while current :
                if current.key == key :
                    if previous :
                        previous.next = current.next
                    else :
                        self.table[index] = current .next
                    return
                previous = current
                current = current.next

    def search(self,key) :
        index = self.hashh(key)
        current = self.table[index]
        while current :
            if current.key == key :
                print("value", current.value)
            current = current.next

    def display(self) :
        for key ,value in enumerate(self.table) :
            if value is not None :
                print(f"Key- {key} : Index- {value}")

sc=Seperatechaining(10)
sc.insert("rambo" , 23)
sc.insert("raju" ,2)
print("gau")
sc.insert("gautham" ,3)
print("dis")
sc.display()
sc.search("raju")
sc.remove("raju")
sc.display()


import random

class Maze:

    def __init__(self, width, height):
        self.width = width // 2 * 2 + 1
        self.height = height // 2 * 2 + 1
        self.cells = [
            [True for x in range(self.width)] for y in range(self.height)
        ]

    def set_path(self, x, y):
        self.cells[y][x] = False

    def set_wall(self, x, y):
        self.cells[y][x] = True

    def is_wall(self, x, y):
        if 0 <= x < self.width and 0 <= y < self.height:
            return self.cells[y][x]
        else:
            return False

    def create_maze(self, x, y):
        self.set_path(x, y)
        all_directions = [[1, 0], [-1, 0], [0, 1], [0, -1]]
        random.shuffle(all_directions)
        while len(all_directions) > 0:
            direction_to_try = all_directions.pop()
            node_x = x + (direction_to_try[0] * 2)
            node_y = y + (direction_to_try[1] * 2)
            if self.is_wall(node_x, node_y):
                link_cell_x = x + direction_to_try[0]
                link_cell_y = y + direction_to_try[1]
                self.set_path(link_cell_x, link_cell_y)
                self.create_maze(node_x, node_y)
        return self.cells


import queue
import shlex
import ssl
import subprocess
import sys
import time
from abc import ABC, abstractmethod
from shutil import copyfileobj
from threading import Thread
from typing import Tuple, Union
from urllib.error import URLError
from urllib.request import urlopen

from downloader.constants import K_DOWNLOADER_RETRIES, K_DOWNLOADER_SIZE_MB_LIMIT, \
    K_DOWNLOADER_PROCESS_LIMIT, K_DOWNLOADER_TIMEOUT, K_CURL_SSL, K_DEBUG, FILE_MiSTer_new, FILE_MiSTer, \
    FILE_MiSTer_old, K_DOWNLOADER_OLD_IMPLEMENTATION, K_DOWNLOADER_THREADS_LIMIT
from downloader.logger import DebugOnlyLoggerDecorator
from downloader.other import calculate_url
from downloader.target_path_repository import TargetPathRepository

class FileDownloaderFactory(ABC):
    @abstractmethod
    def create(self, config, parallel_update, silent=False, hash_check=True):
        """Created a Parallel or Serial File Downloader"""

def make_file_downloader_factory(file_system_factory, local_repository, waiter, logger):
    return _FileDownloaderFactoryImpl(file_system_factory, local_repository, waiter, logger)

class _FileDownloaderFactoryImpl(FileDownloaderFactory):
    def __init__(self, file_system_factory, local_repository, waiter, logger):
        self._file_system_factory = file_system_factory
        self._local_repository = local_repository
        self._waiter = waiter
        self._logger = logger

    def create(self, config, parallel_update, silent=False, hash_check=True):
        logger = DebugOnlyLoggerDecorator(self._logger) if silent else self._logger
        file_system = self._file_system_factory.create_for_config(config)
        target_path_repository = TargetPathRepository(config, file_system)
        if config[K_DOWNLOADER_OLD_IMPLEMENTATION]:
            self._logger.print('Using old downloader implementation...')
            if parallel_update:
                return _CurlCustomParallelDownloader(config, file_system, self._local_repository, logger, hash_check, target_path_repository)
            else:
                return _CurlSerialDownloader(config, file_system, self._local_repository, logger, hash_check, target_path_repository)
        else:
            thread_limit = config[K_DOWNLOADER_THREADS_LIMIT] if parallel_update else 1
            low_level_factory = _LowLevelMultiThreadingFileDownloaderFactory(thread_limit, config, self._waiter, logger)
            return HighLevelFileDownloader(hash_check, config, file_system, target_path_repository, low_level_factory, logger)

class FileDownloader(ABC):
    @abstractmethod
    def queue_file(self, file_description, file_path):
        """queues a file for downloading it later"""

    @abstractmethod
    def set_base_files_url(self, base_files_url):
        """sets the base_files_url from a database"""

    @abstractmethod
    def mark_unpacked_zip(self, zip_id, base_zips_url):
        """indicates that a zip is being used, useful for reporting"""

    @abstractmethod
    def download_files(self,  first_run):
        """download all the queued files"""

    @abstractmethod
    def errors(self):
        """all files with errors"""

    @abstractmethod
    def correctly_downloaded_files(self):
        """all correctly downloaded files"""

    @abstractmethod
    def needs_reboot(self):
        """returns true if a file that needs reboot has been downloaded"""

class LowLevelFileDownloader(ABC):
    def fetch(self, files_to_download, paths):
        """"files_to_download is a dictionary with file_path as keys and file_description as values"""

    def network_errors(self):
        """returns errors that happened during download_files"""

    def downloaded_files(self):
        """returns files downloaded during download_files"""

class LowLevelFileDownloaderFactory(ABC):
    def create_low_level_file_downloader(self, high_level):
        """"returns instance of LowLevelFileDownloader"""

class DownloadValidator(ABC):
    def validate_download(self, file_path: str, file_hash: str) -> Tuple[int, Union[str, Tuple[str, str]]]:
        """Validates that the downloaded file is correctly installed and moves it if necessary. Returned int is 1 if validation was correct."""

class HighLevelFileDownloader(FileDownloader, DownloadValidator):
    def __init__(self, hash_check, config, file_system, target_path_repository, low_level_file_downloader_factory, logger):
        self._hash_check = hash_check
        self._file_system = file_system
        self._target_path_repository = target_path_repository
        self._config = config
        self._low_level_file_downloader_factory = low_level_file_downloader_factory
        self._logger = logger
        self._run_files = []
        self._queued_files = {}
        self._base_files_url = None
        self._unpacked_zips = {}
        self._needs_reboot = False
        self._errors = []
        self._correct_files = []

    def queue_file(self, file_description, file_path):
        self._queued_files[file_path] = file_description

    def set_base_files_url(self, base_files_url):
        self._base_files_url = base_files_url

    def mark_unpacked_zip(self, zip_id, base_zips_url):
        self._unpacked_zips[zip_id] = base_zips_url

    def download_files(self, _):
        for _ in range(self._config[K_DOWNLOADER_RETRIES] + 1):
            self._errors = self._download_try()
            if len(self._errors):
                continue

            break

        if self._file_system.is_file(FILE_MiSTer_new):
            self._logger.print('')
            self._logger.print('Copying new MiSTer binary:')
            if self._file_system.is_file(FILE_MiSTer):
                self._file_system.move(FILE_MiSTer, FILE_MiSTer_old)
            self._file_system.move(FILE_MiSTer_new, FILE_MiSTer)

            if self._file_system.is_file(FILE_MiSTer):
                self._logger.print('New MiSTer binary copied.')
            else:
                # This error message should never happen.
                # If it happens it would be an unexpected case where file_system is not moving files correctly
                self._logger.print('CRITICAL ERROR!!! Could not restore the MiSTer binary!')
                self._logger.print('Please manually rename the file MiSTer.new as MiSTer')
                self._logger.print('Your system won\'nt be able to boot until you do so!')
                sys.exit(1)

    def _download_try(self):
        if len(self._queued_files) == 0:
            self._logger.print("Nothing new to download from given sources.")
            return []

        self._logger.print("Downloading %d files:" % len(self._queued_files))

        low_level = self._low_level_file_downloader_factory.create_low_level_file_downloader(self)

        files_to_download = []
        skip_files = []
        for file_path, file_description in self._queued_files.items():
            if self._hash_check and self._file_system.is_file(file_path):
                path_hash = self._file_system.hash(file_path)
                if path_hash == file_description['hash']:
                    if 'zip_id' in file_description and file_description['zip_id'] in self._unpacked_zips:
                        self._logger.print('Unpacked: %s' % file_path)
                    else:
                        self._logger.print('No changes: %s' % file_path)  # @TODO This scenario might be redundant now, since it's also checked in the Online Importer
                    skip_files.append(file_path)
                    continue
                else:
                    self._logger.debug('%s: %s != %s' % (file_path, file_description['hash'], path_hash))

            if 'url' not in file_description:
                file_description['url'] = calculate_url(self._base_files_url, file_path)

            self._file_system.make_dirs_parent(file_path)
            target_path = self._target_path_repository.create_target(file_path, file_description)
            files_to_download.append((file_path, self._file_system.download_target_path(target_path)))
            self._run_files.append(file_path)

        for file_path in skip_files:
            self._correct_files.append(file_path)
            self._queued_files.pop(file_path)

        low_level.fetch(files_to_download, self._queued_files)

        self._check_downloaded_files(low_level.downloaded_files())

        return low_level.network_errors()

    def validate_download(self, file_path: str, file_hash: str) -> Tuple[int, Union[str, Tuple[str, str]]]:
        target_path = self._target_path_repository.access_target(file_path)
        if not self._file_system.is_file(target_path):
            return 2, (file_path, 'Missing %s' % file_path)

        path_hash = self._file_system.hash(target_path)
        if self._hash_check and path_hash != file_hash:
            self._target_path_repository.clean_target(file_path)
            return 2, (file_path, 'Bad hash on %s (%s != %s)' % (file_path, file_hash, path_hash))

        self._target_path_repository.finish_target(file_path)
        self._logger.debug('+', end='', flush=True)

        return 1, file_path

    def _check_downloaded_files(self, files):
        for path in files:
            self._correct_files.append(path)
            if self._queued_files[path].get('reboot', False):
                self._needs_reboot = True

            self._queued_files.pop(path)

    def errors(self):
        return self._errors

    def correctly_downloaded_files(self):
        return self._correct_files

    def needs_reboot(self):
        return self._needs_reboot

    def run_files(self):
        return self._run_files

def context_from_curl_ssl(curl_ssl):
    context = ssl.create_default_context()

    if curl_ssl.startswith('--cacert '):
        cacert_file = curl_ssl[len('--cacert '):]
        context.load_verify_locations(cacert_file)
    elif curl_ssl == '--insecure':
        context.check_hostname = False
        context.verify_mode = ssl.CERT_NONE

    return context

class _LowLevelMultiThreadingFileDownloaderFactory(LowLevelFileDownloaderFactory):
    def __init__(self, threads_limit, config, waiter, logger):
        self._threads_limit = threads_limit
        self._config = config
        self._waiter = waiter
        self._logger = logger

    def create_low_level_file_downloader(self, download_validator):
        return _LowLevelMultiThreadingFileDownloader(self._threads_limit, self._config, context_from_curl_ssl(self._config[K_CURL_SSL]), self._waiter, self._logger, download_validator)

class _LowLevelMultiThreadingFileDownloader(LowLevelFileDownloader):
    def __init__(self, threads_limit, config, context, waiter, logger, download_validator):
        self._threads_limit = threads_limit
        self._config = config
        self._context = context
        self._waiter = waiter
        self._logger = logger
        self._download_validator = download_validator
        self._network_errors = _DownloadErrors(self._logger)
        self._downloaded_files = []
        self._pending_notifications = []
        self._endl_pending = False

    def fetch(self, files_to_download, descriptions):
        job_queue = queue.Queue()
        notify_queue = queue.Queue()
        for path, target in files_to_download:
            job_queue.put((descriptions[path]['url'], path, target), False)

        threads = [Thread(target=self._thread_worker, args=(job_queue, notify_queue)) for _ in range(min(self._threads_limit, len(files_to_download)))]
        for thread in threads:
            thread.start()

        remaining_notifications = len(files_to_download) * 2

        while remaining_notifications > 0:
            remaining_notifications -= self._read_notifications(descriptions, notify_queue, True)
            self._waiter.sleep(1)

        job_queue.join()

        for thread in threads:
            thread.join()

        self._read_notifications(descriptions, notify_queue, False)
        self._logger.print()

    def _read_notifications(self, descriptions, notify_queue, in_progress):
        new_files = False
        read_notifications = 0
        while not notify_queue.empty():
            state, path = notify_queue.get(False)
            notify_queue.task_done()

            if state == 0:
                if self._endl_pending:
                    self._endl_pending = False
                    self._logger.print()
                self._logger.print(path, flush=True)

                new_files = True
            elif state == 1:
                self._pending_notifications.append(self._download_validator.validate_download(path, descriptions[path]['hash']))
            else:
                self._pending_notifications.append((state, path))

            read_notifications += 1

        if new_files:
            return read_notifications

        if len(self._pending_notifications) > 0:
            for state, pack in self._pending_notifications:
                if state == 1:
                    path = pack
                    self._downloaded_files.append(path)
                    self._logger.print('.', end='', flush=True)
                else:
                    path, message = pack
                    self._network_errors.add_debug_report(path, message)
                    self._logger.print('~', end='', flush=True)

        elif in_progress:
            self._logger.print('*', end='', flush=True)

        self._endl_pending = in_progress
        self._pending_notifications.clear()
        return read_notifications

    def network_errors(self):
        return self._network_errors.list()

    def downloaded_files(self):
        return self._downloaded_files

    def _thread_worker(self, job_queue, notify_queue):
        while not job_queue.empty():
            url, path, target = job_queue.get(False)

            notify_queue.put((0, path), False)
            try:
                with urlopen(url, timeout=self._config[K_DOWNLOADER_TIMEOUT], context=self._context) as in_stream, open(target, 'wb') as out_file:
                    if in_stream.status == 200:
                        copyfileobj(in_stream, out_file)
                        notify_queue.put((1, path), False)
                    else:
                        notify_queue.put((2, (path, 'Bad http status! %s: %s' % (path, in_stream.status))), False)

            except URLError as e:
                notify_queue.put((2, (path, 'HTTP error! %s: %s' % (path, e.reason))), False)
            except ConnectionResetError as e:
                notify_queue.put((2, (path, 'Connection reset error! %s: %s' % (path, str(e)))), False)
            except Exception as e:
                notify_queue.put((2, (path, 'Exception during download! %s: %s' % (path, str(e)))), False)

            job_queue.task_done()

class CurlDownloaderAbstract(FileDownloader):
    def __init__(self, config, file_system, local_repository, logger, hash_check, temp_files_registry):
        self._config = config
        self._file_system = file_system
        self._logger = logger
        self._local_repository = local_repository
        self._hash_check = hash_check
        self._temp_files_registry = temp_files_registry
        self._curl_list = {}
        self._errors = _DownloadErrors(logger)
        self._http_oks = _HttpOks()
        self._correct_downloads = []
        self._needs_reboot = False
        self._base_files_url = None
        self._unpacked_zips = dict()

    def queue_file(self, file_description, file_path):
        self._curl_list[file_path] = file_description

    def set_base_files_url(self, base_files_url):
        self._base_files_url = base_files_url

    def mark_unpacked_zip(self, zip_id, base_zips_url):
        self._unpacked_zips[zip_id] = base_zips_url

    def download_files(self, first_run):
        self._download_files_internal(first_run)

        if self._file_system.is_file(FILE_MiSTer_new):
            self._logger.print()
            self._logger.print('Copying new MiSTer binary:')
            if self._file_system.is_file(FILE_MiSTer):
                self._file_system.move(FILE_MiSTer, FILE_MiSTer_old)
            self._file_system.move(FILE_MiSTer_new, FILE_MiSTer)

            if self._file_system.is_file(FILE_MiSTer):
                self._logger.print('New MiSTer binary copied.')
            else:
                # This error message should never happen.
                # If it happens it would be an unexpected case where file_system is not moving files correctly
                self._logger.print('CRITICAL ERROR!!! Could not restore the MiSTer binary!')
                self._logger.print('Please manually rename the file MiSTer.new as MiSTer')
                self._logger.print('Your system won\'nt be able to boot until you do so!')
                sys.exit(1)

    def _download_files_internal(self, first_run):
        if len(self._curl_list) == 0:
            self._logger.print("Nothing new to download from given sources.")
            return

        self._logger.print("Downloading %d files:" % len(self._curl_list))

        for path in sorted(self._curl_list):
            description = self._curl_list[path]
            if self._hash_check and self._file_system.is_file(path):
                path_hash = self._file_system.hash(path)
                if path_hash == description['hash']:
                    if 'zip_id' in description and description['zip_id'] in self._unpacked_zips:
                        self._logger.print('Unpacked: %s' % path)
                    else:
                        self._logger.print('No changes: %s' % path)  # @TODO This scenario might be redundant now, since it's also checked in the Online Importer
                    self._correct_downloads.append(path)
                    continue
                else:
                    self._logger.debug('%s: %s != %s' % (path, description['hash'], path_hash))

            if first_run:
                if 'delete' in description:
                    for _ in description['delete']:  # @TODO This is Deprecated
                        self._file_system.delete_previous(path)
                        break
                elif 'delete_previous' in description and description['delete_previous']:
                    self._file_system.delete_previous(path)

            self._download(path, description)

        self._wait()
        self._check_hashes()

        for retry in range(self._config[K_DOWNLOADER_RETRIES]):

            if self._errors.none():
                return

            for path in self._errors.consume():
                self._download(path, self._curl_list[path])

            self._wait()
            self._check_hashes()

    def _check_hashes(self):
        if self._http_oks.none():
            return

        self._logger.print()
        self._logger.print('Checking hashes...')

        for path in self._http_oks.consume():
            if not self._file_system.is_file(self._temp_files_registry.access_target(path)):
                self._errors.add_debug_report(path, 'Missing %s' % path)
                continue

            path_hash = self._file_system.hash(self._temp_files_registry.access_target(path))
            if self._hash_check and path_hash != self._curl_list[path]['hash']:
                self._errors.add_debug_report(path, 'Bad hash on %s (%s != %s)' % (path, self._curl_list[path]['hash'], path_hash))
                self._temp_files_registry.clean_target(path)
                continue

            self._temp_files_registry.finish_target(path)
            self._logger.print('+', end='', flush=True)
            self._correct_downloads.append(path)
            if self._curl_list[path].get('reboot', False):
                self._needs_reboot = True

        self._logger.print()

    def _download(self, path, description):
        if 'zip_path' in description:
            raise FileDownloaderError('zip_path is not a valid field for the file "%s", please contain the DB maintainer' % path)

        self._logger.print(path)
        self._file_system.make_dirs_parent(path)

        if 'url' not in description:
            description['url'] = calculate_url(self._base_files_url, path)

        target_path = self._temp_files_registry.create_target(path, description)

        if self._config[K_DEBUG] and target_path.startswith('/tmp/') and not description['url'].startswith('http'):
            self._file_system.copy(description['url'], target_path)
            self._run(description, 'echo > /dev/null', path)
            return

        self._run(description, self._command(target_path, description['url']), path)

    def _command(self, target_path, url):
        return 'curl %s --show-error --fail --location -o "%s" "%s"' % (self._config[K_CURL_SSL], target_path, url)

    def errors(self):
        return self._errors.list()

    def correctly_downloaded_files(self):
        return self._correct_downloads

    def needs_reboot(self):
        return self._needs_reboot

    @abstractmethod
    def _wait(self):
        """"waits until all downloads are completed"""

    @abstractmethod
    def _run(self, description, command, path):
        """"starts the downloading process"""

class _CurlCustomParallelDownloader(CurlDownloaderAbstract):
    def __init__(self, config, file_system, local_repository, logger, hash_check, temp_file_registry):
        super().__init__(config, file_system, local_repository, logger, hash_check, temp_file_registry)
        self._processes = []
        self._files = []
        self._acc_size = 0

    def _run(self, description, command, file):
        self._acc_size = self._acc_size + description['size']

        result = subprocess.Popen(shlex.split(command), shell=False, stderr=subprocess.DEVNULL,
                                  stdout=subprocess.DEVNULL)

        self._processes.append(result)
        self._files.append(file)

        more_accumulated_size_than_limit = self._acc_size > (1000 * 1000 * self._config[K_DOWNLOADER_SIZE_MB_LIMIT])
        more_processes_than_limit = len(self._processes) > self._config[K_DOWNLOADER_PROCESS_LIMIT]

        if more_accumulated_size_than_limit or more_processes_than_limit:
            self._wait()

    def _wait(self):
        count = 0
        start = time.time()
        while count < len(self._processes):
            some_completed = False
            for i, p in enumerate(self._processes):
                if p is None:
                    continue
                result = p.poll()
                if result is not None:
                    self._processes[i] = None
                    some_completed = True
                    count = count + 1
                    start = time.time()
                    self._logger.print('.', end='', flush=True)
                    if result == 0:
                        self._http_oks.add(self._files[i])
                    else:
                        self._errors.add_debug_report(self._files[i], 'Bad http code! %s: %s' % (result, self._files[i]))
            end = time.time()
            if (end - start) > self._config[K_DOWNLOADER_TIMEOUT]:
                for i, p in enumerate(self._processes):
                    if p is None:
                        continue
                    self._errors.add_debug_report(self._files[i], 'Timeout! %s' % self._files[i])
                break

            time.sleep(1)
            if not some_completed:
                self._logger.print('*', end='', flush=True)

        self._logger.print(flush=True)
        self._processes = []
        self._files = []
        self._acc_size = 0

class _CurlSerialDownloader(CurlDownloaderAbstract):
    def __init__(self, config, file_system, local_repository, logger, hash_check, temp_file_registry):
        super().__init__(config, file_system, local_repository, logger, hash_check, temp_file_registry)

    def _run(self, description, command, file):
        result = subprocess.run(shlex.split(command), shell=False, stderr=subprocess.STDOUT)
        if result.returncode == 0:
            self._http_oks.add(file)
        else:
            self._errors.add_print_report(file, 'Bad http code! %s: %s' % (result.returncode, file))

        self._logger.print()

    def _wait(self):
        pass

class _DownloadErrors:
    def __init__(self, logger):
        self._logger = logger
        self._errors = []

    def add_debug_report(self, path, message):
        self._logger.print('~', end='', flush=True)
        self._logger.debug(message, flush=True)
        self._errors.append(path)

    def add_print_report(self, path, message):
        self._logger.print(message, flush=True)
        self._errors.append(path)

    def none(self):
        return len(self._errors) == 0

    def consume(self):
        errors = self._errors
        self._errors = []
        return errors

    def list(self):
        return self._errors

class _HttpOks:
    def __init__(self):
        self._oks = []

    def add(self, path):
        self._oks.append(path)

    def consume(self):
        oks = self._oks
        self._oks = []
        return oks

    def none(self):
        return len(self._oks) == 0

class FileDownloaderError(Exception):
    pass


from pathlib import Path

from downloader.config import default_config, UpdateLinuxEnvironment
from downloader.constants import K_DATABASES, K_DB_URL, K_SECTION, K_VERBOSE, K_CONFIG_PATH, K_USER_DEFINED_OPTIONS, \
    K_COMMIT, K_UPDATE_LINUX_ENVIRONMENT, K_FAIL_ON_FILE_ERROR, K_UPDATE_LINUX
from downloader.full_run_service import FullRunService as ProductionFullRunService
from test.fake_os_utils import SpyOsUtils
from test.fake_waiter import NoWaiter
from test.fake_external_drives_repository import ExternalDrivesRepository
from test.fake_file_downloader_factory import FileDownloaderFactory
from test.fake_importer_implicit_inputs import FileSystemState
from test.fake_base_path_relocator import BasePathRelocator
from test.fake_db_gateway import DbGateway
from test.fake_file_system_factory import FileSystemFactory
from test.fake_linux_updater import LinuxUpdater
from test.fake_local_repository import LocalRepository
from test.fake_logger import NoLogger
from test.fake_online_importer import OnlineImporter
from test.fake_offline_importer import OfflineImporter
from test.fake_reboot_calculator import RebootCalculator
from test.objects import db_empty
from test.fake_certificates_fix import CertificatesFix

class FullRunService(ProductionFullRunService):
    def __init__(self, config=None, db_gateway=None, file_system_factory=None, linux_updater=None, os_utils=None, certificates_fix=None, external_drives_repository=None):
        config = config or default_config()
        file_system_factory = FileSystemFactory() if file_system_factory is None else file_system_factory
        system_file_system = file_system_factory.create_for_system_scope()
        file_downloader_factory = FileDownloaderFactory(file_system_factory=file_system_factory)
        linux_updater = linux_updater or LinuxUpdater(system_file_system)
        super().__init__(config,
                         NoLogger(),
                         LocalRepository(config=config, file_system=system_file_system),
                         db_gateway or DbGateway(config, file_system_factory=file_system_factory),
                         OfflineImporter(file_downloader_factory=file_downloader_factory),
                         OnlineImporter(file_system_factory=file_system_factory),
                         linux_updater,
                         RebootCalculator(file_system=system_file_system),
                         BasePathRelocator(),
                         certificates_fix or CertificatesFix(),
                         external_drives_repository or ExternalDrivesRepository(file_system=system_file_system),
                         os_utils or SpyOsUtils(),
                         NoWaiter())

    @staticmethod
    def with_single_empty_db() -> ProductionFullRunService:
        config = default_config()
        config.update({
            K_DATABASES: {
                db_empty: {
                    K_DB_URL: db_empty,
                    K_SECTION: db_empty,
                    'base_files_url': '',
                    'zips': {}
                }
            },
            K_VERBOSE: False,
            K_CONFIG_PATH: Path(''),
            K_USER_DEFINED_OPTIONS: [],
            K_COMMIT: 'test', K_UPDATE_LINUX_ENVIRONMENT: UpdateLinuxEnvironment.TRUE, K_FAIL_ON_FILE_ERROR: True
        })

        file_system_state = FileSystemState(files={db_empty: {'unzipped_json': {}}})
        file_system_factory = FileSystemFactory(state=file_system_state)

        return FullRunService(
            config,
            DbGateway(config, file_system_factory=file_system_factory),
            file_system_factory=file_system_factory
        )

    @staticmethod
    def with_single_db(db_id, db_descr, linux_updater=None, linux_update_environment=None, update_linux=None, os_utils=None, certificates_fix=None) -> ProductionFullRunService:
        update_linux = update_linux if update_linux is not None else True
        config = default_config()
        config.update({
                K_DATABASES: {
                    db_id: {
                        K_DB_URL: db_id,
                        K_SECTION: db_id,
                        'base_files_url': '',
                        'zips': {}
                    }
                },
                K_VERBOSE: False,
                K_USER_DEFINED_OPTIONS: [],
                K_CONFIG_PATH: Path(''),
                K_COMMIT: 'test',
                K_UPDATE_LINUX: update_linux,
                K_UPDATE_LINUX_ENVIRONMENT: linux_update_environment or UpdateLinuxEnvironment.TRUE,
                K_FAIL_ON_FILE_ERROR: True
            })
        return FullRunService(
            config,
            DbGateway.with_single_db(db_id, db_descr, config=config),
            linux_updater=linux_updater,
            os_utils=os_utils,
            certificates_fix=certificates_fix
        )

    @staticmethod
    def with_no_dbs() -> ProductionFullRunService:
        config = default_config()
        config.update({
            K_DATABASES: {}, K_VERBOSE: False, K_CONFIG_PATH: Path(''), K_USER_DEFINED_OPTIONS: [],
            K_COMMIT: 'test', K_UPDATE_LINUX_ENVIRONMENT: UpdateLinuxEnvironment.TRUE, K_FAIL_ON_FILE_ERROR: True
        })
        return FullRunService(
            config,
            DbGateway(config),
        )


# -*- coding: utf-8 -*-
"""
@author: ali_shehzad
"""

"""Finger exercise 13: Implement a function that meets the specification below.
   Use a try-except block."""

def sumDigits(s):
    """Assumes s is a string
       Returns the sum of the decimal digits in s
       For example is s is 'a2b3c' it returns 5"""
    total = 0
    for i in s:
        try:
            total += int(i)
            print(total)
        except ValueError:
            print("Ignoring exception")
    return total

Week4- Good Programming Practices/Lecture8- Exceptions and Assertions/Summing Digits in a String.py


import urllib.request
import json
import os
import ssl
from decouple import config

def allowSelfSignedHttps(allowed):
    # bypass the server certificate verification on client side
    if (
        allowed
        and not os.environ.get("PYTHONHTTPSVERIFY", "")
        and getattr(ssl, "_create_unverified_context", None)
    ):
        ssl._create_default_https_context = ssl._create_unverified_context

def Calculate(form_data):
    allowSelfSignedHttps(
        True
    )  # this line is needed if you use self-signed certificate in your scoring service.

    # Request data goes here
    # The example below assumes JSON formatting which may be updated
    # depending on the format your endpoint expects.
    # More information can be found here:
    # https://docs.microsoft.com/azure/machine-learning/how-to-deploy-advanced-entry-script

    data = {
        "Inputs": {"data": [form_data]},
        "GlobalParameters": 0.0,
    }
    print(data)
    body = str.encode(json.dumps(data))

    url = "http://c12792b1-ec04-4c37-ac90-8140af8e7225.centralindia.azurecontainer.io/score"
    # Replace this with the primary/secondary key or AMLToken for the endpoint
    api_key = config("API_KEY")
    if not api_key:
        raise Exception("A key should be provided to invoke the endpoint")

    headers = {
        "Content-Type": "application/json",
        "Authorization": ("Bearer " + api_key),
    }

    req = urllib.request.Request(url, body, headers)

    try:
        response = urllib.request.urlopen(req)

        result = response.read()
        return result
    except urllib.error.HTTPError as error:
        print("The request failed with status code: " + str(error.code))

        # Print the headers - they include the requert ID and the timestamp, which are useful for debugging the failure
        print(error.info())
        print(error.read().decode("utf8", "ignore"))


from enum import Enum

users = []
class Account(Enum):
    USD = "USD"
    KZT = "KZT"
    RUB = "RUB"
    EUR = "EUR"

class BankAccount:
    name: str
    surname: str
    amount: int = 0
    account: Account = 'KZT'

    def __init__(self, name:str, surname:str, account:Account) -> None:
        self.name = name
        self.surname = surname
        self.account = account

    def set_user(self, name: str, surname: str, account: Account) -> None:
        self.name = name
        self.surname = surname
        self.account = account
    def set_amount(self, amount:int):
        self.amount = amount

    def name(self) -> str:
        return self.name
    def surname(self) -> str:
        return self.surname
    def account(self) -> Account:
        return self.account
    def amount(self) -> int:
        return self.amount

    def addToBankAccount(self, x:int):
        self.amount += x
        print("Счет успешно пополнен")

    def substractFromBankAccount(self, x:int):
        if self.amount < x:
            print("Недостаточно средств")
        else:
            self.amount -= x
            print("Вы успешно сняли деньги")

    def moneyConversion(self, b):
        a = self.account
        kurs_kzt = {"KZT":1, "RUB":7.53, "USD":470.69, "EUR":496.17}
        kurs_rub = {"RUB":1, "KZT":0.13, "USD":62.52, "EUR":65.90 }
        kurs_usd = {"RUB":0.016, "KZT":0.0021, "USD":1, "EUR":1.05 }
        kurs_eur = {"RUB":0.015, "KZT":0.0020, "USD":0.95, "EUR":1}
        if b == "KZT":
            self.amount *= kurs_kzt[a]
            self.account = "KZT"
        elif b == "RUB":
            self.amount *= kurs_rub[a]
            self.account = "RUB"
        elif b == "USD":
            self.amount *= kurs_usd[a]
            self.account = "USD"
        elif b == "EUR":
            self.amount *= kurs_eur[a]
            self.account = "EUR"

    def __repr__(self):
        return f'{self.name} {self.surname} {self.amount} {self.account}'

def create_account(name: str, surname: str, amount:int, account: Account) -> BankAccount:
    user = BankAccount(name=name, surname=surname, account=account)
    user.set_amount(amount=amount)
    users.append(user)

    return user

def get_user(name: str, surname: str) -> BankAccount | None:
    user = next((u for u in users if name == u.name and surname == u.surname), None)
    if not user:
        print('User not found')
        return
    return f'{user.name} {user.surname}. Ваш счет: {user.amount} {user.account}'

def delete_user(name: str, surname: str) -> BankAccount | None:
    user = next((u for u in users if name == u.name and surname == u.surname), None)
    if not user:
        print('User not found')
        return
    users.remove(user)
    print("Пользователь удален")

fake_account = BankAccount(name="Mark", surname="Doe", account="RUB")

fake_account.amount = 1500
users.append(fake_account)

d = {"1":"KZT", "2":"RUB", "3":"USD", "4":"EUR"}
while(True):
    inp = input("Выберите действие: \n 1. Создание пользователя \n 2. Выбрать пользователя \n 3. Удалить пользователя \n 0. Выход \n")
    if inp == '0':
        break
    elif inp == "1":
        name = input("Введите имя: ")
        surname = input("Введите фмаилия: ")
        v = input("Выберите курс валют: \n 1. KZT \n 2. RUB \n 3. USD \n 4. EUR \n")
        account = d[v]
        user = create_account(name=name, surname=surname, account=account, amount=0)
        print("Ваш аккаунт создан. ")
    elif inp == '2':
        name = input("Введите имя: ")
        surname = input("Введите фмаилия: ")
        print(get_user(name=name, surname=surname))
        while(True):
            inp2 = input("Выберите операцию: \n 1. Добавить на счет \n 2. Снять деньги \n 3. Сконвертировать \n 0. Назад\n")
            if inp2 == "0":
                break
            elif inp2 == "1":
                x = int(input("Введите суммy: \n"))
                user.addToBankAccount(x)
                print(get_user(name=name, surname=surname))
            elif inp2 == "2":
                x = int(input("Введите суммy: \n"))
                user.substractFromBankAccount(x)
                print(get_user(name=name, surname=surname))
            elif inp2 == "3":
                b = input("Выберите курс валют: \n 1. KZT \n 2. RUB \n 3. USD \n 4. EUR \n")
                account = d[b]
                user.moneyConversion(account)
                print(get_user(name=name, surname=surname))
    elif inp == "3":
        name = input("Введите имя: ")
        surname = input("Введите фмаилия: ")
        delete_user(name=name, surname=surname)


from __future__ import unicode_literals

import unittest
import os

import dxfgrabber

filename = os.path.join(os.path.dirname(__file__), "assure_3d_coords.dxf")
DWG = dxfgrabber.readfile(filename, {"assure_3d_coords": True})

pcoords = [(1., 1., 0.), (-3., 2., 0.), (7., -1., 0.), (10., 10., 0.)]

class TestAssure3dCoords(unittest.TestCase):
    def test_line(self):
        line = [e for e in DWG.entities if e.dxftype == 'LINE'][0]
        self.assertEqual((1., 1., 0.), line.start)
        self.assertEqual((2., 2., 0.), line.end)

    def test_circle(self):
        circle = [e for e in DWG.entities if e.dxftype == 'CIRCLE'][0]
        self.assertEqual((12., 24., 0.), circle.center)

    def test_lwpolyline(self):
        # LWPOLYLINE can not return 3d coordinates (x, y, start_width, end_width, bulge)
        lwpolyline = [e for e in DWG.entities if e.dxftype == 'LWPOLYLINE'][0]
        self.assertEqual(pcoords, lwpolyline.points)

    def test_polyline2d(self):
        polyline = [e for e in DWG.entities if e.dxftype == 'POLYLINE'][0]
        self.assertEqual(pcoords, list(polyline.points))

if __name__ == '__main__':
    unittest.main()


import numpy as np
import matplotlib.pyplot as plt
import ipywidgets as wd
import pandas as pd
from IPython.display import display, update_display, Javascript, HTML
from inspect import signature
from graphviz import Digraph
import scipy.stats as sp
import plotly.express as px
import plotly.graph_objects as go
import warnings
import re

'''
Documentation for instructors:

How to set up a causal network:
1.

How to initialise the causal network in a notebook:
1. This .py script has to be imported
from causality_simulation2 import *
2. Define init_data = { 'node_name': [array_of_values], 'node_name2': [array_of_values], ... }, where the node_names have to be those causal nodes that have init=True, and the array of values have to be the fixed initial sample data, e.g. heights of students or coordinates of trees

How to set up an experiment:
1. experiment_name = Experiment(causal_network_name, init_data)
Every new experiment needs a new instance of Experiment
2. For group assignment
experiment_name.assignment()
Add argument config=[assignment_group1, assignment_group2, ...] for fixed (greyed out) assignment
assignment_group = { 'name': 'Name of group', 'samples_str': '1-10,15,20-30' }
If samples_str == '' for all groups, then samples are randomly assigned
3. For experimental setup
experiment_name.setting(show=['Name of node to show', ...])
Add argument disable='all' to disallow editing of settings
Add argument config=[intervention_group1, intervention_group2, ...], where intervention_groups have format { 'name': 'Name of group', 'intervention': { 'Name of node': ['fixed', 0], ...} }
See code for detail
4. For plotting of collected data
experiment_name.plot(show=['Name of node to show', ...])
'''

display(HTML('''<style>
    [title="Assigned samples:"] { min-width: 150px; }
</style>'''))

def dialog(title, body, button):
    display(Javascript("require(['base/js/dialog'], function(dialog) {dialog.modal({title: '%s', body: '%s', buttons: {'%s': {}}})});" % (title, body, button)))

class CausalNode:
    def __init__(self, vartype, func, name, causes=None, min=0, max=100, categories=[], init=False):
        '''
        name: string, must be unique
        vartype: 'categorical', 'discrete', 'continuous'
        causes: (node1, ..., nodeN)
        func: f
        f is a function of N variables, matching the number of nodes and their types, returns a single number matching the type of this node
        self.network: {node_name: node, ...}, all nodes that the current node depends on
        init: True/False, whether variable is an initial immutable attribute
        '''
        # n_func_args = len(signature(func).parameters)
        # n_causes = 0 if causes == None else len(causes)
        # if n_func_args != n_causes:
        #   raise ValueError('The number of arguments in func does not match the number of causes.')
        self.name = name
        self.causes = causes
        self.func = func
        self.network = self.nodeDict()
        self.vartype = vartype
        self.min = min
        self.max = max
        self.categories = categories
        self.init = init

    def traceNetwork(self):
        '''
        Generates set of all nodes that current node depends on
        '''
        nodes = {self}
        if self.causes != None:
            for c in self.causes:
                nodes = nodes.union(c.traceNetwork())
        return nodes

    def nodeDict(self):
        '''
        Generates a dictionary of name, node pairs for easier lookup of nodes by name
        '''
        nodes = self.traceNetwork()
        network = {}
        for n in nodes:
            network[n.name] = n
        return network

    def generateSingle(self, fix={}):
        '''
        Generates a single multidimensional data point. Returns dict of name, value pairs
        fix: {node_name: val, ...}
        '''
        data = {}
        while len(data) != len(self.network):
            for m, n in self.network.items():
                if m not in data.keys():
                    if n.causes == None:
                        if m not in fix.keys():
                            data[m] = n.func()
                        else:
                            data[m] = fix[m]
                    else:
                        ready = True
                        for c in n.causes:
                            if c.name not in data.keys():
                                ready = False
                                break
                        if ready:
                            parents_val = [data[c.name] for c in n.causes]
                            if m not in fix.keys():
                                data[m] = n.func(*parents_val)
                            else:
                                data[m] = fix[m]
        return data

    def generate(self, n, intervention={}):
        '''
        Generates n data points. Returns dict of name, np.array(values) pairs
        intervention: {node_name: [type, other_args]}
        intervention format:
        ['fixed', val] (val could be number or name of category)
        ['range', start, end]
        ['array', [...]] array size must be n
        '''
        fix_all = {} # {name: [val, ...], ...}
        for name, args in intervention.items():
            if args[0] == 'fixed':
                fix_all[name] = np.array([args[1] for i in range(n)])
            elif args[0] == 'range':
                fix_all[name] = np.random.permutation(np.linspace(args[1], args[2], n))
                if self.vartype == 'discrete':
                    fix_all[name] = np.rint(fix_all[name])
            elif args[0] == 'array':
                fix_all[name] = np.array(args[1])
        fixes = [None] * n # Convert to [{name: val, ...}, ...]
        for i in range(n):
            fixes[i] = {}
            for name, arr in fix_all.items():
                fixes[i][name] = arr[i]
        data_dicts = [self.generateSingle(fix=fix) for fix in fixes]
        data = {}
        for name in self.network:
            data[name] = np.array([d[name] for d in data_dicts])
        return pd.DataFrame(data)

    def drawNetwork(self):
        g = Digraph(name=self.name)
        def draw_edges(node, g):
            if node.causes:
                for cause in node.causes:
                    g.edge(cause.name, node.name)
                    draw_edges(cause, g)
        draw_edges(self, g)
        return g

class CausalNetwork:
    def __init__(self, root_node):
        self.root_node = root_node
        self.init_attr = [name for name, node in self.root_node.network.items() if node.init] # List of immutable attributes

    def drawNetwork(self):
        return self.root_node.drawNetwork()

    def generate(self, init_data, config, runs):
        '''
        Performs experiment many times (runs) according to config, returns data [i][group][var]
        config: dict {'name': group_name, 'samples_str': '1-100', 'intervention': {...}}
        '''
        self.data = []
        for i in range(runs):
            exp = Experiment(self, init_data)
            is_random = ''.join([g['samples_str'] for g in config]) == ''
            samples = randomAssign(exp.N, len(config)) if is_random else [text2Array(g['samples_str']) for g in config]
            groups = [{'name': config[i]['name'], 'samples': samples[i]} for i in range(len(config))]
            exp.setAssignment(groups)
            exp.doExperiment(config)
            self.data.append(exp.data)

    def statsContinuous(self, group, varx, vary):
        '''
        Calculates distribution of Pearson r and p-value between varx and vary (names of variables)
        '''
        runs = len(self.data)
        results = np.zeros((runs, 2))
        for i in range(runs):
            x = self.data[i][group][varx]
            y = self.data[i][group][vary]
            results[i] = sp.pearsonr(x, y)
        fig, ax = plt.subplots(1, 2, figsize=(14, 5))
        fig.suptitle(vary + ' vs. ' + varx + ', ' + str(runs) + ' runs')
        ax[0].hist(results[:,0])
        ax[0].set_title('Pearson r')
        ax[1].hist(np.log(results[:,1]))
        ax[1].set_title('log(p)')

    # def statsAB(self, group0, group1, var):
    #     '''
    #     Calculates distribution of Welch's t and p-value of var between the null hypothesis (group0) and intervention (group1)
    #     '''
    #     runs = len(self.data)
    #     results = np.zeros((runs, 2))
    #     for i in range(runs):
    #         a = self.data[i][group0][var]
    #         b = self.data[i][group1][var]
    #         results[i] = sp.ttest_ind(a, b, equal_var=False)
    #     fig, ax = plt.subplots(1, 2, figsize=(14, 5))
    #     fig.suptitle(var + ' between groups ' + group0 + ' and ' + group1 + ', ' + str(runs) + ' runs')
    #     ax[0].hist(results[:,0])
    #     ax[0].set_title("Welch's t")
    #     ax[1].hist(np.log(results[:,1]))
    #     ax[1].set_title('log(p)')

    def statsAB(self, group0, group1, var, resamples=1000):
        '''
        Permutation test
        '''
        runs = len(self.data)
        results = np.zeros((runs, 2))
        for i in range(runs):
            a = self.data[i][group0][var]
            b = self.data[i][group1][var]
            na = len(a)
            nb = len(b)
            sample_all = np.concatenate((a, b))
            results[i,0] = abs(np.mean(a) - np.mean(b))
            mean_diffs = np.zeros(resamples)
            for j in range(resamples):
                permuted = np.random.permutation(sample_all)
                mean_diffs[j] = abs(np.mean(permuted[0:na]) - np.mean(permuted[na+1:]))
            results[i,1] = np.sum(mean_diffs>=results[i,0]) / resamples
        fig, ax = plt.subplots(1, 2, figsize=(14, 5))
        fig.suptitle(var + ' between groups ' + group0 + ' and ' + group1 + ', ' + str(runs) + ' runs')
        ax[0].hist(results[:,0])
        ax[0].set_title("Difference in mean")
        ax[1].hist(np.log(results[:,1]))
        ax[1].set_title('log(p)') # p = probability that a random assignment into A, B groups will give (abs) mean greater than the observed one

class Experiment:
    def __init__(self, network, init_data):
        '''
        init_data: dict of name, array to initialise basic immutable attributes. Keys must match init_attr in instance of Network
        '''
        self.node = network.root_node
        l = []
        for key, arr in init_data.items():
            l.append(len(arr))
        if max(l) != min(l):
            raise ValueError('Every array in init_data must have the same length.')
        self.init_data = init_data
        if set(init_data.keys()) != set(network.init_attr):
            raise ValueError("init_data doesn't match the causal network's init_attr.")
        self.N = l[0] # Sample size
        self.data = {} # {group_name: {node_name: [val, ...], ...}, ...}
        self.assigned = False
        self.done = False
        self.p = None

    def assignment(self, config=None, hide_random=False):
        '''
        UI for group assignment of samples
        config: list of dicts, each being {'name': group_name, 'samples_str': string}
        samples_str: e.g. '1-25,30,31-34', if all groups have empty string '' then assume randomise
        '''
        self.group_assignment = groupAssignment(self)
        if config is not None:
            self.group_assignment.setAssignment(config, hide_random)
            self.submitAssignment()

    def setAssignment(self, groups):
        '''
        Sets assignment into self.groups without UI
        groups: list of dicts, each being {'name': group_name, samples: [array]}
        '''
        self.groups = groups
        seen = set()
        self.group_ids = dict()
        for i in range(len(self.groups)):
            name = self.groups[i]['name']
            if name not in seen:
                seen.add(name)
                self.group_ids[name] = i
            else:
                dialog('Duplicate group names', 'Some of the groups have been given the same name. Please choose a unique name for each group.', 'OK')
                return
        self.group_names = list(self.group_ids.keys())

    def submitAssignment(self, sender=None):
        '''
        Collects the group assignments from UI
        self.groups: list of dicts, each being {'name': group_name, samples: [array]}
        self.group_ids: dict {'group_name': id} for easier reverse lookup
        Checks for duplicate group names
        '''
        self.setAssignment(self.group_assignment.getAssignment())
        self.assigned = True
        # Populate self.data for plotOrchard
        for g in self.groups:
            mask = [i in g['samples'] for i in range(self.N)]
            d = dict()
            for node_name, arr in self.init_data.items():
                d[node_name] = arr[mask]
            d['id'] = np.array(g['samples'])+1
            self.data[g['name']] = pd.DataFrame(d)
        if self.p:
            self.p.updateAssignments()
        else:
            if self.node.name == 'Success Rate':
                self.plotAssignment(plot='Basketball')
            else:
                self.plotAssignment()

    def plotAssignment(self, plot='Truffula'):
        '''
        Can be implemented differently in different scenarios
        '''
        self.p = assignmentPlot(self, plot)

    def setting(self, show='all', config=None, disable=[]):
        '''
        Let user design experiment
        disabled: array of names
        show: array of names
        '''
        if not self.assigned:
            dialog('Groups not assigned', 'You have not yet assigned any groups! Click on "Visualise assignment" before running this box.', 'OK')
            return
        disable = self.node.network if disable == 'all' else disable
        self.intervention_setting = interventionSetting(self, show=show, disable=disable)
        if config is not None:
            self.intervention_setting.setIntervention(config)
            self.doExperiment(config)

    def doExperiment(self, intervention, msg=False):
        '''
        Perform experiment under intervention
        intervention: list of dictionaries, each being {'name': group_name, 'intervention': {'node_name', [...]}}
        '''
        self.data = dict()
        for g in intervention:
            j = self.group_ids[g['name']]
            mask = [i in self.groups[j]['samples'] for i in range(self.N)]
            for node_name, arr in self.init_data.items():
                g['intervention'][node_name] = ['array', arr[mask]]
            N_samples = len(self.groups[self.group_ids[g['name']]]['samples'])
            self.data[g['name']] = self.node.generate(N_samples, intervention=g['intervention'])
        self.done = True
        if msg:
            display(wd.Label(value='Data from experiment collected!'))

    def plot(self, show='all'):
        '''
        Plots data after doExperiment has been called
        '''
        if not self.done:
            dialog('Experiment not performed', 'You have not yet performed the experiment! Click on "Perform experiment" before running this box.', 'OK')
            return
        p = interactivePlot(self, show)
        self.p = p
        p.display()

    def plotOrchard(self, gradient=None, show='all'):
        '''
        Takes in the name of the group in the experiment and the name of the
        variable used to create the color gradient
        '''
        if not self.done:
            dialog('Experiment not performed', 'You have not yet performed the experiment! Click on "Perform experiment" before running this box.', 'OK')
            return
        o = orchardPlot(self, gradient=gradient, show=show)
        self.o = o
        o.display()

class groupAssignment:
    def __init__(self, experiment):
        '''
        UI for group assignment of samples
        submitAssignment: callback function
        '''
        self.experiment = experiment
        wd.Label(value='Sample size: %d' % self.experiment.N)
        self.randomise_button = wd.Button(description='Randomise assignment', layout=wd.Layout(width='180px'))
        self.group_assignments = [singleGroupAssignment(1)]
        self.add_group_button = wd.Button(description='Add another group')
        self.submit_button = wd.Button(description='Visualise assignment')
        self.box = wd.VBox([g.box for g in self.group_assignments])
        display(self.randomise_button, self.box, self.add_group_button, self.submit_button)
        self.randomise_button.on_click(self.randomise)
        self.add_group_button.on_click(self.addGroup)
        self.submit_button.on_click(self.experiment.submitAssignment)

    def setAssignment(self, config, hide_random):
        for i in range(len(config)-1):
            self.addGroup()
        self.greyAll()
        for i in range(len(config)):
            self.group_assignments[i].setName(config[i]['name'])
        if ''.join([g['samples_str'] for g in config]) == '':
            self.randomise()
        else:
            for i in range(len(config)):
                self.group_assignments[i].setSamples(config[i]['samples_str'])
        if hide_random:
            self.randomise_button.layout.visibility = 'hidden'

    def addGroup(self, sender=None):
        i = self.group_assignments[-1].i
        self.group_assignments.append(singleGroupAssignment(i+1))
        self.box.children = [g.box for g in self.group_assignments]

    def getAssignment(self):
        '''
        Reads the settings and returns a list of dictionaries
        '''
        return [g.getAssignment() for g in self.group_assignments]

    def randomise(self, sender=None):
        '''
        Randomly assigns samples to groups and changes settings in UI
        '''
        N = self.experiment.N
        N_group = len(self.group_assignments)
        assigned_ids = randomAssign(N, N_group)
        for i in range(N_group):
            self.group_assignments[i].samples.value = array2Text(assigned_ids[i])

    def greyAll(self):
        self.randomise_button.disabled = True
        self.add_group_button.disabled = True
        self.submit_button.disabled = True
        for g in self.group_assignments:
            g.greyAll()

class singleGroupAssignment:
    def __init__(self, i):
        '''
        UI for a single line of group assignment
        '''
        self.i = i # Group number
        self.name = 'Group %d' % i
        i_text = wd.Label(value=self.name, layout=wd.Layout(width='70px'))
        self.group_name = wd.Text(description='Name:')
        self.samples = wd.Text(description='Assigned samples:', layout=wd.Layout(width='400px'))
        self.box = wd.HBox([i_text, self.group_name, self.samples])

    def getAssignment(self):
        '''
        Returns dict {'name': group_name, 'samples': [list_of_sample_ids]}
        '''
        assignment = dict()
        self.name = self.name if self.group_name.value == '' else self.group_name.value
        assignment['name'] = self.name
        assignment['samples'] = text2Array(self.samples.value)
        return assignment

    def setName(self, name):
        self.group_name.value = name

    def setSamples(self, samples):
        self.samples.value = samples

    def greyAll(self):
        self.group_name.disabled = True
        self.samples.disabled = True

class interventionSetting:
    def __init__(self, experiment, show='all', disable=[]):
        self.experiment = experiment
        self.group_settings = [singleGroupInterventionSetting(self.experiment, g, show=show, disable=disable) for g in self.experiment.groups]
        submit = wd.Button(description='Perform experiment')
        display(submit)
        submit.on_click(self.submit)

    def submit(self, sender=None):
        self.experiment.doExperiment(self.getIntervention(), msg=True)

    def getIntervention(self):
        return [{'name': s.name, 'N': s.N, 'intervention': s.getIntervention()} for s in self.group_settings]

    def setIntervention(self, config):
        for c in config:
            j = self.experiment.group_ids[c['name']]
            self.group_settings[j].setIntervention(c)

class singleGroupInterventionSetting:
    def __init__(self, experiment, config, show='all', disable=[]):
        '''
        UI settings for a single group
        config: {'name': group_name, 'samples': [sample_ids]}
        '''
        self.experiment = experiment
        self.name = config['name']
        self.N = len(config['samples'])
        group_text = wd.Label(value='Group name: %s, %d samples' % (self.name, self.N))
        display(group_text)
        to_list = list(self.experiment.node.network.keys()) if show == 'all' else show
        to_list.sort()
        self.node_settings = [singleNodeInterventionSetting(self.experiment.node.network[name], disable=name in disable) for name in to_list]

    def getIntervention(self):
        intervention = dict()
        for s in self.node_settings:
            inter = s.getIntervention()
            if inter is not None:
                intervention[s.name] = inter
        return intervention

    def setIntervention(self, config):
        for s in self.node_settings:
            if s.name in config['intervention'].keys():
                s.setIntervention(config['intervention'][s.name])

class singleNodeInterventionSetting:
    def __init__(self, node, disable=False):
        '''
        Single line of radio buttons and text boxes for intervening on a single variable in a single group
        '''
        self.name = node.name
        self.disable = disable
        self.is_categorical = node.vartype == 'categorical'
        self.indent = wd.Label(value='', layout=wd.Layout(width='20px'))
        self.text = wd.Label(value=self.name, layout=wd.Layout(width='180px'))
        self.none = wd.RadioButtons(options=['No intervention'], layout=wd.Layout(width='150px'))
        self.fixed = wd.RadioButtons(options=['Fixed'], layout=wd.Layout(width='70px'))
        self.fixed.index = None
        if self.is_categorical:
            fixed_arg = wd.Dropdown(options=node.categories, disabled=True, layout=wd.Layout(width='100px'))
        else:
            fixed_arg = wd.BoundedFloatText(disabled=True, layout=wd.Layout(width='70px'))
        self.fixed_arg = fixed_arg
        self.range_visibility = 'hidden' if self.is_categorical else 'visible'
        self.range = wd.RadioButtons(options=['Range'], layout=wd.Layout(width='70px', visibility=self.range_visibility))
        self.range.index = None
        self.range_arg1_text = wd.Label(value='from', layout=wd.Layout(visibility=self.range_visibility, width='30px'))
        self.range_arg1 = wd.BoundedFloatText(min=node.min, max=node.max, disabled=True, layout=wd.Layout(width='70px', visibility=self.range_visibility))
        self.range_arg2_text = wd.Label(value='to', layout=wd.Layout(visibility=self.range_visibility, width='15px'))
        self.range_arg2 = wd.BoundedFloatText(min=node.min, max=node.max, disabled=True, layout=wd.Layout(width='70px', visibility=self.range_visibility))
        self.none.observe(self.none_observer, names=['value'])
        self.fixed.observe(self.fixed_observer, names=['value'])
        self.range.observe(self.range_observer, names=['value'])
        self.box = wd.HBox([self.indent, self.text, self.none, self.fixed, self.fixed_arg, self.range, self.range_arg1_text, self.range_arg1, self.range_arg2_text, self.range_arg2])
        display(self.box)
        if self.disable:
            self.greyAll()

    def greyAll(self):
        self.none.disabled = True
        self.fixed.disabled = True
        self.fixed_arg.disabled = True
        self.range.disabled = True
        self.range_arg1.disabled = True
        self.range_arg2.disabled = True

    def setIntervention(self, intervention):
        if intervention[0] == 'fixed':
            self.fixed.index = 0
            self.fixed_arg.value = intervention[1]
        elif intervention[0] == 'range':
            self.range.index = 0
            self.range_arg1.value = intervention[1]
            self.range_arg2.value = intervention[2]

    # Radio button .index = None if off, .index = 0 if on
    def none_observer(self, sender):
        if self.none.index == 0:
            self.fixed.index = None
            self.fixed_arg.disabled = True
            self.range.index = None
            self.range_arg1.disabled = True
            self.range_arg2.disabled = True
        if self.disable:
            self.greyAll()

    def fixed_observer(self, sender):
        if self.fixed.index == 0:
            self.none.index = None
            self.fixed_arg.disabled = False
            self.range.index = None
            self.range_arg1.disabled = True
            self.range_arg2.disabled = True
        if self.disable:
            self.greyAll()

    def range_observer(self, sender):
        if self.range.index == 0:
            self.none.index = None
            self.fixed.index = None
            self.fixed_arg.disabled = True
            self.range_arg1.disabled = False
            self.range_arg2.disabled = False
        if self.disable:
            self.greyAll()

    def getIntervention(self):
        '''
        generates intervention from UI settings
        '''
        if self.none.index == 0: # None is deselected, 0 is selected
            return None
        elif self.fixed.index == 0:
            return ['fixed', self.fixed_arg.value]
        elif self.range.index == 0:
            return ['range', self.range_arg1.value, self.range_arg2.value]

class assignmentPlot:
    def __init__(self, experiment, plot='Truffula'):
        self.experiment = experiment
        self.group_names = experiment.group_names
        self.data = experiment.data
        self.plot = plot
        self.buildTraces()
        if self.plot == 'Truffula':
            self.layout = go.Layout(title=dict(text='Tree Group Assignments'),barmode='overlay', height=650, width=800,
                                  xaxis=dict(title='Longitude', fixedrange=True), yaxis=dict(title='Latitude', fixedrange=True),
                                  hovermode='closest',
                                  margin=dict(b=80, r=200, autoexpand=False),
                                  showlegend=True)
        else:
            self.layout = go.Layout(title=dict(text='Student Group Assignments'),barmode='overlay', height=650, width=800,
                                  xaxis=dict(title='Student', fixedrange=True),
                                  yaxis=dict(title='Height', fixedrange=True, range=(120, 200)),
                                  hovermode='closest',
                                  margin=dict(b=80, r=200, autoexpand=False),
                                  showlegend=True)
        self.plot = go.FigureWidget(data=self.traces, layout=self.layout)
        display(self.plot)

    def buildTraces(self):
        self.traces = []
        self.group_names = self.experiment.group_names
        self.data = self.experiment.data
        if self.plot == 'Truffula':
            for i, name in enumerate(self.group_names):
                self.traces += [go.Scatter(x=self.data[name]['Longitude'], y=self.data[name]['Latitude'], mode='markers', hovertemplate='Latitude: %{x} <br>Longitude: %{y} <br>', marker_symbol=i, name=name)]
        else:
            for i, name in enumerate(self.group_names):
                self.traces += [go.Bar(x=self.data[name]['id'], y=self.data[name]['Height (cm)'], hovertemplate='Student: %{x} <br>Height: %{y} cm<br>', name=name)]

    def updateAssignments(self):
        self.buildTraces()
        with self.plot.batch_update():
            self.plot.data = []
            for trace in self.traces:
                self.plot.add_traces(trace)
            self.plot.layout = self.layout

class orchardPlot:
    def __init__(self, experiment, gradient=None, show='all'):
        self.data = experiment.data
        self.experiment = experiment
        self.options = self.data[experiment.group_names[0]].columns.tolist()
        if show != 'all':
            for i in self.options.copy():
                if i not in show:
                    self.options.remove(i)
        for name in experiment.node.nodeDict():
            if experiment.node.nodeDict()[name].vartype == 'categorical' and name in show:
                self.options.remove(name)
        self.options.sort()
        if not gradient:
            gradient = self.options[0]
        self.textbox = wd.Dropdown(
                description='Gradient: ',
                value=gradient,
                options=self.options
        )
        self.textbox.observe(self.response, names="value")
        self.plotOrchard(gradient)

    def validate(self):
        return self.textbox.value in self.options

    def response(self, change):
        if self.validate():
            with self.g.batch_update():
                for i, name in enumerate(self.experiment.group_names):
                    self.g.data[i].marker.color = self.data[name][self.textbox.value]
                    self.g.update_layout({'coloraxis':{'colorscale':'Plasma', 'colorbar':{'title':self.textbox.value}}})
                    self.g.data[i].hovertemplate = 'Latitude: %{x} <br>Longitude: %{y} <br>' + self.textbox.value + ': %{marker.color}<br>'

    def plotOrchard(self, gradient):
        """Takes in the name of the group in the experiment and the name of the
        variable used to create the color gradient"""
        traces = []
        for i, name in enumerate(self.experiment.group_names):
            traces += [go.Scatter(x=self.data[name]['Longitude'], y=self.data[name]['Latitude'],
                                 marker=dict(color=self.data[name][gradient], coloraxis='coloraxis'),
                                 mode='markers',
                                 name=name,
                                 hovertemplate='Latitude: %{x} <br>Longitude: %{y} <br>'+ self.textbox.value + ': %{marker.color}<br>', hoverlabel=dict(namelength=0), marker_symbol=i)]
        width = 700 if (len(self.experiment.group_names) == 1) else 725 + max([len(name) for name in self.experiment.group_names])*6.5
        go_layout = go.Layout(title=dict(text='Orchard Layout'),barmode='overlay', height=650, width=width,
                              xaxis=dict(title='Longitude', fixedrange=True, range=[-50, 1050]),
                              yaxis=dict(title='Latitude', fixedrange=True, range=[-50, 1050]),
                              hovermode='closest', legend=dict(yanchor="top", y=1, xanchor="left", x=1.25),
                              coloraxis={'colorscale':'Plasma', 'colorbar':{'title':gradient}})
        self.g = go.FigureWidget(data=traces, layout=go_layout)

    def display(self):
        container = wd.HBox([self.textbox])
        display(wd.VBox([container, self.g]))

class interactivePlot:
    def __init__(self, experiment, show='all'):
        self.experiment = experiment
        self.x_options = list(experiment.node.network.keys())
        self.y_options = self.x_options.copy()
        if show != 'all':
            for i in self.x_options.copy():
                if i not in show:
                    self.x_options.remove(i)
                    self.y_options.remove(i)
        self.x_options.sort()
        self.y_options.sort()
        self.y_options += ['None (Distributions Only)']
        self.textbox1 = wd.Dropdown(
            description='x-Axis Variable: ',
            value=self.x_options[0],
            options=self.x_options
        )
        self.textbox2 = wd.Dropdown(
            description='y-Axis Variable: ',
            value=self.y_options[0],
            options=self.y_options
        )
        self.button = wd.RadioButtons(
            options=list(experiment.data.keys()) + ['All'],
            layout={'width': 'max-content'},
            description='Group',
            disabled=False
        )
        self.observe()
        self.initTraces()

    def display(self):
        container = wd.HBox([self.textbox1, self.textbox2])
        display(wd.VBox([container, self.g]))
        display(self.button)
        display(Nothing(), display_id='1')
        self.button.layout.display = 'none'

    def display_values(self, group):
        text = ""
        xType, yType = self.experiment.node.nodeDict()[self.textbox1.value].vartype, self.experiment.node.nodeDict()[self.textbox2.value].vartype
        if xType != 'categorical' and yType != 'categorical':
            with warnings.catch_warnings():
                warnings.simplefilter("ignore")
                r = sp.pearsonr(self.experiment.data[group][self.textbox1.value], self.experiment.data[group][self.textbox2.value])
            text += group + ': ' + 'Correlation (r) is ' + '{0:#.3f}, '.format(r[0]) + 'P-value is ' + '{0:#.3g}'.format(r[1])
        return text

    def createTraces(self, x, y):
        traces = []
        annotations = []
        annotation_y = -0.20 - 0.02*len(self.experiment.group_names)
        traceType = self.choose_trace(x, y)
        if traceType == 'histogram':
            for group in self.experiment.group_names:
                data = self.experiment.data[group]
                if self.experiment.node.nodeDict()[x].vartype == 'categorical':
                    opacity = 1
                else:
                    opacity = 0.75
                traces += [go.Histogram(x=data[x], name=group, bingroup=1, opacity=opacity)]
                y = 'Count'
                barmode = 'overlay'
        elif traceType == 'scatter':
            for group in self.experiment.group_names:
                data = self.experiment.data[group]
                traces += [go.Scatter(x=data[x], y=data[y], mode='markers', opacity=0.75, name=group)]
                annotations += [dict(xref='paper',yref='paper',x=0.5, y=annotation_y, showarrow=False, text=self.display_values(group))]
                annotation_y += -0.05
                barmode = 'overlay'
        elif traceType == 'bar':
            for group in self.experiment.group_names:
                avg = self.experiment.data[group].groupby(x).agg('mean')
                std = self.experiment.data[group].groupby(x).agg('std')[y]
                traces += [go.Bar(x=list(avg.index), y=avg[y], name=group, error_y=dict(type='data', array=std))]
                annotations += [dict(xref='paper',yref='paper',x=0.5, y=annotation_y, showarrow=False, text=self.display_values(group))]
                annotation_y += -0.05
                barmode = 'group'
        elif traceType == 'barh':
            for group in self.experiment.group_names:
                avg = self.experiment.data[group].groupby(y).agg('mean')
                std = self.experiment.data[group].groupby(y).agg('std')[x]
                traces += [go.Bar(x=avg[x], y=list(avg.index), name=group, error_x=dict(type='data', array=std), orientation='h')]
                annotations += [dict(xref='paper',yref='paper',x=0.5, y=annotation_y, showarrow=False, text=self.display_values(group))]
                annotation_y += -0.05
                barmode = 'group'
        go_layout = go.Layout(title=dict(text=x if traceType == 'histogram' else x + " vs. " + y ),
                              barmode=barmode,
                              height=500+50,
                              width=800,
                              xaxis=dict(title=x), yaxis=dict(title=y),
                              annotations = annotations,
                              margin=dict(b=80+50, r=200, autoexpand=False))
        return traces, go_layout

    def initTraces(self):
        traces, layout = self.createTraces(self.x_options[0], self.y_options[0])
        self.g = go.FigureWidget(layout=layout)
        for t in traces:
            self.g.add_traces(t)

    def updateTraces(self):
        self.g.data = []
        traces, layout = self.createTraces(self.textbox1.value, self.textbox2.value)
        for t in traces:
            self.g.add_traces(t)
        self.g.layout.annotations = layout.annotations
        self.g.layout = layout

    def observe(self):
        self.textbox1.observe(self.response, names="value")
        self.textbox2.observe(self.response, names="value")
        self.button.observe(self.update_table, names='value')

    def choose_trace(self, x, y):
        if y == 'None (Distributions Only)':
            return 'histogram'
        xType, yType = self.experiment.node.nodeDict()[x].vartype, self.experiment.node.nodeDict()[y].vartype
        if xType != 'categorical' and yType != 'categorical':
            return 'scatter'
        elif xType == 'categorical' and yType != 'categorical':
            return 'bar'
        elif xType != 'categorical' and yType == 'categorical':
            return 'barh'
        else:
            return 'table'

    def pivot_table(self):
        if self.textbox1.value == self.textbox2.value:
            df = "Cannot create a pivot table with only one variable"
            return df
        if self.button.value == 'All':
            for group in self.experiment.group_names:
                df = pd.DataFrame()
                df = pd.concat([df, self.experiment.data[group]])
            df = df.groupby([self.textbox1.value, self.textbox2.value]).agg('count').reset_index().pivot(self.textbox1.value, self.textbox2.value, self.options[0])
        else:
            df = self.experiment.data[self.button.value].groupby([self.textbox1.value, self.textbox2.value]).agg('count').reset_index().pivot(self.textbox1.value, self.textbox2.value, self.options[0])
        return df

    def update_table(self, change):
        update_display(self.pivot_table(), display_id='1');
        self.button.layout.display = 'flex'

    def validate(self):
        return self.textbox1.value in self.x_options and self.textbox2.value in (self.x_options + ['None (Distributions Only)'])

    def response(self, change):
        if self.validate():
            traceType = self.choose_trace(self.textbox1.value, self.textbox2.value)
            with self.g.batch_update():
                if traceType == 'table':
                    self.g.update_layout({'height':10, 'width':10})
                    self.g.layout.xaxis.title = ""
                    self.g.layout.yaxis.title = ""
                    self.g.layout.title = ""
                    self.button.layout.display = 'flex'
                else:
                    self.updateTraces()
                    update_display(Nothing(), display_id='1')
                    self.button.layout.display = 'none'

class Nothing:
    def __init__(self):
        None
    def __repr__(self):
        return ""

def text2Array(text):
    text = text.replace(' ', '')
    if re.fullmatch(r'^((\d+)(|-(\d+)),)*(\d+)(|-(\d+))$', text) is None:
        return None
    matches = re.findall(r'((\d+)-(\d+))|(\d+)', text)
    ids = []
    for m in matches:
        if m[3] != '':
            ids = np.concatenate((ids, [int(m[3])-1])) # Subtract one because text starts at 1, array starts at 0
        else:
            if int(m[2]) < int(m[1]):
                return None
            else:
                ids = np.concatenate((ids, np.arange(int(m[1])-1, int(m[2]))))
    uniq = list(set(ids))
    uniq.sort()
    if len(ids) != len(uniq):
        return None
    return uniq

def array2Text(ids):
    ids.sort()
    ids = np.array(ids)+1 # Add one because text starts at 1, array starts at 0
    segments = []
    start = ids[0]
    end = ids[0]
    for j in range(len(ids)):
        if j == len(ids)-1:
            end = ids[j]
            s = str(start) if start == end else '%d-%d' % (start, end)
            segments.append(s)
        elif ids[j+1] != ids[j]+1:
            end = ids[j]
            s = str(start) if start == end else '%d-%d' % (start, end)
            segments.append(s)
            start = ids[j+1]
    return ','.join(segments)

def randomAssign(N, N_group):
    '''
    Randomly assigns N total items into N_group groups
    Returns a list of lists of ids
    '''
    arr = np.arange(N)
    np.random.shuffle(arr)
    result = []
    for i in range(N_group):
        start = i*N//N_group
        end = min((i+1)*N//N_group, N)
        result.append(arr[start:end])
    return result

# Some functions for causal relations
def gaussian(mean, std):
    def f():
        return np.random.normal(mean, std)
    return f

def constant(x):
    def f():
        return x
    return f

def uniform(a, b):
    def f():
        return np.random.random()*(b-a) + a
    return f

def poisson(rate):
    def f():
        return np.random.poisson(lam=rate)
    return f

def choice(opts, weights=None, replace=True):
    def f():
        nonlocal weights
        if weights is None:
            chosen = np.random.choice(opts, replace=replace)
        else:
            weights = np.array(weights)
            p = weights/sum(weights)
            chosen = np.random.choice(opts, p=p, replace=replace)
        return chosen
    return f

# Solves for the coefficients given a set of points
def solveLinear(*points):
    n = len(points)
    A = np.zeros((n, n))
    b = np.zeros(n)
    for i in range(n):
        A[i] = np.append(points[i][0:-1], 1)
        b[i] = points[i][-1]
    sol = np.linalg.solve(A, b)
    return sol[0:-1], sol[-1]

def linear(x1, y1, x2, y2, fuzz=0):
    M, c = solveLinear((x1, y1), (x2, y2))
    def f(x):
        return M[0]*x + c + np.random.normal(0, fuzz)
    return f

def linearFunc(x1, m1, c1, x2, m2, c2, func, fuzz=0, integer=False):
    # Applies linear function on the input of func(*args[0:-1]), where the slope and intercept are determined by args[-1] according to x1, m1, c1, x2, m2, c2
    M_m, c_m = solveLinear((x1, m1), (x2, m2))
    M_c, c_c = solveLinear((x1, c1), (x2, c2))
    def f(*args):
        x = args[-1]
        m = M_m[0]*x + c_m
        c = M_c[0]*x + c_c
        number = m*func(*args[0:-1]) + c + np.random.normal(0, fuzz)
        if integer:
            number = max(int(number), 0)
        return number
    return f

def dependentPoisson(*points):
    M, c = solveLinear(*points)
    def f(*args):
        rate = max(M@np.array(args) + c, 0)
        return np.random.poisson(lam=rate)
    return f

def dependentGaussian(x1, mean1, std1, x2, mean2, std2):
    M_mean, c_mean = solveLinear((x1, mean1), (x2, mean2))
    M_std, c_std = solveLinear((x1, std1), (x2, std2))
    def f(x): # x is input value used to calculate mean and std of new distribution
        mean = M_mean[0]*x + c_mean
        std = max(M_std[0]*x + c_std, 0)
        return abs(np.random.normal(mean, std))
    return f

def categoricalLin(data): # data: {'category': (m, c, fuzz), etc}
    def f(x, y): # y is the category, x is the input value
        fuzz = data[y][2] if len(data[y]) == 3 else 0
        return data[y][0] * x + data[y][1] + np.random.normal(0, fuzz)
    return f

def categoricalGaussian(data): # data: {'category': (mean, std), etc}
    def f(x): # x is the category
        return np.random.normal(data[x][0], data[y][1])
    return f

'''
truffula
'''
# Uniformly distributed from 0m to 1000m
latitude_node = CausalNode('continuous', choice(np.linspace(0, 1000, 50), replace=False), name='Latitude', min=0, max=1000, init=True)
longitude_node = CausalNode('continuous', choice(np.linspace(0, 1000, 50), replace=False), name='Longitude', min=0, max=1000, init=True)
# Gaussian+absolute value, more wind in south
wind_node = CausalNode('continuous', lambda x,y: dependentGaussian(0, 2, 5, 1000, 10, 10)(x) + dependentGaussian(0, 6, 3, 1000, 2, 4)(x), name='Wind Speed', causes=[latitude_node, longitude_node], min=0, max=40)
supplement_node = CausalNode('categorical', constant('Water'), name='Supplement', categories=['Water', 'Kombucha', 'Milk', 'Tea'])
fertilizer_node = CausalNode('continuous', gaussian(10, 2), 'Fertilizer', min=0, max=20)
supplement_soil_effects = {'Water': (1, 0), 'Kombucha': (0.6, -5), 'Milk': (1.2, 10), 'Tea': (0.7, 0)}
# Fertilizer improves soil, kombucha destroys it
soil_node = CausalNode('continuous', lambda x, y: categoricalLin(supplement_soil_effects)(linear(0, 10, 20, 100, fuzz=5)(x), y), 'Soil Quality', causes=[fertilizer_node, supplement_node], min=0, max=100)
supplement_bees_effects = {'Water': (1, 0), 'Kombucha': (1.3, 0), 'Milk': (1, 0), 'Beer': (0.2, 0)}
# Beehive in north, bees avoid wind, love kombucha
bees_node = CausalNode('discrete', lambda x, y, z: categoricalLin(supplement_bees_effects)(dependentPoisson((0, 0, 250), (500, 30, 10), (0, 30, 40))(x, y), z), name='Number of Bees', causes=[latitude_node, wind_node, supplement_node], min=0, max=300)
# Bees and good soil improve fruiting
fruit_node = CausalNode('discrete', dependentPoisson((0, 0, 0), (100, 200, 28), (100, 50, 16)), name='Number of Fruits', causes=[soil_node, bees_node])
# fruit_node.drawNetwork()

truffula = CausalNetwork(fruit_node)

'''
basketball
'''
shottype_node = CausalNode('categorical', choice(['Above head', 'Layup', 'Hook shot'], weights=[6, 3, 2]), name='Shot Type', categories=['Above head', 'Layup', 'Hook shot'])
hours_node = CausalNode('continuous', choice(np.linspace(0, 14, 30), weights=1/np.linspace(1, 15, 30)), name='Hours Practised per Week')
height_node = CausalNode('continuous', gaussian(170, 10), name='Height (cm)', min=150, max=190, init=True)
ability_node = CausalNode('continuous', linearFunc(0, 1, 0, 10, 1, 20, linear(150, 40, 190, 60, fuzz=10)), name='Ability', causes=[height_node, hours_node])
shottype_modifier = {'Above head': (1, 0), 'Layup': (0.6, 0), 'Hook shot': (0.3, 0)}
success_node = CausalNode('continuous', categoricalLin(shottype_modifier), name='Success Rate', causes=[ability_node, shottype_node])

basketball = CausalNetwork(success_node)