from copy import deepcopy
from random import randint

from kedja.core import get_rid_map
from kedja.core.mutator import Mutator

from kedja.utils import utcnow, init_schema

EXPORT_VERSION = 1

def export_structure(context, request, contained=None):
    """ Export schema data from context and everything contained within it.
        Note that things that aren't saved in the schema won't be stored here!
    """
    is_export_root = contained is None
    if contained is None:
        contained = []
    data = context.__json__(request)
    if not is_export_root:
        contained.append(data)
    if len(context):
        data['contained'] = contained_data = []
        for x in context.values():
            export_structure(x, request, contained=contained_data)
    return data

def export_appstruct(export, version=EXPORT_VERSION, title=None, created=utcnow()):
    """ A helper to create a dict that conforms with export structure.

        Important keys

        - version (export version)
        - export (the actual data from export structure)
        - title
        - created
    """
    if title is None:
        # The first item should be the walls title
        title = export['data']['title']
    maxint = 2**53-1
    return {
        'export': export,
        'version': version,
        'title': title,
        'created': created,
        'id': randint(-maxint, maxint)
    }

def import_structure(context, request, data:dict, new_rids=True):
    """ Import a nested structure where data contains:
        - type_name
        - rid
        - data (a new dict to send to the mutator)
        - contained (a list of nested objects)

        Note that the base of the import is a single object. Usually a wall.
    """
    if new_rids:
        data = deepcopy(data)
        rid_map = get_rid_map(context)
        mapping = set_new_rids(rid_map, data)
        adjust_wall_relations(data, mapping)
        new_rids = False
    content = request.registry.content
    new_resource = content(data['type_name'])
    new_resource.rid = data['rid']
    context.add(str(data['rid']), new_resource)

    schema_factory = request.get_default_schema(new_resource)
    schema = init_schema(schema_factory, resource=new_resource, registry=request.registry)
    with Mutator(new_resource, schema, registry=request.registry) as m:
        m.update(data['data'])

    for contained_data in data.get('contained', []):
        import_structure(new_resource, request, contained_data, new_rids=new_rids)
    return new_resource

def set_new_rids(rid_map, data:dict, mapping=None, relations=None):
    """ Make sure the RIDs within the import data gets set to new values.
        Adjusts data in place!
    """
    if mapping is None:
        mapping = {}
    assert rid_map is not None
    old_rid = data['rid']
    new_rid = rid_map.new_rid()
    mapping[old_rid] = new_rid
    data['rid'] = new_rid
    for x in data.get('contained', ()):
        set_new_rids(rid_map, x, mapping=mapping, relations=relations)
    return mapping

def adjust_wall_relations(data:dict, mapping):
    assert data['type_name'] == 'Wall', "Only walls contain relations"
    for rel in data['data'].get('relations', []):
        members = []
        for rid in rel['members']:
            members.append(mapping[rid])
        rel['members'] = members


# 알고리즘이란?

print('세 정수의 최댓값을 구합니다.')

a = int(input('정수 a의 값을 구하시오.'))
b = int(input('정수 b의 값을 구하시오.'))
c = int(input('정수 c의 값을 구하시오.'))

maximum = a  # maximum에 a의 값을 대입
if b > maximum: maximum = b  # b의 값이 max보다 크면 max에 b 값을 대입
if c > maximum: maximum = c  # c의 값이 max보다 크면 max에 c 값을 대입

print(f'최댓값은 {maximum}입니다.')

# 문자열과 숫자

print('이름을 입력하세요.:', end='')
name = input()
print(f'안녕하세요? {name} 님')

# f'aaa {AA} aa' {}안에는 input의 변수를 출력하기 위해 print문 맨 앞에 f를 넣는다

# 세 정수의 최댓값 구하기

def max3(a, b, c):
    maximum = a
    if b > maximum: maximum = b
    if c > maximum: maximum = c
    return maximum

print(f'max3(3,2,1)={max3(3, 2, 1)}')  # a>b>c
print(f'max3(3,2,2)={max3(3, 2, 2)}')  # a>b=cd

# 세 정수를 입력받아 중앙값 구하기

def med3(a, b, c):
    if a >= b:
        if b >= c:
            return b
        elif a <= c:
            return a
        else:
            return c
    elif a > c:
        return a
    elif b > c:
        return c
    else:
        return b

print('세 정수의 중앙값을 구합니다.')
a = int(input('정수 a의 값을 입력하세요.:'))
b = int(input('정수 b의 값을 입력하세요.:'))
c = int(input('정수 c의 값을 입력하세요.:'))

print(f'중앙값을 {med3(a, b, c)}입니다.')

# 중앙값 구하는 또 다른 방법

def med3(a, b, c):
    if (b >= a and c <= a) or (b <= a and c >= a):
        return a
    elif (a > b and c < b) or (a < b and c > b):
        return b
    else:
        return c

print('세 정수의 중앙값을 구합니다.')
a = int(input('정수 a의 값을 입력하세요.:'))
b = int(input('정수 b의 값을 입력하세요.:'))
c = int(input('정수 c의 값을 입력하세요.:'))

print(f'중앙값을 {med3(a, b, c)}입니다.')

#입력받은 정수의 부호 (양수,음수,0) 출력하기

n=int(input('정수를 입력하셈'))

if n>0:
    print('양수')
elif n<0:
    print('음수')
else:
    print('0 입니다.')

#분기하는 조건문

n=int(input('정수 입력'))

if n==1:
    print('A')
elif n==2:
    print('B')
else:
    print('C')

#4개로 분기

n=int(input('정수 입력'))

if n==1:
    print('A')
elif n==2:
    print('B')
elif n==3:
    print('C')
else:
    pass


#!/usr/bin/python3
"""Takes in a letter and sends a POST request"""
import requests
import sys

if __name__ == "__main__":
    if len(sys.argv) == 2:
        q = {'q': sys.argv[1]}
    else:
        q = {'q': ""}
    x = requests.post('http://0.0.0.0:5000/search_user', q)
    try:
        json_dict = x.json()
        if len(json_dict) == 2:
            print('[{}] {}'.format(json_dict.get('id'), json_dict.get('name')))
        else:
            print('No result')
    except:
        print('Not a valid JSON')


import re
from typing import Dict, List

def helper(bag: str, bag_dict: Dict[str, List[str]]) -> int:
    # return 1 if leads to shiny gold bag else 0
    if len(bag_dict[bag]) == 0:
        return 0
    elif "shiny gold bags" in bag_dict[bag]:
        return 1
    else:
        counts = 0
        for b in bag_dict[bag]:
            counts += helper(b, bag_dict)
        if counts > 0:
            return 1
        else:
            return 0

def part1(data: List[str]) -> int:
    line_re = re.compile(r"^([\w\s]+)\scontain\s([\w\s,]+).$")
    bag_re = re.compile(r"\d\s([a-zA-Z\s]+)")

    bag_dict = {}
    for line in data:
        start_bag, end_bags = line_re.match(line.strip()).groups()
        bag_dict[start_bag] = list(map(lambda x: x if x[-1] == "s" else x + "s", bag_re.findall(end_bags)))

    count = 0
    for start_bag, values in bag_dict.items():
        if "shiny gold bags" in values:
            count += 1
        else:
            for bag in values:
                res = helper(bag, bag_dict)
                if res == 1:
                    count += 1
                    break
    return count

def helper2(bag: str, bag_dict: Dict[str, List[str]]) -> int:
    if len(bag_dict[bag]) == 0:
        return 1
    else:
        return sum(helper2(b, bag_dict) for b in bag_dict[bag]) + 1

def part2(data: List[str]) -> int:
    line_re = re.compile(r"^([\w\s]+)\scontain\s([\w\s,]+).$")
    bag_re = re.compile(r"(\d)\s([a-zA-Z\s]+)")

    bag_dict = {}
    for line in data:
        start_bag, end_bags = line_re.match(line.strip()).groups()
        bag_dict[start_bag] = []
        for num, bag in bag_re.findall(end_bags):
            bag_dict[start_bag] += [(bag if bag[-1] == "s" else bag + "s") for _ in range(int(num))]

    return helper2("shiny gold bags", bag_dict) - 1

if __name__ == "__main__":
    data = []
    with open("input.txt") as file:
        for line in file:
            data.append(line.strip())
    print(part1(data))
    print(part2(data))


import sqlite3
import json

connection = sqlite3.connect('wpd.db')
cursor = connection.cursor()

def create_db():
    create_table_query = """
    CREATE TABLE kelimeler (
        id INTEGER PRIMARY KEY,
        kelime TEXT,
        anlam TEXT,
        fonetik_en TEXT,
        fonetik_us TEXT
    )
    """
    cursor.execute(create_table_query)

def drop_table():
    drop_table_query = """
    DROP TABLE kelimeler
    """
    cursor.execute(drop_table_query)
    create_db()

def add_word():
    try:
        with open('dictionary.json', 'r', encoding='utf-8') as file:
            data = json.load(file)
            for kelime, anlam in data.items():
                try:
                    insert_query = 'INSERT INTO kelimeler (kelime, anlam) VALUES (?, ?)'
                    cursor.execute(insert_query, (kelime, anlam))
                except sqlite3.Error as e:
                    print("Veritabanı hatası:", e)
            connection.commit()
    except sqlite3.Error as e:
        print("Veritabanı hatası:", e)
    finally:
        connection.close()

if __name__ == '__main__':
    try:
        create_db()
        add_word()
    except:
        drop_table()
        add_word()


"""
Suppose you are given two lists of n points,
one list p1, p2, ..., pn on the line y = 0
and the other list q1, q2, ..., qn on the line y = 1.
Imagine a set of n line segments connecting each point pi to qi.
Write an algorithm to determine how many pairs of the line segments intersect.
"""
from typing import List

def num_intersections(p: List[int], q: List[int]) -> int:
    size = len(p)
    count = 0

    for i in range(size - 1):
        for j in range(i + 1, size):
            if (p[i] < p[j] and q[i] < q[j]) or (p[i] > p[j] and q[i] > q[j]):
                continue
            count += 1

    return count

if __name__ == "__main__":
    assert num_intersections([1, 4, 5], [4, 2, 3]) == 2
    assert num_intersections([1, 4, 5], [2, 3, 4]) == 0


#Problem: https://www.hackerrank.com/challenges/mini-max-sum/problem

from itertools import combinations

def miniMaxSum(arr):
    sums = []
    for i in combinations(arr, 4):
        sum_ = sum(i)
        sums.append(sum_)
    minn = min(sums)
    maxx = max(sums)
    print(minn, maxx)


import sys, os
sys.path.append('/home/fujenchu/projects/affordanceContext/DANet/')

import os
import numpy as np
from tqdm import tqdm
import cv2

import torch
from torch.utils import data
import torchvision.transforms as transform
from torch.nn.parallel.scatter_gather import gather

import encoding.utils as utils
from encoding.nn import SegmentationLosses, BatchNorm2d
from encoding.parallel import DataParallelModel, DataParallelCriterion
from encoding.datasets import get_segmentation_dataset, test_batchify_fn
from encoding.models import get_model, get_segmentation_model, MultiEvalModule

from option import Options
torch_ver = torch.__version__[:3]
if torch_ver == '0.3':
    from torch.autograd import Variable

import scipy.io as sio
# SAVE_ROOT = '/media/fujenchu/home3/data/affordanceSeg/SRFAff_v1/data/Affordance_Part_Data'
SAVE_ROOT = '/home/fujenchu/projects/affordanceContext/DANet/datasets/UMD_affordance'
SAVE_FOLDER_NAME = 'danetpspsharedbn_bg'
SAVE_FOLDER_NAME = 'danet_umd_ours'
SAVE_FOLDER_NAME = 'pred_seg_kp_umdself'

color_dict = {1: [0,0,205], #grasp red
              2: [34,139,34], #cut green
              3: [0,255,255], #scoop bluegreen
              4: [165,42,42], #contain dark blue
              5: [128,64,128], #pound purple
              6: [184,134,11]} #wrap-grasp light blue

def test(args):
    # output folder
    outdir = '%s/danet_vis'%(args.dataset)
    if not os.path.exists(outdir):
        os.makedirs(outdir)
    # data transforms
    input_transform = transform.Compose([
        transform.ToTensor(),
        transform.Normalize([.358388, .348858, .294015], [.153398, .137741, .230031])])
    # dataset
    if args.eval:
        testset = get_segmentation_dataset(args.dataset, split='val', mode='testval',
                                           transform=input_transform)
    else:#set split='test' for test set
        testset = get_segmentation_dataset(args.dataset, split='val', mode='vis',
                                           transform=input_transform)
    # dataloader
    loader_kwargs = {'num_workers': args.workers, 'pin_memory': True} \
        if args.cuda else {}
    test_data = data.DataLoader(testset, batch_size=args.test_batch_size,
                                drop_last=False, shuffle=False, **loader_kwargs)

    if args.model_zoo is not None:
        model = get_model(args.model_zoo, pretrained=True)
    else:
        model = get_segmentation_model(args.model, dataset=args.dataset,
                                       backbone=args.backbone, aux=args.aux,
                                       se_loss=args.se_loss, norm_layer=BatchNorm2d,
                                       base_size=args.base_size, crop_size=args.crop_size,
                                       multi_grid=args.multi_grid, multi_dilation=args.multi_dilation)
        # resuming checkpoint
        if args.resume is None or not os.path.isfile(args.resume):
            raise RuntimeError("=> no checkpoint found at '{}'" .format(args.resume))
        checkpoint = torch.load(args.resume)
        # strict=False, so that it is compatible with old pytorch saved models
        model.load_state_dict(checkpoint['state_dict'], strict=False)

    print(model)

    model.cuda()
    model.eval()

    tbar = tqdm(test_data)
    def eval_batch(image, dst, img_path, model, eval_mode):
        if eval_mode:
            # evaluation mode on validation set
            targets = dst
            outputs = model(image)

            # create save path
            savename = img_path[0].split('/')[-1]
            savename_splits = savename.split('_')
            save_obj = savename_splits[0]
            save_objsub = savename_splits[0] + '_' + savename_splits[1]
            save_mat = savename_splits[0] + '_' + savename_splits[1] + '_' + savename_splits[2] + '_rgbd_pred.mat'

            SAVE_FOLDER = os.path.join(SAVE_ROOT, SAVE_FOLDER_NAME, save_obj, save_objsub)
            if not os.path.exists(SAVE_FOLDER):
                os.makedirs(SAVE_FOLDER)

            # save mat file
            output_np = np.asarray(outputs['sasc'].data.cpu())
            output_np = output_np[0].transpose(1, 2, 0)
            output_np = np.asarray(np.argmax(output_np, axis=2), dtype=np.uint8)

            sio.savemat(os.path.join(SAVE_FOLDER, save_mat), {'pred_label': output_np}, do_compression=True)

    for i, (image, dst, img_path) in enumerate(tbar):
        with torch.no_grad():
            image = image.cuda()
            for key, value in dst.items():
                if not dst[key].is_cuda:
                    dst[key] = value.cuda()
            eval_batch(image, dst, img_path, model, args.eval)

def eval_models(args):
    if args.resume_dir is None or not os.path.isdir(args.resume_dir):
        raise RuntimeError("=> no checkpoint found at '{}'".format(args.resume_dir))
    for resume_file in os.listdir(args.resume_dir):
        if os.path.splitext(resume_file)[1] == '.tar':
            args.resume = os.path.join(args.resume_dir, resume_file)
            assert os.path.exists(args.resume)
            if not args.eval:
                test(args)
                continue
            test(args)

    print('Saving affordance mask prediction is finished!!!')

if __name__ == "__main__":
    args = Options().parse()
    torch.manual_seed(args.seed)
    args.test_batch_size = torch.cuda.device_count()
    eval_models(args)


import queue
import logging
import threading
import traceback
from time import sleep
from contextlib import suppress

from ..core.helpers.threadpool import ThreadPoolWrapper
from ..core.errors import ScanCancelledError, ValidationError, WordlistError

from bbot.core.event.base import is_event

class BaseModule:

    # Event types to watch
    watched_events = []
    # Event types to produce
    produced_events = []
    # Module description, etc.
    meta = {"auth_required": False, "description": "Base module"}
    # Flags, must include either "passive" or "active"
    flags = []

    # python dependencies (pip install ____)
    deps_pip = []
    # apt dependencies (apt install ____)
    deps_apt = []
    # other dependences as shell commands
    # uses ansible.builtin.shell (https://docs.ansible.com/ansible/latest/collections/ansible/builtin/shell_module.html)
    deps_shell = []
    # list of ansible tasks for when other dependency installation methods aren't enough
    deps_ansible = []
    # Whether to accept incoming duplicate events
    accept_dupes = False
    # Whether to block outgoing duplicate events
    suppress_dupes = True

    # Scope distance modifier - accept/deny events based on scope distance
    # None == accept all events
    # 2 == accept events up to and including the scan's configured search distance plus two
    # 1 == accept events up to and including the scan's configured search distance plus one
    # 0 == accept events up to and including the scan's configured search distance
    # -1 == accept events up to and including the scan's configured search distance minus one
    #       (this is the default setting because when the scan's configured search distance == 1
    #       [the default], then this is equivalent to in_scope_only)
    # -2 == accept events up to and including the scan's configured search distance minus two
    scope_distance_modifier = -1
    # Only accept the initial target event(s)
    target_only = False
    # Only accept explicitly in-scope events (scope distance == 0)
    # Use this options if your module is aggressive or if you don't want it to scale with
    #   the scan's search distance
    in_scope_only = False

    # Options, e.g. {"api_key": ""}
    options = {}
    # Options description, e.g. {"api_key": "API Key"}
    options_desc = {}
    # Maximum concurrent instances of handle_event() or handle_batch()
    max_event_handlers = 1
    # Max number of concurrent calls to submit_task()
    max_threads = 1
    # Batch size
    # If batch size > 1, override handle_batch() instead of handle_event()
    batch_size = 1
    # Seconds to wait before force-submitting batch
    batch_wait = 10
    # When set to false, prevents events generated by this module from being automatically marked as in-scope
    # Useful for low-confidence modules like speculate and ipneighbor
    _scope_shepherding = True
    # Exclude from scan statistics
    _stats_exclude = False
    # outgoing queue size
    _qsize = 100
    # Priority of events raised by this module, 1-5, lower numbers == higher priority
    _priority = 3
    # Name, overridden automatically
    _name = "base"
    # Type, for differentiating between normal modules and output modules, etc.
    _type = "scan"

    def __init__(self, scan):
        self.scan = scan
        self.errored = False
        self._log = None
        self._event_queue = None
        self._event_semaphore = threading.Semaphore(self._qsize)
        self._batch_idle = 0
        self.thread_pool = ThreadPoolWrapper(
            self.scan._thread_pool.executor, max_workers=self.config.get("max_threads", self.max_threads)
        )
        self._internal_thread_pool = ThreadPoolWrapper(
            self.scan._internal_thread_pool.executor, max_workers=self.max_event_handlers
        )
        # additional callbacks to be executed alongside self.cleanup()
        self.cleanup_callbacks = []
        self._cleanedup = False
        self._watched_events = None

    def setup(self):
        """
        Perform setup functions at the beginning of the scan.
        Optionally override this method.

        Must return True or False based on whether the setup was successful
        """
        return True

    def handle_event(self, event):
        """
        Override this method if batch_size == 1.
        """
        pass

    def handle_batch(self, *events):
        """
        Override this method if batch_size > 1.
        """
        pass

    def filter_event(self, event):
        """
        Accept/reject events based on custom criteria

        Override this method if you need more granular control
        over which events are distributed to your module
        """
        return True

    def finish(self):
        """
        Perform final functions when scan is nearing completion

        For example,  if your module relies on the word cloud, you may choose to wait until
        the scan is finished (and the word cloud is most complete) before running an operation.

        Note that this method may be called multiple times, because it may raise events.
        Optionally override this method.
        """
        return

    def report(self):
        """
        Perform a final task when the scan is finished, but before cleanup happens

        This is useful for modules that aggregate data and raise summary events at the end of a scan
        """
        return

    def cleanup(self):
        """
        Perform final cleanup after the scan has finished
        This method is called only once, and may not raise events.
        Optionally override this method.
        """
        return

    def get_watched_events(self):
        """
        Override if you need your watched_events to be dynamic
        """
        if self._watched_events is None:
            self._watched_events = set(self.watched_events)
        return self._watched_events

    def submit_task(self, *args, **kwargs):
        return self.thread_pool.submit_task(self.catch, *args, **kwargs)

    def catch(self, *args, **kwargs):
        try:
            lock_brutes = kwargs.pop("_lock_brutes", False) and "brute-force" in self.flags
            lock_acquired = False
            if lock_brutes:
                lock_acquired = self.scan._brute_lock.acquire()
            return self.scan.manager.catch(*args, **kwargs)
        finally:
            if lock_brutes and lock_acquired:
                self.scan._brute_lock.release()

    def _handle_batch(self, force=False):
        if self.num_queued_events > 0 and (force or self.num_queued_events >= self.batch_size):
            self._batch_idle = 0
            on_finish_callback = None
            events, finish, report = self.events_waiting
            if finish:
                on_finish_callback = self.finish
            elif report:
                on_finish_callback = self.report
            if events:
                self.debug(f"Handling batch of {len(events):,} events")
                self._internal_thread_pool.submit_task(
                    self.catch,
                    self.handle_batch,
                    *events,
                    _on_finish_callback=on_finish_callback,
                    _lock_brutes=True,
                )
                return True
        return False

    def make_event(self, *args, **kwargs):
        raise_error = kwargs.pop("raise_error", False)
        try:
            event = self.scan.make_event(*args, **kwargs)
        except ValidationError as e:
            if raise_error:
                raise
            self.warning(f"{e}")
            return
        if not event.module:
            event.module = self
        return event

    def emit_event(self, *args, **kwargs):
        if self.scan.stopping:
            return
        on_success_callback = kwargs.pop("on_success_callback", None)
        abort_if = kwargs.pop("abort_if", lambda e: False)
        quick = kwargs.pop("quick", False)
        event = self.make_event(*args, **kwargs)
        if event:
            okay = False
            while not self.scan.stopping:
                okay = self._event_semaphore.acquire(timeout=0.1)
                if okay:
                    self.scan.manager.emit_event(
                        event,
                        abort_if=abort_if,
                        on_success_callback=on_success_callback,
                        quick=quick,
                    )
                    break

    @property
    def events_waiting(self):
        """
        yields all events in queue, up to maximum batch size
        """
        events = []
        finish = False
        report = False
        left = int(self.batch_size)
        while left > 0 and self.event_queue:
            try:
                event = self.event_queue.get_nowait()
                if type(event) == str:
                    if event == "FINISHED":
                        finish = True
                    elif event == "REPORT":
                        report = True
                else:
                    left -= 1
                    events.append(event)
            except queue.Empty:
                break
        return events, finish, report

    @property
    def num_queued_events(self):
        ret = 0
        if self.event_queue:
            ret = self.event_queue.qsize()
        return ret

    def start(self):
        self.thread = threading.Thread(target=self._worker, daemon=True)
        self.thread.start()

    def _setup(self):

        status_codes = {False: "hard-fail", None: "soft-fail", True: "success"}

        status = False
        self.debug(f"Setting up module {self.name}")
        try:
            result = self.setup()
            if type(result) == tuple and len(result) == 2:
                status, msg = result
            else:
                status = result
                msg = status_codes[status]
            self.debug(f"Finished setting up module {self.name}")
        except Exception as e:
            self.set_error_state()
            if isinstance(e, WordlistError):
                status = None
            msg = f"{e}"
            self.debug(traceback.format_exc())
        return status, str(msg)

    @property
    def _force_batch(self):
        """
        Determine whether a batch should be forcefully submitted
        """
        # if we've been idle long enough
        if self._batch_idle >= self.batch_wait:
            return True
        # if scan is finishing
        if self.scan.status == "FINISHING":
            return True
        # if there's a batch stalemate
        batch_modules = [m for m in self.scan.modules.values() if m.batch_size > 1]
        if all([(not m.running) for m in batch_modules]):
            return True
        return False

    def _worker(self):
        # keep track of how long we've been running
        iterations = 0
        try:
            while not self.scan.stopping:
                iterations += 1
                if self.batch_size > 1:
                    if iterations % 3 == 0:
                        self._batch_idle += 1
                    force = self._force_batch
                    if force:
                        self._batch_idle = 0
                    submitted = self._handle_batch(force=force)
                    if not submitted:
                        sleep(0.3333)

                else:
                    try:
                        if self.event_queue:
                            e = self.event_queue.get_nowait()
                        else:
                            self.debug(f"Event queue is in bad state")
                            return
                    except queue.Empty:
                        sleep(0.3333)
                        continue
                    self.debug(f"Got {e} from {getattr(e, 'module', e)}")
                    # if we receive the special "FINISHED" event
                    if type(e) == str:
                        if e == "FINISHED":
                            self._internal_thread_pool.submit_task(self.catch, self.finish)
                        elif e == "REPORT":
                            self._internal_thread_pool.submit_task(self.catch, self.report)
                    else:
                        if self._type == "output":
                            self.catch(self.handle_event, e)
                        else:
                            self._internal_thread_pool.submit_task(self.catch, self.handle_event, e, _lock_brutes=True)

        except KeyboardInterrupt:
            self.debug(f"Interrupted")
            self.scan.stop()
        except ScanCancelledError as e:
            self.verbose(f"Scan cancelled, {e}")
        except Exception as e:
            self.set_error_state(f"Exception ({e.__class__.__name__}) in module {self.name}:\n{e}")
            self.debug(traceback.format_exc())

    def _filter_event(self, event):
        # special "FINISHED" event
        if type(event) == str:
            if event in ("FINISHED", "REPORT"):
                return True
            else:
                return False
        # exclude non-watched types
        if not any(t in self.get_watched_events() for t in ("*", event.type)):
            return False
        # built-in filtering based on scope distance, etc.
        acceptable, reason = self.event_acceptable(event)
        if not acceptable:
            self.debug(f"Not accepting {event} because {reason}")
            return False
        # custom filtering
        try:
            if not self.filter_event(event):
                self.debug(f"{event} did not meet custom filter criteria")
                return False
        except Exception as e:
            import traceback

            self.error(f"Error in filter_event({event}): {e}")
            self.debug(traceback.format_exc())
        return True

    @property
    def max_scope_distance(self):
        if self.in_scope_only or self.target_only:
            return 0
        return max(0, self.scan.scope_search_distance + self.scope_distance_modifier)

    def event_acceptable(self, e):
        """
        Return the max scope distance for an event that this module is qualified to accept
        """
        acceptable = True
        reason = ""
        if self.target_only:
            if "target" not in e.tags:
                acceptable = False
                reason = "it did not meet target_only filter criteria"
        if self.in_scope_only:
            if e.scope_distance > 0:
                acceptable = False
                reason = "it did not meet in_scope_only filter criteria"
        if self.scope_distance_modifier is not None:
            if e.scope_distance < 0:
                acceptable = False
                reason = f"its scope_distance ({e.scope_distance}) is invalid."
            elif e.scope_distance > self.max_scope_distance:
                acceptable = False
                reason = f"its scope_distance ({e.scope_distance}) exceeds the maximum allowed by the scan ({self.scan.scope_search_distance}) + the module ({self.scope_distance_modifier}) == {self.max_scope_distance}"

        # if the event is an IP address that came from a CIDR
        source_is_range = getattr(e.source, "type", "") == "IP_RANGE"
        if source_is_range and e.type == "IP_ADDRESS" and str(e.module) == "speculate" and self.name != "speculate":
            # and the current module listens for both ranges and CIDRs
            if all([x in self.watched_events for x in ("IP_RANGE", "IP_ADDRESS")]):
                # then skip the event.
                # this helps avoid double-portscanning both an individual IP and its parent CIDR.
                acceptable = False
                reason = "module consumes IP ranges directly"

        return acceptable, reason

    def _cleanup(self):
        if not self._cleanedup:
            self._cleanedup = True
            for callback in [self.cleanup] + self.cleanup_callbacks:
                if callable(callback):
                    self.catch(callback, _force=True)

    def queue_event(self, e):
        if self.event_queue is not None and not self.errored:
            if self._filter_event(e):
                if is_event(e):
                    self.scan.stats.event_consumed(e, self)
                self.event_queue.put(e)
        else:
            self.debug(f"Not in an acceptable state to queue event")

    def set_error_state(self, message=None):
        if message is not None:
            self.error(str(message))
        if not self.errored:
            self.debug(f"Setting error state for module {self.name}")
            self.errored = True
            # clear incoming queue
            if self.event_queue:
                self.debug(f"Emptying event_queue")
                with suppress(queue.Empty):
                    while 1:
                        self.event_queue.get_nowait()
                # set queue to None to prevent its use
                # if there are leftover objects in the queue, the scan will hang.
                self._event_queue = False

    @property
    def name(self):
        return str(self._name)

    @property
    def helpers(self):
        return self.scan.helpers

    @property
    def status(self):
        main_pool = self.thread_pool.num_tasks
        internal_pool = self._internal_thread_pool.num_tasks
        pool_total = main_pool + internal_pool
        incoming_qsize = 0
        if self.event_queue:
            incoming_qsize = self.event_queue.qsize()
        outgoing_qsize = self._qsize - self._event_semaphore._value
        status = {
            "events": {"incoming": incoming_qsize, "outgoing": outgoing_qsize},
            "tasks": {"main_pool": main_pool, "internal_pool": internal_pool, "total": pool_total},
            "errored": self.errored,
        }
        status["running"] = self._is_running(status)
        return status

    @staticmethod
    def _is_running(module_status):
        for pool, count in module_status["tasks"].items():
            if count > 0:
                return True
        return False

    @property
    def running(self):
        """
        Indicates whether the module is currently processing data.
        """
        return self._is_running(self.status)

    @property
    def config(self):
        config = self.scan.config.get("modules", {}).get(self.name, {})
        if config is None:
            config = {}
        return config

    @property
    def event_queue(self):
        if self._event_queue is None:
            self._event_queue = queue.SimpleQueue()
        return self._event_queue

    @property
    def priority(self):
        return int(max(1, min(5, self._priority)))

    @property
    def auth_required(self):
        return self.meta.get("auth_required", False)

    @property
    def log(self):
        if self._log is None:
            self._log = logging.getLogger(f"bbot.modules.{self.name}")
        return self._log

    def __str__(self):
        return self.name

    def stdout(self, *args, **kwargs):
        self.log.stdout(*args, extra={"scan_id": self.scan.id}, **kwargs)

    def debug(self, *args, **kwargs):
        self.log.debug(*args, extra={"scan_id": self.scan.id}, **kwargs)

    def verbose(self, *args, **kwargs):
        self.log.verbose(*args, extra={"scan_id": self.scan.id}, **kwargs)

    def hugeverbose(self, *args, **kwargs):
        self.log.hugeverbose(*args, extra={"scan_id": self.scan.id}, **kwargs)

    def info(self, *args, **kwargs):
        self.log.info(*args, extra={"scan_id": self.scan.id}, **kwargs)

    def hugeinfo(self, *args, **kwargs):
        self.log.hugeinfo(*args, extra={"scan_id": self.scan.id}, **kwargs)

    def success(self, *args, **kwargs):
        self.log.success(*args, extra={"scan_id": self.scan.id}, **kwargs)

    def hugesuccess(self, *args, **kwargs):
        self.log.hugesuccess(*args, extra={"scan_id": self.scan.id}, **kwargs)

    def warning(self, *args, **kwargs):
        self.log.warning(*args, extra={"scan_id": self.scan.id}, **kwargs)

    def hugewarning(self, *args, **kwargs):
        self.log.hugewarning(*args, extra={"scan_id": self.scan.id}, **kwargs)

    def error(self, *args, **kwargs):
        self.log.error(*args, extra={"scan_id": self.scan.id}, **kwargs)

    def critical(self, *args, **kwargs):
        self.log.critical(*args, extra={"scan_id": self.scan.id}, **kwargs)


import volatility.plugins.windows.pslist as pslist
import volatility.plugins.windows.vadinfo as vadinfo

from volatility.framework import interfaces, symbols
from volatility.framework import renderers
from volatility.framework.configuration import requirements
from volatility.framework.objects import utility
from volatility.framework.renderers import format_hints

class Malfind(interfaces.plugins.PluginInterface):
    """Lists process memory ranges that potentially contain injected code"""

    @classmethod
    def get_requirements(cls):
        # Since we're calling the plugin, make sure we have the plugin's requirements
        return [
            requirements.TranslationLayerRequirement(
                name = 'primary', description = 'Memory layer for the kernel', architectures = ["Intel32", "Intel64"]),
            requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols")
        ] + pslist.PsList.list_processes_filter_requirements

    @classmethod
    def is_vad_empty(self, proc_layer, vad):
        """Check if a VAD region is either entirely unavailable
        due to paging, entirely consisting of zeros, or a
        combination of the two. This helps ignore false positives
        whose VAD flags match task._injection_filter requirements
        but there's no data and thus not worth reporting it.

        Args:
            proc_layer: the process layer
            vad: the MMVAD structure to test
        """

        CHUNK_SIZE = 0x1000
        all_zero_page = "\x00" * CHUNK_SIZE

        offset = 0
        vad_length = vad.get_end() - vad.get_start()

        while offset < vad_length:
            next_addr = vad.get_start() + offset
            if proc_layer.is_valid(next_addr) and proc_layer.read(next_addr, CHUNK_SIZE) != all_zero_page:
                return False
            offset += CHUNK_SIZE

        return True

    @classmethod
    def list_injections(cls, context: interfaces.context.ContextInterface, symbol_table: str,
                        proc: interfaces.objects.ObjectInterface):
        """Generate memory regions for a process that may contain
        injected code.

        Args:
            proc: an _EPROCESS instance
        """

        proc_layer_name = proc.add_process_layer()
        proc_layer = context.memory[proc_layer_name]

        for vad in proc.get_vad_root().traverse():
            protection_string = vad.get_protection(
                vadinfo.VadInfo.protect_values(context, proc_layer_name, symbol_table), vadinfo.winnt_protections)
            write_exec = "EXECUTE" in protection_string and "WRITE" in protection_string

            # the write/exec check applies to everything
            if not write_exec:
                continue

            if (vad.get_private_memory() == 1
                    and vad.get_tag() == "VadS") or (vad.get_private_memory() == 0
                                                     and protection_string != "PAGE_EXECUTE_WRITECOPY"):
                if cls.is_vad_empty(proc_layer, vad):
                    continue

                data = proc_layer.read(vad.get_start(), 64, pad = True)
                yield vad, data

    def _generator(self, procs):
        # determine if we're on a 32 or 64 bit kernel
        is_32bit_arch = not symbols.symbol_table_is_64bit(self.context, self.config["nt_symbols"])

        for proc in procs:
            process_name = utility.array_to_string(proc.ImageFileName)

            for vad, data in self.list_injections(self.context, self.config["nt_symbols"], proc):

                # if we're on a 64 bit kernel, we may still need 32 bit disasm due to wow64
                if is_32bit_arch or proc.get_is_wow64():
                    architecture = "intel"
                else:
                    architecture = "intel64"

                disasm = interfaces.renderers.Disassembly(data, vad.get_start(), architecture)

                yield (0, (proc.UniqueProcessId, process_name, format_hints.Hex(vad.get_start()),
                           format_hints.Hex(vad.get_end()), vad.get_tag(),
                           vad.get_protection(
                               vadinfo.VadInfo.protect_values(self.context, proc.vol.layer_name,
                                                              self.config["nt_symbols"]), vadinfo.winnt_protections),
                           vad.get_commit_charge(), vad.get_private_memory(), format_hints.HexBytes(data), disasm))

    def run(self):
        filter_func = pslist.PsList.create_filter([self.config.get('pid', None)])

        return renderers.TreeGrid([("PID", int), ("Process", str), ("Start VPN", format_hints.Hex),
                                   ("End VPN", format_hints.Hex), ("Tag", str), ("Protection", str),
                                   ("CommitCharge", int), ("PrivateMemory", int), ("Hexdump", format_hints.HexBytes),
                                   ("Disasm", interfaces.renderers.Disassembly)],
                                  self._generator(
                                      pslist.PsList.list_processes(
                                          context = self.context,
                                          layer_name = self.config['primary'],
                                          symbol_table = self.config['nt_symbols'],
                                          filter_func = filter_func)))


import timings
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np

# plotting params
props = dict(boxstyle='round', facecolor='wheat', alpha=0.5)

#
# 1. Cache vs. NoCache
#
ranks = range(1,13,1)
ms = [100]
bs = [2]
cs = [True,False]
ns = [200]
algs = ['vector']
df = timings.time_mist(ms=ms, ns=ns, bs=bs, rs=ranks, cs=cs, q=200, algs=algs)
r = df[df['cache'] == True]['r']
cache = df[df['cache'] == True]['time'] * 1e6
nocache = df[df['cache'] == False]['time'] * 1e6
fig,ax = plt.subplots()
ax.plot(r,cache,label="cache")
ax.plot(r,nocache,label="nocache")
ax.text(0.20, 0.90, "m=%s,b=%s,n=%s,\nalg=%s,q=200" %(str(ms),str(bs),str(ns), str(algs)), transform=ax.transAxes, fontsize=10, verticalalignment='top', bbox=props)
ax.legend()
ax.set_ylabel('us')
ax.set_xlabel('threads')
plt.savefig("scaling1.tif")

#
# 2. Thread scaling, nocache
#
ranks = range(1,13,1)
ms = [10000]
bs = [2]
cs = [False]
ns = [100]
algs = ['vector']
df = timings.time_mist(ms=ms, ns=ns, bs=bs, rs=ranks, cs=cs, q=200, algs=algs)
r = df['r']
t = df['time'] * 1e3
fig,ax = plt.subplots()
ax.plot(r,t)
ax.text(0.20, 0.90, "m=%s,b=%s,n=%s,cache=%s\nalg=%s,q=200" %(str(ms),str(bs),str(ns), str(cs), str(algs)), transform=ax.transAxes, fontsize=10, verticalalignment='top', bbox=props)
ax.legend()
ax.set_ylabel('ms')
ax.set_xlabel('threads')
plt.savefig("scaling2.tif")

#
# 3. Thread scaling d3, cache
#
df = timings.time_mist(ms=[1000], ns=[100], bs=[2], ds=[3], rs=range(1,21,2), cs=[True], q=10, algs=['vector'], progress=True)
ranks = range(1,13,1)
ms = [1000]
bs = [2]
ds = [3]
cs = [True]
ns = [100]
algs = ['vector']
df = timings.time_mist(ms=ms, ns=ns, bs=bs, ds=ds, rs=ranks, cs=cs, q=200, algs=algs)
r = df['r']
t = df['time']
fig,ax = plt.subplots()
ax.plot(r,t)
ax.text(0.20, 0.90, "m=%s,b=%s,n=%s,d=%s\ncache=%s,alg=%s,q=200" %(str(ms),str(bs),str(ns), str(ds), str(cs), str(algs)), transform=ax.transAxes, fontsize=10, verticalalignment='top', bbox=props)
ax.legend()
ax.set_ylabel('s')
ax.set_xlabel('threads')
plt.savefig("scaling3.tif")

#
# 4. N scaling at different thread counts
#
ranks = [2,4,6]
ms=[100]
bs=[2]
cs=[False]
ns=[50,100,150,200,250,300]
algs=['vector']
df = timings.time_mist(ms=ms, ns=ns, bs=bs, rs=ranks, cs=cs, q=100, algs=algs)
n = ns
fig,ax = plt.subplots()
for rank in ranks:
    t = df[df['r'] == rank]['time'] * 1e6
    ax.plot(n,t,label="%d threads" % rank)
ax.text(0.40, 0.90, "m=%s,b=%s,cache=%s,\nalg=%s" %(str(ms),str(bs),str(cs), str(algs)),
        transform=ax.transAxes, fontsize=10, verticalalignment='top', bbox=props)
ax.legend()
ax.set_ylabel('us')
ax.set_xlabel('n')
plt.savefig("scaling4.tif")

#
# 5. Thread scaling at different N counts
#
ranks = range(10,130,10)
ms = [1000]
bs = [2]
cs = [False]
ns = range(100,1000,200)
algs = ['vector']
df = timings.time_mist(ms=ms, ns=ns, bs=bs, rs=ranks, cs=cs, q=200, algs=algs)
fig,ax = plt.subplots()
for n in ns:
    t = df[df['n'] == n]['time'] * 1e6
    ax.plot(ranks,t,label="n=%d" % n)
ax.text(0.20, 0.90, "m=%s,b=%s,cache=%s,\nalg=%s,q=200" %(str(ms),str(bs),str(cs), str(algs)), transform=ax.transAxes, fontsize=10, verticalalignment='top', bbox=props)
ax.legend()
ax.set_ylabel('us')
ax.set_xlabel('threads')
plt.savefig("scaling5.tif")

#
# 6. Thread scaling at different M counts
#
ranks = range(1,121,20) # salyut
ranks = range(1,13,1)   # galas155335
ms = range(100,2000,200)
bs = [2]
cs = [False]
ns = [300]
algs = ['vector']
df = timings.time_mist(ms=ms, ns=ns, bs=bs, rs=ranks, cs=cs, q=200, algs=algs)
fig,ax = plt.subplots()
for m in ms:
    t = df[df['m'] == m]['time'] * 1e6
    ax.plot(ranks,t,label="m=%d" % m)
ax.text(0.20, 0.90, "n=%s,b=%s,cache=%s,\nalg=%s,q=200" %(str(ns),str(bs),str(cs), str(algs)), transform=ax.transAxes, fontsize=10, verticalalignment='top', bbox=props)
ax.legend()
ax.set_ylabel('us')
ax.set_xlabel('threads')
plt.savefig("scaling6.tif")

#
# 7. Demonstrate Amdahl's Law
#
ranks = range(1,121,10)
ms = [1000]
bs = [2]
cs = [False]
ns = [300]
algs = ['vector']
df = timings.time_mist(ms=ms, ns=ns, bs=bs, rs=ranks, cs=cs, q=200, algs=algs)
fig,ax = plt.subplots()
t = df['time'] * 1e6
speedup = t[0] / t
ax.plot(ranks, speedup)
props = dict(boxstyle = 'round', facecolor = 'wheat', alpha = 0.5)
ax.text(0.20, 0.10, "m=%s,b=%s,n=%s,cache=%s\nalg=%s,q=200" %(str(ms),str(bs),str(ns), str(cs), str(algs)), transform=ax.transAxes, fontsize=10, verticalalignment='top', bbox=props)
ax.set_ylabel('speedup')
ax.set_xlabel('threads')
plt.savefig("scaling7.tif")


import textwrap
from dataclasses import dataclass
from typing import Callable, List, Optional

import pandas as pd

import app.models.network_graph_models as models
from app.funcs.annotate_entity import entity_id_match, entity_name_match
from app.utils import hex_to_rgb
from app.utils.logging import logger  # noqa:F401
from app.utils.meta_graph import (
    color_palette,
    find_orig_node,
    line_color_dict,
    meta_node_def,
    meta_rel_def,
)
from app.utils.visjs_config import node_alpha, node_wrap, visjs_option

@dataclass
class NetworkNodeSchema:
    """
    - id_col: column as the identifier of the node
    - label_col: column as the label to show on the node
    - label_formatter: a function to display the label, will override
                       `label_col`
    - meta_node: a node type (meta node) that can be found from `meta_graph_def`
    - cols: columns that are used in the processing of `url_formatter` or
            `title_formatter`
    - url_formatter: a function that takes the row of the datafraem as the
                     argument to format a clickable url
    - hover_title_formatter: a function that takes the row of the dataframe as the
                             argument to format the hover title
    - match_node_by:
      ["id", "name", None], when `url_formatter` is not set,
      and `match_node_scheme` is not None, then when double click on the node,
      will redirect to the node search view using `id_col`,
      and how the node is searched is determined how `id_col` is treated:
      - "id": treat `id_col` as "id" and search by exact matching
      - "name": treat `id_col` as "name" and search by fuzzy matching
    """

    meta_node: str
    id_col: str
    label_col: Optional[str] = None
    label_formatter: Optional[Callable] = None
    cols: Optional[List[str]] = None
    url_formatter: Optional[Callable] = None
    hover_title_formatter: Optional[Callable] = None
    match_node_by: Optional[str] = "id"

    def __post_init__(self) -> None:
        if self.label_col is None:
            self.label_col = self.id_col
        if self.hover_title_formatter is None:
            self.hover_title_formatter = lambda row: """
            <b>{orig_node}</b>: {label}
            """.format(
                orig_node=find_orig_node(row["meta_node"]),
                label=row[self.label_col],
            )
        if self.url_formatter is None and self.match_node_by is not None:
            if self.match_node_by == "id":
                self.url_formatter = id_match_formatter
            elif self.match_node_by == "name":
                self.url_formatter = name_match_formatter

@dataclass
class NetworkEdgeSchema:
    """
    - from_col: column used as the identifier of the "from" node
    - to_col: column used as the identifier of the "to" node
    - cols: columns that are used in the processing the formatters
    - from_meta_node: a node type (meta node) that can be found frrom `meta_graph_def`
    - to_meta_node: a node type (meta node) that can be found frrom `meta_graph_def`
    - arrows: If true then arrows will be used
    - hover_title_formatter: a function that takes the row of the dataframe as the
                             argument to format the hover title
    """

    from_col: str
    to_col: str
    cols: Optional[List[str]] = None
    from_meta_node: Optional[str] = None
    to_meta_node: Optional[str] = None
    arrows: bool = True
    dashes: bool = False
    hover_title_formatter: Optional[Callable] = None
    meta_rel: Optional[str] = None

class NetworkGraph:
    def __init__(
        self,
        df: pd.DataFrame,
        node_schemas: List[NetworkNodeSchema],
        edge_schemas: List[NetworkEdgeSchema],
        limit: Optional[int] = 50,
        visjs_option=visjs_option,
    ):
        """"""
        self.num_paths_total = len(df)
        if limit is not None:
            df = df[:limit]
        self.num_paths_displayed = len(df)

        self.visjs_option = visjs_option

        self.node_schemas = node_schemas
        self.edge_schemas = edge_schemas

        self.nodes_df = nodes_df_from_schema(df, self.node_schemas)
        logger.info(f"nodes_df: \n {self.nodes_df.head()}")

        self.edges_df = edges_df_from_schema(
            df, self.nodes_df, self.edge_schemas
        )
        logger.info(f"edges_df: \n {self.edges_df.head()}")

        self.nodes = render_nodes(self.nodes_df)
        self.nodes_3d = render_nodes_3d(self.nodes_df)
        logger.info(f"nodes: \n {self.nodes[0:2]}")

        self.edges = render_edges(self.edges_df)
        self.edges_3d = render_edges_3d(self.edges_df)
        logger.info(f"edges: \n {self.edges[0:2]}")

    def to_visjs(self) -> models.VisData:
        self.graph_data: models.VisData = {
            "nodes": self.nodes,
            "edges": self.edges,
            "nodes_3d": self.nodes_3d,
            "edges_3d": self.edges_3d,
            "option": self.visjs_option,
            "num_paths_total": self.num_paths_total,
            "num_paths_displayed": self.num_paths_displayed,
        }
        return self.graph_data

def network_graph(
    df: pd.DataFrame,
    node_schemas: List[NetworkNodeSchema],
    edge_schemas: List[NetworkEdgeSchema],
    limit: Optional[int],
) -> models.VisData:
    "Create visjs graph from query data."
    graph = NetworkGraph(
        df=df,
        node_schemas=node_schemas,
        edge_schemas=edge_schemas,
        limit=limit,
    )
    visjs_graph = graph.to_visjs()
    return visjs_graph

def process_node(df: pd.DataFrame, schema: NetworkNodeSchema) -> pd.DataFrame:
    df = df.dropna(subset=[schema.id_col])
    df = df.assign(orig_id=lambda df: df[schema.id_col].astype(str))
    # Node label: label_formatter > label_col
    if schema.label_formatter is not None:
        df = df.assign(
            label=lambda df: df.apply(schema.label_formatter, axis=1)
        )
    else:
        df = df.assign(label=lambda df: df[schema.label_col].astype(str))
    df = df.assign(meta_node=schema.meta_node)
    if schema.hover_title_formatter is not None:
        df = df.assign(
            title=lambda df: df.apply(schema.hover_title_formatter, axis=1)
        )
    else:
        df = df.assign(title=None)
    if schema.url_formatter is not None:
        df = df.assign(url=lambda df: df.apply(schema.url_formatter, axis=1))
    else:
        df = df.assign(url=None)
    df = df[
        ["orig_id", "label", "meta_node", "title", "url"]
    ].drop_duplicates()
    return df

def process_edge(df: pd.DataFrame, schema: NetworkEdgeSchema) -> pd.DataFrame:
    if schema.cols is not None and schema.cols != []:
        subset_cols = list(set([schema.from_col, schema.to_col, *schema.cols]))
    else:
        subset_cols = [schema.from_col, schema.to_col]
    for col in subset_cols:
        if col not in df.columns:
            return pd.DataFrame(None)
    df = df.dropna(subset=subset_cols)
    if len(df) == 0:
        return df
    df = df.assign(
        from_orig_id=lambda df: df[schema.from_col].astype(str),
        to_orig_id=lambda df: df[schema.to_col].astype(str),
    ).assign(
        from_meta_node=schema.from_meta_node, to_meta_node=schema.to_meta_node
    )
    if schema.hover_title_formatter is not None:
        df = df.assign(
            title=lambda df: df.apply(schema.hover_title_formatter, axis=1)
        )
    else:
        df = df.assign(title=None)
    df = df[
        [
            "from_orig_id",
            "to_orig_id",
            "from_meta_node",
            "to_meta_node",
            "title",
        ]
    ].drop_duplicates()
    # Set up color, arrows, etc
    # If not meta rel defined, follow source meta node
    if schema.meta_rel in meta_rel_def.index:
        df = (
            df.assign(meta_rel=schema.meta_rel)
            .merge(
                meta_rel_def[["color", "arrows", "dashes"]],
                how="left",
                left_on="meta_rel",
                right_index=True,
            )
            .drop(columns=["meta_rel"])
        )
    else:
        df = df.assign(arrows=schema.arrows, dashes=schema.dashes).merge(
            meta_node_def[["bg"]].rename(columns={"bg": "color"}),
            how="left",
            left_on="from_meta_node",
            right_index=True,
        )
    return df

def nodes_df_from_schema(
    df: pd.DataFrame, schema_list: List[NetworkNodeSchema]
) -> pd.DataFrame:
    nodes_df = (
        pd.concat(
            [process_node(df, schema) for schema in schema_list], sort=False
        )
        .drop_duplicates()
        .reset_index(drop=True)
        # Turn id into 1-indexed as required by visjs
        .assign(id=lambda df: df["orig_id"].astype("category").cat.codes + 1)
        # Add meta node info
        .merge(
            meta_node_def[["bg", "fg", "shape"]],
            left_on="meta_node",
            right_index=True,
        )
    )
    return nodes_df

def edges_df_from_schema(
    df: pd.DataFrame,
    nodes_df: pd.DataFrame,
    schema_list: List[NetworkEdgeSchema],
) -> pd.DataFrame:
    edges_df = (
        pd.concat(
            [process_edge(df, schema) for schema in schema_list], sort=False
        )
        .merge(
            nodes_df.rename(
                columns={"id": "from", "orig_id": "from_orig_id"}
            ).loc[:, ["from", "from_orig_id"]],
            how="left",
        )
        .merge(
            nodes_df.rename(columns={"id": "to", "orig_id": "to_orig_id"}).loc[
                :, ["to", "to_orig_id"]
            ],
            how="left",
        )
    )
    return edges_df

def render_nodes(nodes_df: pd.DataFrame) -> List[models.VisNode]:
    """The final step in processing nodes.

    Create a json-like list of nodes from nodes_df.
    """
    nodes = nodes_df.apply(
        lambda df: {
            "id": df["id"],
            "label": textwrap.fill(df["label"], node_wrap),
            "title": df["title"],
            "url": df["url"],
            "color": {
                "background": hex_to_rgb(df["bg"], alpha=node_alpha),
                "highlight": {
                    "background": df["bg"],
                    "border": color_palette["red"]["600"],
                },
                "hover": {
                    "background": df["bg"],
                    "border": color_palette["red"]["600"],
                },
            },
            "shape": df["shape"],
            "font": {"color": df["fg"]},
        },
        axis=1,
    ).tolist()
    return nodes

def render_nodes_3d(nodes_df: pd.DataFrame) -> List[models.VisNode3d]:
    """The final step in processing nodes.

    Create a json-like list of nodes from nodes_df.
    """
    nodes = nodes_df.apply(
        lambda df: {
            "id": df["id"],
            "name": df["label"],
            "color": df["bg"],
            "url": df["url"],
        },
        axis=1,
    ).tolist()
    return nodes

def render_edges(edges_df: pd.DataFrame) -> List[models.VisEdge]:
    """The final step in processing edges.

    Create a json-like list of nodes from edges_df.
    """
    edges = edges_df.apply(
        lambda df: {
            "from": df["from"],
            "to": df["to"],
            "arrows": {"to": df["arrows"]},
            "scaling": {"min": 0.2, "max": 5},
            "arrowStrikethrough": df["arrows"],
            "value": 1,
            "dashes": df["dashes"],
            "title": df["title"],
            "color": {"color": df["color"], "hover": line_color_dict["hover"]},
        },
        axis=1,
    ).tolist()
    return edges

def render_edges_3d(edges_df: pd.DataFrame) -> List[models.VisEdge3d]:
    """The final step in processing edges.

    Create a json-like list of nodes from edges_df.
    """
    edges = edges_df.apply(
        lambda df: {
            "source": df["from"],
            "target": df["to"],
            "color": df["color"],
        },
        axis=1,
    ).tolist()
    return edges

def id_match_formatter(row: pd.Series) -> str:
    url = entity_id_match(meta_node=row["meta_node"], id=row["orig_id"])
    return url

def name_match_formatter(row: pd.Series) -> str:
    url = entity_name_match(meta_node=row["meta_node"], name=row["orig_id"])
    return url


from .piece import Piece, Position

class Queen(Piece):
    # Class of the queen piece
    init_position = {0:  Position(3, 0), 1:  Position(3, 7)}
    init_moves = []

    def __init__(self, color: int):
        super().__init__(color,
                         Queen.init_position[color],
                         Queen.init_moves,
                         piece_type=1,
                         piece_number=0)


###------------------------------------------------------Coded By Reza Haghgoo------------------------------------------------------###
###------------------------------------------------------Last Edit : 8/12/2023------------------------------------------------------###
###----------------------------------------------Seasonality Classification of All Meters-------------------------------------------###

###-------------------------------------------------------Importing Libraries-------------------------------------------------------###

import pandas as pd

###----------------------------------------------Importing Residential Consumption Data---------------------------------------------###

ResConsumptionData = pd.read_csv('Consumption Data of All Residential Meters.csv')

"""---------------------------------------------------------------------------------------------------------------------------------"""
"""----------------------------------------------------Preprocessing of Dataset-----------------------------------------------------"""
"""---------------------------------------------------------------------------------------------------------------------------------"""

###-----------------------------------------------------Primary Edit of Dataset-----------------------------------------------------###

ResConsumptionData = ResConsumptionData.set_index(ResConsumptionData['ID'])
ResConsumptionData.drop('ID', axis=1, inplace=True)
ResConsumptionDataT = ResConsumptionData.transpose()

###-------------------------------------------------------Indexing to Dataset-------------------------------------------------------###

Dateindex = pd.date_range(start='2009-01-01 00:29:59', end='2010-06-20 23:59:59', freq='30min')
ResConsumptionDataT = ResConsumptionDataT.set_index(Dateindex)
RCD_Day = ResConsumptionDataT.resample('D').sum()
indx_Date = RCD_Day.index
indx_Date = indx_Date.astype(str)
RCD_WeekDay = RCD_Day.copy()
indx1 = RCD_WeekDay.index
RCD_WeekDay.index = indx1.strftime('%A')
indx_WeekDay = RCD_WeekDay.index
RCD_WeekNumber = RCD_Day.copy()
indx2 = RCD_WeekNumber.index
RCD_WeekNumber.index = indx2.strftime('%U')
indx_WeekNumber = RCD_WeekNumber.index
indx_WeekNumber = pd.to_numeric(indx_WeekNumber)
RCD_Year = RCD_Day.copy()
indx3 = RCD_Year.index
RCD_Year.index = indx3.strftime('%Y')
indx_Year = RCD_Year.index
indx_Year = pd.to_numeric(indx_Year)

indx_Date = pd.DataFrame(indx_Date)
indx_Year = pd.DataFrame(indx_Year)
indx_WeekNumber = pd.DataFrame(indx_WeekNumber)
indx_WeekDay = pd.DataFrame(indx_WeekDay)

Load = RCD_Day.copy()
Load['Date'] = indx_Date.values
Load['Year'] = indx_Year.values
Load['NumberWeeks'] = indx_WeekNumber.values
Load['WeekDay'] = indx_WeekDay.values

Load = Load.set_index([Load['Date'], Load['Year'], Load['NumberWeeks'], Load['WeekDay']])
Load.drop(['Date', 'Year', 'NumberWeeks', 'WeekDay'], axis=1, inplace=True)

"""---------------------------------------------------------------------------------------------------------------------------------"""
"""---------------------------------------------------Classification of Dataset-----------------------------------------------------"""
"""---------------------------------------------------------------------------------------------------------------------------------"""

###----------------------------------------Calculating Mean Value of Each Week for Each Meter---------------------------------------###

WeekMean = pd.DataFrame(columns = Load.columns)
WeekMean['Year'] = []
WeekMean['Weeknumber'] = []
for id in Load.columns:
    i = 0
    for year in range(2009,2011):
        for weeknumber in range(0,53):
            WeekMean.loc[i,id] = Load.loc[:,year,weeknumber,:][[id]].mean().values[0]
            if id == Load.columns[0]:
                WeekMean.loc[i,'Year'] = year
                WeekMean.loc[i,'Weeknumber'] = weeknumber
            i+=1

WeekMean = WeekMean.set_index([WeekMean['Year'], WeekMean['Weeknumber']])
WeekMean = WeekMean.drop('Year', axis=1)
WeekMean = WeekMean.drop('Weeknumber', axis=1)

###---------------------------Comparison Between Mean Value and Week Day Values of Each Week for Each Meter-------------------------###

MeterID = Load.columns
WeekDays = ['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday']
Col = pd.MultiIndex.from_product([MeterID, WeekDays], names=["Meter ID", "Week Days"])
SeasonalDay = pd.DataFrame(columns = Col)
SeasonalDay['Year'] = []
SeasonalDay['Weeknumber'] = []
for id in Load.columns:
    for WD in WeekDays:
        i=0
        for Year in range(2009,2011):
            for Weeknumber in range(0,53):
                SeasonalDay.loc[i,(id,WD)] = 1 if Load.loc[:,Year,Weeknumber,WD][[id]].values > WeekMean.loc[Year,Weeknumber][[id]].values else 0
                if id == Load.columns[0] and WD == WeekDays[0]:
                    SeasonalDay.loc[i,'Year'] = Year
                    SeasonalDay.loc[i,'Weeknumber'] = Weeknumber
                i+=1

SeasonalDay = SeasonalDay.set_index([SeasonalDay['Year'], SeasonalDay['Weeknumber']])
SeasonalDay = SeasonalDay.drop('Year', axis=1)
SeasonalDay = SeasonalDay.drop('Weeknumber', axis=1)

###--------------------------------------------Deleting Rows That Are Out of Dataset's Range----------------------------------------###

for i in range(0,27):
    SeasonalDay.drop((2010,26 + i), inplace=True)

###--------------------------------------Calculating Continuity of Seasonality of Seasonal Meters-----------------------------------###

SeasonalDaySum = SeasonalDay.sum()
SeasonalDaySum = pd.DataFrame(SeasonalDaySum)
SeasonalDaySum = SeasonalDaySum.transpose()

###--------Comparison Continuity of Seasonality of Seasonal Meters Values with Continusity Factor and Creating Output Tables--------###

ContinusityFactor = 0.8
STable = pd.DataFrame(columns = WeekDays)
USTable = pd.DataFrame(columns = ['UnSeasonal Meters'])
for id in Load.columns:
    if SeasonalDaySum.loc[:,(id,'Sunday')].values > ContinusityFactor*SeasonalDay.shape[0] or SeasonalDaySum.loc[:,(id,'Monday')].values > ContinusityFactor*SeasonalDay.shape[0] or SeasonalDaySum.loc[:,(id,'Tuesday')].values > ContinusityFactor*SeasonalDay.shape[0] or SeasonalDaySum.loc[:,(id,'Wednesday')].values > ContinusityFactor*SeasonalDay.shape[0] or SeasonalDaySum.loc[:,(id,'Thursday')].values > ContinusityFactor*SeasonalDay.shape[0] or SeasonalDaySum.loc[:,(id,'Friday')].values > ContinusityFactor*SeasonalDay.shape[0] or SeasonalDaySum.loc[:,(id,'Saturday')].values > ContinusityFactor*SeasonalDay.shape[0]:
        for Day in ['Sunday','Monday','Tuesday','Wednesday','Thursday','Friday','Saturday']:
            if SeasonalDaySum.loc[:,(id,Day)].values > ContinusityFactor*SeasonalDay.shape[0]:
                STable.loc[id,Day] = 'Seasonal'
            else:
                STable.loc[id,Day] = 'UnSeasonal'
    else:
        USTable.loc[id,'UnSeasonal Meters'] = 'UnSeasonal'

"""---------------------------------------------------------------------------------------------------------------------------------"""
"""---------------------------------------------Exporting Outputs of Classification-------------------------------------------------"""
"""---------------------------------------------------------------------------------------------------------------------------------"""

###------------------------------------------------------Exporting Outputs----------------------------------------------------------###

STable.to_csv('List of Seasonal Meters.csv')
USTable.to_csv('List of UnSeasonal Meters.csv')

Seasonality Classification of Electricity Residential Customers/3) Code of Seasonality Classification of All Meters/Seasonality Classification of All Meters.py


#!/usr/bin/env python

'''
Advent of Code 2022 - Day 11: Monkey In The Middle (Part 1)
https://adventofcode.com/2022/day/11
'''

from collections import OrderedDict, deque
import time
import re
from dataclasses import dataclass

@dataclass
class Monkey:
    number: int
    items: deque
    operation: str
    divisor: int
    true_throw_to: int
    false_throw_to: int
    num_inspected: int = 0

    def add_item(self, item):
        self.items.append(item)

    def inspect(self, monkeys: dict):
        while self.items:
            item = self.items.popleft()
            self.num_inspected += 1
            # Evaluate the string operation as an expression
            def f(old): return eval(self.operation)
            worry_level = f(item)  # Change the worry level
            worry_level = worry_level // 3  # Floor division
            throw_to = self.true_throw_to if worry_level % self.divisor == 0 else self.false_throw_to
            monkeys[throw_to].add_item(worry_level)  # Throw item to monkey

def create_monkey(note: re.Match) -> Monkey:
    monkey_no = int(note.group(1))
    items = note.group(2).split(',')
    items = [int(item) for item in items]  # Convert items to integers
    operation = note.group(3)
    divisor = int(note.group(4))
    true = int(note.group(5))
    false = int(note.group(6))

    return Monkey(number=monkey_no, items=deque(
        items), operation=operation, divisor=divisor, true_throw_to=true, false_throw_to=false)

def main():

    filename = input("What is the input file name? ")

    try:
        with open(filename, "r") as file:

            start = time.time()

            monkeys = {}

            # Read the notes
            note = ''
            for line in file:
                # Process note if a blank line
                if line == '\n':
                    # Generated regex using regex101.com (https://regex101.com/r/hLTCaG/1)
                    result = re.search(
                        "^Monkey (\d){1}:\n^\s+Starting items: ([\d,\s]+)\n^\s+Operation: new = (.+)\n^\s+Test: divisible by (\d+)\n^\s+If true: throw to monkey (\d+)\n^\s+If false: throw to monkey (\d)+$", note, re.MULTILINE)
                    if not result:
                        print("Could not find an observation note for a monkey")
                        continue

                    monkey = create_monkey(result)
                    monkeys[monkey.number] = monkey
                    note = ''  # reset to blank
                    continue

                note += line

            # Add the last monkey note
            result = re.search(
                "^Monkey (\d){1}:\n^\s+Starting items: ([\d,\s]+)\n^\s+Operation: new = (.+)\n^\s+Test: divisible by (\d+)\n^\s+If true: throw to monkey (\d+)\n^\s+If false: throw to monkey (\d)+$", note, re.MULTILINE)
            if result:
                monkey = create_monkey(result)
                monkeys[monkey.number] = monkey

            # Sort the monkeys into ascending order
            monkeys = OrderedDict(sorted(monkeys.items()))

            for round in range(20):  # Inspect over 20 rounds
                for _, monkey in monkeys.items():
                    monkey.inspect(monkeys)

            num_inspected = []
            for _, monkey in monkeys.items():
                num_inspected.append(monkey.num_inspected)
                print(
                    f"Monkey {monkey.number} inspected items {monkey.num_inspected} times.")

            num_inspected = sorted(num_inspected, reverse=True)
            print(
                f"The level of monkey business: {num_inspected[0] * num_inspected[1]}")

        end = time.time()
        print(f"Execution time in seconds: {end - start}\n")

    except FileNotFoundError:
        print(f"No such file or directory: '{filename}'")

if __name__ == "__main__":
    main()