import os
import time
import sys
import logging
import asyncio
import traceback
from pathlib import Path
from random import randint
from abc import abstractmethod
from itertools import count
from cached_property import cached_property
from threading import Event as thEvent, Thread
from typing import Optional, NoReturn, Callable, ClassVar, List, Any, Set
from contextlib import asynccontextmanager
from tempfile import gettempdir

from grpc import aio, ChannelConnectivity, ssl_channel_credentials, ssl_server_credentials
from anyio.abc import TaskStatus
from grpc.aio._call import AioRpcError
from anyio import TASK_STATUS_IGNORED
from tenacity import AsyncRetrying, wait_fixed, retry_if_exception_type, RetryCallState, wait_none

from daffi.exceptions import StopComponentError, ControllerUnavailable, RemoteCallError
from daffi.interface import ComponentI
from daffi.components.proto import messager_pb2_grpc as grpc_messager
from daffi.utils.misc import string_uuid

class UnixBase(ComponentI, grpc_messager.MessagerServiceServicer):
    SOCK_FILE = ".sock"

    def __init__(
        self,
        unix_sock_path: Optional[os.PathLike],
        ssl_certificate: Optional[os.PathLike],
        ssl_key: Optional[os.PathLike],
    ):
        self.unix_sock_path = unix_sock_path
        self.ssl_certificate = ssl_certificate
        self.ssl_key = ssl_key

    def info(self) -> str:
        return f"unix socket: [ {self.unix_socket!r} ]"

    @cached_property
    def base_dir(self) -> os.PathLike:
        _base_dir = self.unix_sock_path or Path(gettempdir()) / "daffi"
        _base_dir.mkdir(parents=True, exist_ok=True)
        return _base_dir

    @cached_property
    def unix_socket(self) -> str:
        self.unix_sock_path = self.base_dir
        sock = "unix:///" + os.path.join(self.base_dir, self.SOCK_FILE).strip("unix:///")
        if os.path.exists(sock):
            os.remove(sock)
        return sock

    @asynccontextmanager
    async def connect_listener(self):
        """UNIX Client"""
        if self.ssl_certificate and self.ssl_key:
            with open(self.ssl_certificate, "rb") as f:
                creds = ssl_channel_credentials(f.read())
            async with aio.secure_channel(self.unix_socket, creds) as aio_channel:
                yield aio_channel
        else:
            async with aio.insecure_channel(self.unix_socket) as aio_channel:
                yield aio_channel

    async def create_listener(self) -> NoReturn:
        """UNIX Server"""
        server = aio.server()
        grpc_messager.add_MessagerServiceServicer_to_server(self, server)
        if self.ssl_certificate and self.ssl_key:
            with open(self.ssl_key, "rb") as f:
                private_key = f.read()
            with open(self.ssl_certificate, "rb") as f:
                certificate_chain = f.read()
            server_credentials = ssl_server_credentials(((private_key, certificate_chain),))
            server.add_secure_port(self.unix_socket, server_credentials)
        else:
            server.add_insecure_port(self.unix_socket)
        await server.start()
        return server

class TcpBase(ComponentI, grpc_messager.MessagerServiceServicer):
    def __init__(
        self, host: str, port: Optional[int], ssl_certificate: Optional[os.PathLike], ssl_key: Optional[os.PathLike]
    ):
        self.host = host
        self.port = port
        self.ssl_certificate = ssl_certificate
        self.ssl_key = ssl_key
        if self.host in ("localhost", "0.0.0.0", "127.0.0.1", "192.168.0.1"):
            self.host = "[::]"

    def info(self) -> str:
        return f"tcp: [ host {self.host!r}, port: {self.port!r} ]"

    @asynccontextmanager
    async def connect_listener(self):
        """TCP Client"""
        listen_addr = f"{self.host}:{self.port}"
        if self.ssl_certificate and self.ssl_key:
            with open(self.ssl_certificate, "rb") as f:
                creds = ssl_channel_credentials(f.read())
            async with aio.secure_channel(listen_addr, creds) as aio_channel:
                yield aio_channel
        else:
            async with aio.insecure_channel(listen_addr) as aio_channel:
                yield aio_channel

    async def create_listener(self) -> NoReturn:
        """TCP Server"""
        if not self.port:
            await self.find_random_port()
        server = aio.server()
        grpc_messager.add_MessagerServiceServicer_to_server(self, server)
        listen_addr = f"{self.host}:{self.port}"
        if self.ssl_certificate and self.ssl_key:
            with open(self.ssl_key, "rb") as f:
                private_key = f.read()
            with open(self.ssl_certificate, "rb") as f:
                certificate_chain = f.read()
            server_credentials = ssl_server_credentials(((private_key, certificate_chain),))
            server.add_secure_port(listen_addr, server_credentials)
        else:
            server.add_insecure_port(listen_addr)
        await server.start()
        return server

    async def find_random_port(self, min_port: Optional[int] = 49152, max_port: Optional[int] = 65536) -> NoReturn:
        """
        Bind this controller to a random port in a range.
        If the port range is unspecified, the system will choose the port.
        Args:
            min_port : int, optional
                The minimum port in the range of ports to try (inclusive).
            max_port : int, optional
                The maximum port in the range of ports to try (exclusive).
        """
        while True:
            self.port = randint(min_port, max_port)
            if not await self.check_endpoint_is_busy():
                break

class ComponentsBase(UnixBase, TcpBase):
    RETRY_TIMEOUT = 1  # 1 sec
    IMMEDIATE_ACTION_ERRORS = (AioRpcError,)
    NON_IMMEDIATE_ACTION_ERRORS = (AioRpcError, RuntimeError, ControllerUnavailable)
    STOP_ACTION_ERRORS = (StopComponentError,)

    logger: logging.Logger = None
    components: ClassVar[Set[Callable]] = set()

    def __init__(
        self,
        g: "Global",
        process_name: str,
        host: Optional[str] = None,
        port: Optional[int] = None,
        unix_sock_path: Optional[os.PathLike] = None,
        ssl_certificate: Optional[os.PathLike] = None,
        ssl_key: Optional[os.PathLike] = None,
        async_backend: Optional[str] = None,
        global_terminate_event: Optional[thEvent] = None,
        on_node_connect: Optional[Callable[["Global", str], Any]] = None,
        on_node_disconnect: Optional[Callable[["Global", str], Any]] = None,
    ):
        self._set_keyboard_interrupt_handler()

        self.g = g
        self.process_name = process_name
        self.async_backend = async_backend or "asyncio"
        self.operations = None
        self.ident = string_uuid()
        self.global_terminate_event = global_terminate_event
        self.on_node_connect_cbs = [self.on_node_connect, on_node_connect]
        self.on_node_disconnect_cbs = [self.on_node_disconnect, on_node_disconnect]
        self._stopped = self._connected = False
        self.stop_event: bool = False
        self.stop_callbacks: List[Callable] = []

        self.components.add(self)

        if host:  # Check only host. Full host/port validation already took place before.
            self._base = TcpBase
            self._base.__init__(self, host, port, ssl_certificate, ssl_key)
        else:
            self._base = UnixBase
            self._base.__init__(self, unix_sock_path, ssl_certificate, ssl_key)

    def __hash__(self):
        return hash(self.__class__.__name__)

    @abstractmethod
    async def on_init(self) -> NoReturn:
        raise NotImplementedError

    @abstractmethod
    async def handle_operations(self, task_status: TaskStatus) -> NoReturn:
        raise NotImplementedError

    @abstractmethod
    async def before_connect(self) -> NoReturn:
        raise NotImplementedError

    async def on_stop(self) -> NoReturn:
        while not self.stop_event:
            await asyncio.sleep(0.3)

    def on_node_connect(self, _, process_name: str):
        """
        Default on node connect event handler.
        executed each time when connection to Controller is established
        """
        self.logger.info(f"Node {process_name!r} has been connected to Controller")

    def on_node_disconnect(self, _, process_name: str):
        """
        Default on node disconnect event handler.
        executed each time when connection to Controller is lost
        """
        self.logger.info(f"Node {process_name!r} has been disconnected")

    async def on_terminate(self):
        self.logger.info(f"{self.__class__.__name__} stopped.")
        self._stopped = True
        self.components.discard(self)
        if not self.components:
            # Cancel all existing asyncio tasks if component is the last one in termination chain. Eg if controller
            # is running along with node then controller will be in charge of cleaning.
            self.logger.debug("Cleaning asyncio tasks...")
            tasks = [task for task in asyncio.all_tasks(asyncio.get_running_loop()) if not task.done()]
            for task in tasks:
                task.cancel()

    @abstractmethod
    def on_error(self, exception: Exception) -> NoReturn:
        raise NotImplementedError

    @cached_property
    def info(self) -> str:
        return self._base.info(self)

    def stop(self, wait: Optional[bool] = False):
        self.stop_event = True
        if wait:
            for ind in count(1):

                if self._stopped:
                    break
                if not ind % 100:
                    self.logger.error("Too many iterations of stop event waiting!!!")
                time.sleep(0.5)

    @asynccontextmanager
    async def connect_listener(self):
        async with self._base.connect_listener(self) as stream:
            self._connected = True
            yield stream

    async def create_listener(self):
        if await self.check_endpoint_is_busy():
            self.logger.error(f"{self.info} is already allocated")
            raise StopComponentError()
        listener = await self._base.create_listener(self)
        self._connected = True
        return listener

    async def execute_on_connect(self):
        """Execute `on_connect` lifecycle handler in separate thread to prevent blocking main message handler."""

        def _on_connect_wrapper():
            # In case IPC is not ready yet.
            self.g.ipc.global_condition_event.wait_any_status()
            for cb in filter(None, self.on_node_connect_cbs):
                try:
                    cb(self.g, self.process_name)
                except Exception as e:
                    self.logger.error(
                        f"Exception occurred while execution `on_node_connect` handler: {e}, type: {type(e)}"
                    )

        Thread(target=_on_connect_wrapper).start()

    async def execute_on_disconnect(self, process_name):
        def _on_disconnect_wrapper():
            for cb in filter(None, self.on_node_disconnect_cbs):
                try:
                    cb(self.g, process_name)
                except Exception as e:
                    self.logger.error(
                        f"Exception occurred while execution `on_node_disconnect` handler: {e}, type: {type(e)}"
                    )

        Thread(target=_on_disconnect_wrapper).start()

    def after_exception(self, retry_state: RetryCallState):
        """return the result of the last call attempt"""

        retry_object = retry_state.retry_object
        attempt = retry_state.attempt_number
        exception = retry_state.outcome.exception()
        self.on_error(exception)

        if type(exception) in self.IMMEDIATE_ACTION_ERRORS:
            if type(exception) == AioRpcError:
                if "Deadline Exceeded" in str(exception):
                    retry_object.begin()
                    retry_object.wait = wait_none()
                    return

        elif type(exception) in self.STOP_ACTION_ERRORS:
            self.logger.debug("Perform stop action")
            self.stop()
            self.check_stopped_components()
            return

        if type(exception) in self.NON_IMMEDIATE_ACTION_ERRORS:
            retry_object.wait = wait_fixed(self.RETRY_TIMEOUT)
            if attempt == 3 or not attempt % 6:
                self.logger.error(
                    f"Unable to connect {self.__class__.__name__}"
                    f" {self.process_name!r}. Error = {type(exception)} {exception}. Retrying..."
                )
        else:
            retry_object.wait = wait_fixed(self.RETRY_TIMEOUT)
            self.logger.error(f"Unpredictable error during {self.__class__.__name__} execution")
            traceback.print_tb(exception.__traceback__)
            self.stop()
            self.check_stopped_components()

    def check_stopped_components(self):
        if all(c.stop_event for c in self.components) and self.global_terminate_event:
            self.logger.debug("All components are stopped. Set global terminate event.")
            self.global_terminate_event.set()

    async def handle(self, task_status: TaskStatus = TASK_STATUS_IGNORED) -> NoReturn:
        await self.on_init()

        async for attempt in AsyncRetrying(
            reraise=True,
            retry=retry_if_exception_type((Exception,)),
            after=self.after_exception,
        ):
            with attempt:
                if self.stop_event or self.global_terminate_event.is_set():
                    break

                await self.before_connect()

                stop_task = asyncio.create_task(self.on_stop())
                try:
                    await asyncio.gather(self.handle_operations(task_status), stop_task)
                finally:
                    stop_task.cancel()

        await self.on_terminate()

    def _set_keyboard_interrupt_handler(self) -> NoReturn:
        # Creating a handler
        def handle_unhandled_exception(exc_type, exc_value, exc_traceback):
            if issubclass(exc_type, KeyboardInterrupt):
                # Will call default excepthook
                sys.__excepthook__(exc_type, exc_value, exc_traceback)
                return
                # Create a critical level log message with info from the except hook.
            self.logger.error("Unhandled exception:", exc_info=(exc_type, exc_value, exc_traceback))
            for component in self.components:
                component.stop()
            self.check_stopped_components()

        # Assign the excepthook to the handler
        sys.excepthook = handle_unhandled_exception

    async def check_endpoint_is_busy(self) -> bool:
        """Check if unix socket/host-port is already allocated"""
        async with self.connect_listener() as channel:
            for _ in range(30):
                state = channel.get_state(try_to_connect=True)
                if state == ChannelConnectivity.TRANSIENT_FAILURE:
                    # Ready to connect
                    return False
                elif state == ChannelConnectivity.READY:
                    return True
                await asyncio.sleep(0.3)


from food import Food
from snake import Snake, screen
from scoreboard import Scoreboard

game_is_on = True
snake = Snake()
dot = Food()
score = Scoreboard()

while game_is_on:
    snake.move()

    #Detect collision with food.
    if snake.head.distance(dot) < 15:
        dot.refresh()
        score.increase()
        # last = len(snake.segment_snake)-1
        # last_position = snake.segment_snake[len(snake.segment_snake)-1].position()
        # snake.create_square(last_position)
        snake.last_segment()

    #Detec collision with wall
    if snake.head.xcor() >= 300 or snake.head.xcor() <= -300 or snake.head.ycor() >= 300 or snake.head.ycor() <= -300:
        score.reset_score()
        snake.reset_snake()

    for segment_number in range(len(snake.segment_snake)-1, 0, -1):
        if snake.head.position() == snake.segment_snake[segment_number].position():
            score.reset_score()
            snake.reset_snake()

screen.exitonclick()


from tkinter import *

root = 0
def do():
    global root
    root = Tk()
    root.title("Two")
    root.geometry('800x400')
    canvas = Canvas(root, background = 'green')
    canvas.pack(fill = BOTH, expand = 1)
    canvas.create_text(400, 200, text="Two", font = ("Consolas", 100))
    root.mainloop()
def die():
    root.destroy()


## advent of code 2019
## https://adventofcode.com/2019
## day 03
from PrettyMap2D import *
from aocpuzzle import *
class Move:
    def __init__(self, instruction):
        d = instruction[0]
        self.dist = int(instruction[1:])
        if d == 'R':
            self.direction = RIGHT
        elif d == 'L':
            self.direction = LEFT
        elif d == 'U':
            self.direction = UP
        elif d == 'D':
            self.direction = DOWN

class Puzzle(AoCPuzzle):
    def __init__(self, lines, is_test=False):
        AoCPuzzle.__init__(self, lines, is_test)
        self.always_run_part_1 = True
        self.wires = []
        for line in lines:
            moves = []
            directions = line.split(',')
            for d in directions:
                moves.append(Move(d))
            self.wires.append(moves)
        self.red_map = Map2D('.')
        self.blue_map = self.red_map.add_overlay(1)
        self.red_steps = {}
        self.blue_steps = {}
        self.intersections = []

    def part1(self):
        print("RED_WIRE")
        red_wire = self.wires[0]
        cur = Coord2D(0,0)
        steps = 0
        for m in red_wire:
            for d in range(m.dist):
                steps += 1
                cur += m.direction
                self.red_map[cur] = 'R'
                if not cur in self.red_steps:
                    self.red_steps[cur.copy()] = steps
        print("BLUE_WIRE")
        blue_wire = self.wires[1]
        steps = 0
        cur = Coord2D(0,0)
        for m in blue_wire:
            for d in range(m.dist):
                cur = cur + m.direction
                steps += 1
                if self.red_map[cur] == 'R':
                    self.intersections.append(cur)
                self.blue_map[cur] = 'B'
                if not cur in self.blue_steps:
                    self.blue_steps[cur] = steps
        #self.red_map.print()
        assert(len(self.intersections) > 0)

        return min([x.distance(Coord2D(0,0)) for x in self.intersections ])

    def part2(self):
        assert(len(self.intersections) > 0)
        distances = []
        for i in self.intersections:
            print(i, ": red dist =",self.red_steps[i], ", blue dist =", self.blue_steps[i])
            distances.append(self.red_steps[i] + self.blue_steps[i])

        return min(distances)


## advent of code 2018
## https://adventofcode.com/2018
## day 15

from collections import deque
import copy
from Map2DLayers import *
class Unit:
    def __init__(self, goblin, coord):
        self.goblin = goblin
        self.coord = coord
        self.attack_power = 3
        self.hit_points = 200
        self.alive = True

class Puzzle:
    def __init__(self, lines):
        self.map = InfiniteGridStack(default=".")
        self.player_map = {}
        self.map.add_layer(self.player_map)
        self.original_units = []
        self.valid_coords = set()
        y = 0
        for line in lines:
            x = 0
            for c in line:
                coord = Coord2D(x,y)
                if c == "#":
                    self.map[coord] = "#"
                elif c == "G":
                    self.valid_coords.add(coord)
                    self.original_units.append(Unit(True, coord))
                elif c == "E":
                    self.valid_coords.add(coord)
                    self.original_units.append(Unit(False, coord))
                elif c == ".":
                    self.valid_coords.add(coord)
                elif c == " ":
                    break
                x += 1
            y += 1

        self.reset()

    def reset(self, elf_attack_power=3):
        self.elf_died = False
        self.active_units = copy.deepcopy(self.original_units)
        self.player_map.clear()

        for unit in self.active_units:
            if unit.goblin:
                self.player_map[unit.coord] = "G"
            else:
                unit.attack_power = elf_attack_power
                self.player_map[unit.coord] = "E"

    def get_next_move(self, coordA, coords):

        visited = {}
        queue = deque()
        queue.append((coordA, 0, []))
        found = []
        found_dist = 1000000000
        while queue:
            coord, dist, steps = queue.popleft()
            assert(dist < 100)
            if dist > found_dist:
                break

            if coord in visited:
                # only filter on those coords that were
                # visited in fewer steps.  We want to get the list
                # of all candidate paths to all candidate locations with
                # the minimum number of steps
                vdist, vstep = visited[coord]
                if dist > vdist:
                    continue
                if not (steps[0] < vstep):
                    continue

            if dist > 0:
                visited[coord] = (dist, steps[0])
            else:
                visited[coord] = (dist, Coord2D(0,0))

            if coord in coords:
                found_dist = dist
                found.append((coord, steps))
                continue
            for n in coord.neighbor_coords():
                if self.map[n] == ".":

                    next_steps = steps.copy()
                    next_steps.append(n)
                    queue.append((n, dist + 1, next_steps))

        if len(found) == 0:
            return coordA

        # We have all of the paths to any of the target coords that have the
        # same shortest path length  Now sort them based on the reading order
        # of the destination, then the reading order of the first step
        found.sort(key=lambda x: (x[0], x[1][0]))

        dest, steps = found[0]

        return steps[0]

    def move(self, unit, targets):
        candidate_squares = set()
        for target in targets:
            for loc in target.coord.neighbor_coords():
                if self.map[loc] == ".":
                    candidate_squares.add(loc)

        new_location = self.get_next_move(unit.coord, candidate_squares)
        self.player_map.pop(unit.coord)

        if unit.goblin:
            self.player_map[new_location] = "G"
        else:
            self.player_map[new_location] = "E"
        unit.coord = new_location

    def in_range(self, unit, targets):
        neighbors = set(unit.coord.neighbor_coords())
        potential_victims = []
        for t in targets:
            if t.coord in neighbors:
                if t.hit_points > 0:
                    potential_victims.append(t)
        return potential_victims

    def attack(self, unit, potential_victims):
        if len(potential_victims) == 0:
            return

        potential_victims.sort(key=lambda x:(x.hit_points, x.coord))
        victim = potential_victims[0]
        victim.hit_points -= unit.attack_power

        if victim.hit_points <= 0:
            if not victim.goblin:
                self.elf_died = True
            self.player_map.pop(victim.coord)

    def execute_turn(self, unit):
        targets = [x for x in self.active_units if (not x.goblin == unit.goblin) and (x.hit_points > 0)] #filter(lambda x:not x.goblin == unit.goblin, self.active_units))
        if (len(targets)) == 0:
            return False

        in_range = self.in_range(unit, targets)
        if len(in_range) == 0:
            self.move(unit, targets)
            in_range = self.in_range(unit, targets)
        self.attack(unit, in_range)
        return True

    def remove_the_dead(self):
        new_active_units = []
        for u in self.active_units:
            if u.hit_points > 0:
                new_active_units.append(u)
        self.active_units = new_active_units

    def execute_round(self):

        self.active_units.sort(key=lambda x: x.coord)
        for unit in self.active_units:
            if unit.hit_points > 0:
                if not self.execute_turn(unit):
                    self.remove_the_dead()
                    return False
        self.remove_the_dead()
        return True

    def play_game(self):
        round = 0
        while(self.execute_round()):
            round += 1

        total_hp = 0
        for unit in self.active_units:
            total_hp += unit.hit_points
        outcome = total_hp * round
        return outcome

    def try_with_ap(self, ap):
        print("Trying attack power:", ap)
        self.reset(ap)

        round = 0
        while(self.execute_round()):
            round += 1

            if self.elf_died:
                print("Elf died")
                return 0

        if self.elf_died:
                print("Elf died2")
                return 0
        total_hp = 0
        for unit in self.active_units:
            total_hp += unit.hit_points
        outcome = total_hp * round
        return outcome

    def save_the_elves(self):
        ap = 3
        while True:

            outcome = self.try_with_ap(ap)
            print(outcome)
            if outcome > 0:
                return outcome
            ap += 1

    def part1(self):
        return self.play_game()

    def part2(self):
        return self.save_the_elves()


print("This program count the world place in your sentence.")
user_input = input("Please write a sentence: ")
a = user_input.split()
count = 0
while count < len(a):
    print(a[count], "\t", count)
    count = count + 1
b = len(a) + 1
dictionary = dict(zip(a, range(0, b)))
print(dictionary)


# This files contains your custom actions which can be used to run
# custom Python code.
#
# See this guide on how to implement these action:
# https://rasa.com/docs/rasa/custom-actions

import requests
import json
import inflect

from typing import Any, Text, Dict, List
from rasa_sdk import Action, Tracker
from rasa_sdk.executor import CollectingDispatcher

PREFIX = """PREFIX fo: <http://purl.org/ontology/fo/>
            PREFIX fc: <http://www.freeclass.eu/freeclass_v1#>
            PREFIX ex: <http://example.org/>
            PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
            PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
            prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
            prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>
            prefix foaf: <http://xmlns.com/foaf/0.1/>
            prefix focu: <http://focu.io/schema#>
            prefix dbo: <http://www.dbpedia.org/ontology/>
            prefix dbr: <http://www.dbpedia.org/resource/>
            prefix xsd: <http://www.w3.org/2001/XMLSchema#>
            prefix dcterms: <http://purl.org/dc/terms/>
            prefix dcmitype: <http://purl.org/dc/dcmitype/>"""

# Q1) How many courses are offered at Concordia?
class ActionNumberOfUniCourses(Action):

    def name(self) -> Text:
        return "action_number_of_courses_offered"

    def run(self, dispatcher: CollectingDispatcher,
            tracker: Tracker,
            domain: Dict[Text, Any]) -> List[Dict[Text, Any]]:
        response = requests.post("http://localhost:3030/uni/sparql",
                                 data={'query': PREFIX + """
                                                    SELECT (count(?courseId) as ?CourseCount)
                                                    WHERE {
                                                            ?courseId a focu:Course.
                                                    }
                                                    """
                                       })
        y = json.loads(response.text)

        results = y["results"]
        bindings = results["bindings"]

        numberOfCourses = 0

        for result in bindings:
            for key in result:
                if key == "CourseCount":
                    for subKey in result[key]:
                        if subKey == "value":
                            numberOfCourses = result[key][subKey]

        dispatcher.utter_message(text=f"\n Concordia University offers a total of {numberOfCourses} courses")

# Q2) Which topics are covered in COMP 6741 lectures?
class ActionTopicCoveredCourse(Action):

    def name(self) -> Text:
        return "action_topic_covered_course"

    def run(self, dispatcher: CollectingDispatcher,
            tracker: Tracker,
            domain: Dict[Text, Any]) -> List[Dict[Text, Any]]:

        course = tracker.slots['course'].strip()
        courseName = course[:4].upper()
        courseNumber = course[4:]

        response = requests.post("http://localhost:3030/uni/sparql",
                                 data={'query': PREFIX + """
                                        SELECT ?topics WHERE {
                                         ?sub focu:subject "%s" .
                                         ?sub focu:catalog "%s".
                                         ?sub rdfs:comment ?topics .
                                        }

                                        """ % (courseName, courseNumber)
                                       })

        y = json.loads(response.text)

        results = y["results"]
        bindings = results["bindings"]

        topics = ""

        for result in bindings:
            for key in result:
                if key == "topics":
                    for subKey in result[key]:
                        if subKey == "value":
                            topics += result[key][subKey]

        if len(topics) == 0:
            dispatcher.utter_message(text=f"No results found please try again ")
        else:
            dispatcher.utter_message(text=f"\n Here are topics covered in {course} \n {topics} ")

# Q3 Which topics is Arihant competent in?
class ActionCompetencyOfPerson(Action):

    def name(self) -> Text:
        return "action_person_competency"

    def run(self, dispatcher: CollectingDispatcher,
            tracker: Tracker,
            domain: Dict[Text, Any]) -> List[Dict[Text, Any]]:

        person = tracker.slots['person'].strip()

        response = requests.post("http://localhost:3030/uni/sparql",
                                 data={'query': PREFIX + """
                                                    SELECT ?Competencies WHERE {
                                                    ?sudent a focu:Student.
                                                     ?sudent focu:firstName "%s".
                                                     ?sudent focu:competencies ?Competencies.
                                                    }

                                                    """ % person
                                       })

        #
        y = json.loads(response.text)

        results = y["results"]
        bindings = results["bindings"]

        competencies = ""

        for result in bindings:
            for key in result:
                if key == "Competencies":
                    for subKey in result[key]:
                        if subKey == "value":
                            competencies += result[key][subKey] + "\n"

        if len(competencies) == 0:
            dispatcher.utter_message(text=f"No results found please try again ")
        else:
            dispatcher.utter_message(text=f"\n {person} is  Competent in \n {competencies}")

# Q4) Which courses at Concordia teaches deep learning?
class WhichCourseAtUniTeachTopic(Action):

    def name(self) -> Text:
        return "action_course_teach_topic"

    def response_request(self, topic):
        response = requests.post("http://localhost:3030/uni/sparql",
                                 data={'query': PREFIX + """
                                SELECT ?cname (count (?cname) as ?count)
                                WHERE {
                                ?sub a focu:Course.
                                ?sub focu:courseName ?cname.
                                ?lecture dcterms:isPartOf ?sub.
                                ?lecture foaf:topic ?topicid.
                                ?topicid rdfs:label "AI"
                                } GROUP BY ?cname ORDER BY DESC(?count)
                            """
                                       })
        # Use the json module to load CKAN's response into a dictionary.
        # % topic
        y = json.loads(response.text)

        # the result is a Python dictionary:
        results = y["results"]

        # courses = []
        #
        # for result in results["bindings"]:
        #     courseCode = result["c1"]
        #     courseName = result["cname"]
        #     topicCount = result["topicCount"]
        #     code = courseCode["value"]
        #     name = courseName["value"]
        #     count = topicCount["value"]
        #     course = {"courseCode": code, "courseName": name, "topicCount": count}
        #     courses.append(course)
        #
        # return courses
        return y

    def run(self, dispatcher: CollectingDispatcher,
            tracker: Tracker,
            domain: Dict[Text, Any]) -> List[Dict[Text, Any]]:

        p = inflect.engine()

        t = tracker.slots['topic'].strip()

        t_sing = p.singular_noun(t)

        if t_sing is False:
            t_sing = t

        otopic = t_sing.replace(" ", "_")
        y = self.response_request(otopic)

        results = y["results"]
        bindings = results["bindings"]

        coursesTeaching = ""

        for result in bindings:
            for key in result:
                if key == "cname":
                    for subKey in result[key]:
                        if subKey == "value":
                            coursesTeaching += result[key][subKey] + " "

                if key == "count":
                    for subKey in result[key]:
                        if subKey == "value":
                            coursesTeaching += "With topic: " + otopic + " Appearing " + result[key][
                                subKey] + " times \n"

        if len(coursesTeaching) == 0:
            dispatcher.utter_message(text=f"No results found please try again ")
        else:
            dispatcher.utter_message(text=f"\n Courses which covers {otopic} are \n{coursesTeaching}")
        return []

# Q5) Where can I get more information about [COMP6721](course)?
class ActionMoreInformation(Action):

    def name(self) -> Text:
        return "action_information"

    def run(self, dispatcher: CollectingDispatcher,
            tracker: Tracker,
            domain: Dict[Text, Any]) -> List[Dict[Text, Any]]:

        course = tracker.slots['course'].strip()
        courseName = course[:4].upper()
        courseNumber = course[4:]

        response = requests.post("http://localhost:3030/uni/sparql",
                                 data={'query': PREFIX + """
                                        SELECT ?information
                                        WHERE {
                                        ?subject a focu:Course.
                                         ?subject focu:subject "%s".
                                         ?subject focu:catalog "%s".
                                         ?subject rdfs:seeAlso ?information.
                                        }
                                        """ % (courseName, courseNumber)
                                       })

        y = json.loads(response.text)

        results = y["results"]
        bindings = results["bindings"]

        seeAlso = ""

        for result in bindings:
            for key in result:
                if key == "information":
                    for subKey in result[key]:
                        if subKey == "value":
                            seeAlso += result[key][subKey]

        if len(seeAlso) == 0:
            dispatcher.utter_message(text=f"No results found please try again ")
        else:
            dispatcher.utter_message(text=f"\n Here are some useful links \n {seeAlso} ")

# Q6) What are grades of Brendon(person)?
class ActionGetGrade(Action):

    def name(self) -> Text:
        return "action_grade"

    def run(self, dispatcher: CollectingDispatcher,
            tracker: Tracker,
            domain: Dict[Text, Any]) -> List[Dict[Text, Any]]:

        person = tracker.slots['person'].strip()

        response = requests.post("http://localhost:3030/uni/sparql",
                                 data={'query': PREFIX + """
                                        SELECT ?course ?grade WHERE {
                                         ?student a focu:Student.
                                         ?student focu:firstName "%s".
                                         ?student focu:hasRecord ?record.
                                         ?record focu:courseTaken ?course.
                                         ?record focu:grade ?grade
                                        }
                                    """ % person
                                       })

        y = json.loads(response.text)

        results = y["results"]
        bindings = results["bindings"]

        courseAndGrade = ""

        for result in bindings:
            for key in result:
                if key == "course":
                    for subKey in result[key]:
                        if subKey == "value":
                            iresult=result[key][subKey]
                            iresult=iresult[19:]
                            courseAndGrade += iresult + " "

                if key == "grade":
                    for subKey in result[key]:
                        if subKey == "value":
                            courseAndGrade += result[key][subKey] + " \n"

        if len(courseAndGrade) == 0:
            dispatcher.utter_message(text=f"No results found please try again ")
        else:
            dispatcher.utter_message(text=f"\n Here are grades of {person}\n{courseAndGrade} ")

# Q7) Which COMP courses are taught at Concordia?
class ActionDepartmentCourse(Action):

    def name(self) -> Text:
        return "action_department_courses"

    def run(self, dispatcher: CollectingDispatcher,
            tracker: Tracker,
            domain: Dict[Text, Any]) -> List[Dict[Text, Any]]:

        department = tracker.slots['department'].strip()

        response = requests.post("http://localhost:3030/uni/sparql",
                                 data={'query': PREFIX + """
                                        SELECT ?subjects
                                        WHERE {
                                        ?subject a focu:Course.
                                         ?subject focu:subject "%s".
                                          ?subject focu:courseName ?subjects
                                        } LIMIT 5
                                        """ % department
                                       })

        y = json.loads(response.text)

        results = y["results"]
        bindings = results["bindings"]

        courses = ""

        for result in bindings:
            for key in result:
                if key == "subjects":
                    for subKey in result[key]:
                        if subKey == "value":
                            courses += result[key][subKey] + " \n"

        if len(courses) == 0:
            dispatcher.utter_message(text=f"No results found please try again ")
        else:
            dispatcher.utter_message(text=f"\n Here are courses offered by {department} \n {courses} ")

# Q8
class ActionLectureTopic(Action):
    def name(self) -> Text:
        return "action_lecture_course_topic"

    def run(self, dispatcher: CollectingDispatcher,
            tracker: Tracker,
            domain: Dict[Text, Any]) -> List[Dict[Text, Any]]:

        course = tracker.slots['course'].strip()
        courseName = course[:4].upper()
        courseNumber = course[4:]
        topic = tracker.slots['topic'].strip()

        response = requests.post("http://localhost:3030/uni/sparql",
                                 data={'query': PREFIX + """
                                    SELECT ?lecture
                                    WHERE {
                                    ?subject a focu:Course.
                                    ?subject focu:subject "%s".
                                    ?subject focu:catalog "%s".
                                    ?lecture dcterms:isPartOf ?subject.
                                    ?lecture foaf:topic "%s".
                                }
                                """ % (courseName, courseNumber, topic)
                                       })

        y = json.loads(response.text)

        results = y["results"]
        bindings = results["bindings"]

        courses = ""

        for result in bindings:
            for key in result:
                if key == "lecture":
                    for subKey in result[key]:
                        if subKey == "value":
                            res=result[key][subKey]
                            res=res[-4:]
                            courses += res + " "

        if len(courses) == 0:
            dispatcher.utter_message(text=f"No results found please try again ")
        else:
            dispatcher.utter_message(text=f"\n {topic} is covered in  {course} Lecture: {courses}")

# Q9 Similar topics 2 courses
class ActionCourseSimilarTopic(Action):
    def name(self) -> Text:
        return "action_similar_topic_courses"

    def run(self, dispatcher: CollectingDispatcher,
            tracker: Tracker,
            domain: Dict[Text, Any]) -> List[Dict[Text, Any]]:

        course = tracker.slots['course'].strip()
        courses = course.split("&")

        courseName1 = courses[0][:4].upper()
        courseNumber1 = courses[0][4:]

        courseName2 = courses[1][:4].upper()
        courseNumber2 = courses[1][4:]

        response = requests.post("http://localhost:3030/uni/sparql",
                                 data={'query': PREFIX + """
                                    SELECT ?topics
                                    WHERE {
                                    ?subject1 a focu:Course.
                                     ?subject1 focu:subject "%s".
                                     ?subject1 focu:catalog "%s".
                                     ?subject1 foaf:topic ?topics.
                                     ?subject2 a focu:Course.
                                     ?subject2 focu:subject "%s".
                                     ?subject2 focu:catalog "%s".
                                     ?subject2 foaf:topic ?topics.

                                }
                                """ % (courseName1, courseNumber1, courseName2, courseNumber2)
                                       })

        y = json.loads(response.text)

        results = y["results"]
        bindings = results["bindings"]

        commom = ""

        for result in bindings:
            for key in result:
                if key == "topics":
                    for subKey in result[key]:
                        if subKey == "value":
                            commom += result[key][subKey] + " "

        if len(commom) == 0:
            dispatcher.utter_message(text=f"No results found please try again ")
        else:
            dispatcher.utter_message(text=f"\n {courses[0]} and {courses[1]} covers following similar \n {commom}")

# Q10) Where can I get more information about [COMP6721](course)?
class StudentTakenCourses(Action):

    def name(self) -> Text:
        return "action_students_taken_course"

    def run(self, dispatcher: CollectingDispatcher,
            tracker: Tracker,
            domain: Dict[Text, Any]) -> List[Dict[Text, Any]]:

        course = tracker.slots['course'].strip()
        courseName = course[:4].upper()
        courseNumber = course[4:]
        response = requests.post("http://localhost:3030/uni/sparql",
                                 data={'query': PREFIX + """
                                        SELECT ?firstName ?lastName
                                        WHERE {
                                          ?student a focu:Student.
                                          ?student focu:firstName ?firstName .
                                          ?student focu:lastName ?lastName.
                                          ?student focu:hasRecord ?record.
                                          ?record focu:courseTaken ?course.
                                          ?course a focu:Course.
                                          ?course focu:subject "%s".
                                          ?course focu:catalog "%s".
                                        }
                                        LIMIT 5
                                        """ % (courseName, courseNumber)
                                       })

        y = json.loads(response.text)
        results = y["results"]
        bindings = results["bindings"]

        students = ""

        for result in bindings:
            for key in result:
                if key == "firstName":
                    for subKey in result[key]:
                        if subKey == "value":
                            students += result[key][subKey] + "\n"

        if len(students) == 0:
            dispatcher.utter_message(text=f"No results found please try again ")
        else:
            dispatcher.utter_message(text=f"\n Here is List of students who have taken {course} \n{students} ")

# QN1) What is COMP6741 about [COMP6721](course)?
class ActionGetCourseAbout(Action):

    def name(self) -> Text:
        return "action_course"

    def run(self, dispatcher: CollectingDispatcher,
            tracker: Tracker,
            domain: Dict[Text, Any]) -> List[Dict[Text, Any]]:

        course = tracker.slots['course'].strip()
        course = course.replace(" ", "")
        courseName = course[:4].upper()
        courseNumber = course[4:]

        response = requests.post("http://localhost:3030/uni/sparql",
                                 data={'query': PREFIX + """
                                        SELECT ?desc
                                        WHERE {
                                        ?sub a focu:Course.
                                        ?sub focu:subject "%s" .
                                        ?sub focu:catalog "%s".
                                        ?sub rdfs:comment ?desc
                                        }

                                        """ % (courseName, courseNumber)
                                       })

        y = json.loads(response.text)

        results = y["results"]
        bindings = results["bindings"]

        about = ""

        for result in bindings:
            for key in result:
                if key == "desc":
                    for subKey in result[key]:
                        if subKey == "value":
                            about += result[key][subKey]

        if len(about) == 0:
            dispatcher.utter_message(text=f"No results found please try again ")
        else:
            dispatcher.utter_message(text=f"\n {course} is about \n{about}")

# QN2) Which topics are covered in Lab2 covers?
class ActionGetEvent(Action):

    def name(self) -> Text:
        return "action_event"

    def run(self, dispatcher: CollectingDispatcher,
            tracker: Tracker,
            domain: Dict[Text, Any]) -> List[Dict[Text, Any]]:

        course = tracker.slots['course'].strip()
        course = course.replace(" ", "")
        courseName = course[:4].upper()
        courseNumber = course[4:]

        event = tracker.slots['event'].strip()
        event = event.replace(" ", "").upper()
        events = event.split("#")
        if (events[0] == 'LECTURE'):
            events[0] = 'SLIDES'

        response = requests.post("http://localhost:3030/uni/sparql",
                                 data={'query': PREFIX + """
                                        SELECT ?topic ?source
                                        WHERE {
                                        ?sub a focu:Course.
                                        ?sub focu:subject "%s" .
                                        ?sub focu:catalog "%s".
                                        ?lecture dcterms:isPartOf ?sub.
                                        ?lecture dcmitype:identifier "Lecture %s".
                                        ?lecture foaf:topic ?topicid.
                                        ?topicid rdfs:label ?topic.
                                        ?topicid focu:topicSource "%s"
                                        }
                                        """ % (courseName, courseNumber, events[1], events[0])
                                       })

        y = json.loads(response.text)

        results = y["results"]
        bindings = results["bindings"]

        topics = ""

        for result in bindings:
            for key in result:
                if key == "topic":
                    for subKey in result[key]:
                        if subKey == "value":
                            topics += result[key][subKey] + "\n"

        if len(topics) == 0:
            dispatcher.utter_message(text=f"No results found please try again ")
        else:
            dispatcher.utter_message(text=f"Following topics are covered in {event} of {course} \n{topics}")


from nameko.rpc import MethodProxy, RpcReply
from py_zipkin import zipkin

from nameko_zipkin.utils import stop_span, start_span

class TracedRpcReply(RpcReply):
    def __init__(self, reply_event, zipkin_span):
        super().__init__(reply_event)
        self.zipkin_span = zipkin_span

    def result(self):
        try:
            return super().result()
        finally:
            stop_span(self.zipkin_span)

def monkey_patch(transport_handler):
    _call = MethodProxy._call

    def _call_traced(self: MethodProxy, *args, **kwargs):
        span = zipkin.zipkin_client_span(self.service_name,
                                         self.method_name,
                                         transport_handler=transport_handler)
        start_span(span)
        self.worker_ctx.data.update(zipkin.create_http_headers_for_new_span())
        try:
            reply = _call(self, *args, **kwargs)
        except:
            stop_span(span)
            raise
        return TracedRpcReply(reply.reply_event, span)

    if _call.__name__ != _call_traced.__name__:
        MethodProxy._call = _call_traced


import hsv
import random
from .led_strip import LedStrip
from hsv import norm_hsv

@micropython.viper
def set_all_sat(strip, sat: int):
    n_pix = int(strip.led_count)
    start_idx = int(strip.buff_start_idx)
    hsv_vals = ptr8(strip.hsv_buff)
    for i in range(0, n_pix):
        base = i * 3 + start_idx
        hsv_vals[base + 1] = sat

@micropython.viper
def rotate(strip, shift: int):
    arr = ptr8(strip.hsv_buff)
    len_arr = int(strip.led_count) * 3
    idx_start = int(strip.buff_start_idx)
    pivot = shift % len_arr
    dst = 0
    src = pivot
    while (dst != src):
        arr[dst + idx_start], arr[src + idx_start] = arr[src + idx_start], arr[dst + idx_start]
        dst += 1
        src += 1
        if src == len_arr:
            src = pivot
        elif dst == pivot:
            pivot = src

class XmasTreeLights:
    def __init__(self, strip: LedStrip):
        self.strip = strip
        self._i = 0
        self.spark_likelyhood = (5, 40)
        self.spark_duration = (2, 6)
        self.brightness = 30
        self.saturation = 255
        self.rot_period = 8
        self.light_span = 3
        self.colors = []
        self._spark_countdown = 0
        self.reset()

    def make_lights(self):
        strip = self.strip
        span = self.light_span
        for i in range(0, self.strip.led_count, span):
            color_idx = i // span % len(self.colors)
            for j in range(0, span):
                color = self.colors[color_idx]
                strip.set(i + j, color[0], color[1], color[2])

    def reset(self):
        self.colors = [
            norm_hsv((120, self.saturation, self.brightness)), # green
            norm_hsv((0, self.saturation, self.brightness)), # red
            norm_hsv((240, self.saturation, self.brightness)), # blue
            norm_hsv((60, self.saturation, self.brightness)), # yellow
            norm_hsv((300, self.saturation, self.brightness)), # magenta
        ]
        self.make_lights()

    def stop(self):
        pass

    def update(self):
        if self._i % self.rot_period == 0:
            rotate(self.strip, 3)

        if (self._spark_countdown > 0):
            set_all_sat(self.strip, 0)
            self._spark_countdown -= 1
        else:
            set_all_sat(self.strip, self.saturation)
            pass

        if random.randint(0, self.spark_likelyhood[1]) == 0:
            self._spark_countdown = random.randint(self.spark_duration[0], self.spark_duration[1])
        self._i += 1


import numpy as np
import matplotlib.pyplot as plt
from pylab import scatter, show, legend, xlabel, ylabel

def dataloader(file):
    with open(file,'r') as file:
        lines = file.readlines()

        dataList = []
        for line in lines:
            el = line.split('\n')[0].split(',')
            dataList.append(el)

    return  np.asarray(dataList,dtype=np.float32)

def dataextractor(datamatrix):
    X_rows,X_cols = datamatrix.shape
    X = datamatrix[:,0:-1]
    Y = datamatrix[:,-1]

    X = X.reshape((X_rows,X_cols-1))
    Y = Y.reshape((X_rows,1))

    ones_mat = np.ones((X_rows,1))
    X = np.append(ones_mat,X,axis=1)

    return X,Y

def sigmoid(z):
    return 1/(1 + np.exp(-z))

def costFunctionReg(theta,X,y,lamb):

    h = sigmoid(np.dot(X,theta))
    theta[0] = 0
    J = (np.dot(y.T,log(h)) + np.dot((1 - y).T,log(1 - h)))/(-m) + (0.5*lamb/m) * sum(np.square(theta));

    temp = (np.dot((h - y).T,X).T/m) + np.dot(lamb/m,theta)
    grad = temp

    return J,grad

def plotter(X,Y):
    pos = np.where(Y == 1)
    neg = np.where(Y == 0)

    scatter(X[pos, 0], X[pos, 1], marker='o', c='b')
    scatter(X[neg, 0], X[neg, 1], marker='x', c='r')
    xlabel('Exam 1 score')
    ylabel('Exam 2 score')
    legend(['Not Admitted', 'Admitted'])
    show()

data = dataloader('ex2data1.txt')

X,Y = dataextractor(data)

plotter(X[:,1:3],Y)


"""
This script migrates all of your current rAPId datasets to a given Layer, if none is given then it will just be the default layer.
It also migrates the schemas to dynamodb, removing the need for crawlers in the process.

The resources that get changed are the:
- The data folders in S3 are moved to reflect the layer that the datasets belong to.
- Schemas will be read from S3 and written to DynamoDB.
- Glue tables will be renamed to reflect the layer that the datasets belong to.
- Glue crawlers will be deleted.
- Permissions get updated
    1. Migrate old subjects have access to the migrated layer
    2. Alter the protected domain permissions to include the new possible layers

Please ensure that none of the crawlers are running when you start this script.
"""
import sys
import os
import argparse
from copy import deepcopy
import json
from typing import List
import re
from pprint import pprint

import boto3
import dotenv

dotenv.load_dotenv()

# Add api to PYTHONPATH
current_dir = os.path.dirname(os.path.abspath(__file__))
target_dir = os.path.abspath(os.path.join(current_dir, "..", ".."))
sys.path.append(target_dir)

from api.domain.schema import Schema  # noqa: E402
from api.application.services.schema_service import SchemaService  # noqa: E402
from api.adapter.athena_adapter import AthenaAdapter  # noqa: E402
from api.common.custom_exceptions import ConflictError  # noqa: E402

AWS_REGION = os.environ["AWS_REGION"]
DATA_BUCKET = os.environ["DATA_BUCKET"]
RESOURCE_PREFIX = os.environ["RESOURCE_PREFIX"]
DATA_PATH = "data"
SCHEMAS_PATH = "data/schemas"
RAW_DATA_PATH = "raw_data"
GLUE_DB = f"{RESOURCE_PREFIX}_catalogue_db"
DYNAMODB_PERMISSIONS_TABLE = f"{RESOURCE_PREFIX}_users_permissions"
GLUE_CATALOGUE_DB_NAME = RESOURCE_PREFIX + "_catalogue_db"

def main(
    layer: str,
    all_layers: List[str],
    s3_client,
    glue_client,
    resource_client,
    dynamodb_client,
    schema_service,
    athena_adapter,
):
    migrate_files(layer, s3_client)
    migrate_permissions(layer, all_layers, dynamodb_client)
    schema_errors = migrate_schemas(layer, schema_service, glue_client)
    migrate_tables(layer, glue_client, athena_adapter)
    migrate_crawlers(glue_client, resource_client)

    if schema_errors:
        print("- These were the schema errors that need to be addressed manually")
        pprint(schema_errors)

def migrate_permissions(layer, all_layers, dynamodb_client):
    print("- Migrating the permissions")
    migrate_protected_domain_permissions(layer, all_layers, dynamodb_client)
    migrate_subject_permissions(layer, dynamodb_client)
    print("- Finished migrating the permissions")

def migrate_subject_permissions(layer, dynamodb_client):
    print("-- Migrating the permissions of the subjects")

    PERMISSIONS_TO_REPLACE = [
        "READ_PUBLIC",
        "READ_PROTECTED",
        "READ_PRIVATE",
        "WRITE_PUBLIC",
        "WRITE_PRIVATE",
        "WRITE_PROTECTED",
    ]
    paginator = dynamodb_client.get_paginator("scan")
    page_iterator = paginator.paginate(
        TableName=DYNAMODB_PERMISSIONS_TABLE,
        ScanFilter={
            "PK": {
                "AttributeValueList": [
                    {
                        "S": "SUBJECT",
                    },
                ],
                "ComparisonOperator": "EQ",
            },
        },
    )
    items = [item for page in page_iterator for item in page["Items"]]
    for item in items:
        permissions = item["Permissions"]["SS"]
        to_replace = False
        for idx, permission in enumerate(permissions):
            for permission_to_change in PERMISSIONS_TO_REPLACE:
                # Amend the permission to include the layer
                if permission.startswith(permission_to_change):
                    to_replace = True
                    permissions[idx] = permission.replace(
                        permission_to_change,
                        permission_to_change.replace("_", f"_{layer.upper()}_"),
                    )

        if to_replace:
            print(f"--- Adding layer to the permissions for subject {item['SK']['S']}")
            dynamodb_client.update_item(
                TableName=DYNAMODB_PERMISSIONS_TABLE,
                Key={key: value for key, value in item.items() if key in ["PK", "SK"]},
                UpdateExpression="set #P = :r",
                ExpressionAttributeNames={"#P": "Permissions"},
                ExpressionAttributeValues={
                    ":r": {"SS": permissions},
                },
            )

    print("-- Finished migrating the permissions of the subjects")

def migrate_protected_domain_permissions(layer, all_layers, dynamodb_client):
    print("-- Migrating the protected domain permissions")
    layer_permissions = [layer.upper() for layer in all_layers] + ["ALL"]
    PROTECTED = "PROTECTED"
    actions = ["READ", "WRITE"]

    for action in actions:
        print(f"--- Migrating the {action} protected domain permissions")
        paginator = dynamodb_client.get_paginator("scan")
        page_iterator = paginator.paginate(
            TableName=DYNAMODB_PERMISSIONS_TABLE,
            ScanFilter={
                "PK": {
                    "AttributeValueList": [
                        {
                            "S": "PERMISSION",
                        },
                    ],
                    "ComparisonOperator": "EQ",
                },
                "SK": {
                    "AttributeValueList": [
                        {
                            "S": action,
                        },
                    ],
                    "ComparisonOperator": "BEGINS_WITH",
                },
                "Sensitivity": {
                    "AttributeValueList": [
                        {
                            "S": PROTECTED,
                        },
                    ],
                    "ComparisonOperator": "EQ",
                },
                "Layer": {
                    "ComparisonOperator": "NULL",
                },
            },
        )
        items = [item for page in page_iterator for item in page["Items"]]
        for item in items:
            print(f"---- Adding layer permissions to the permission {item['SK']['S']}")
            for layer in layer_permissions:
                new_item = deepcopy(item)
                current_prefix = f"{action}_{PROTECTED}"
                future_prefix = f"{action}_{layer}_{PROTECTED}"
                new_item["SK"]["S"] = future_prefix + new_item["SK"]["S"].removeprefix(
                    current_prefix
                )
                new_item["Id"]["S"] = future_prefix + new_item["Id"]["S"].removeprefix(
                    current_prefix
                )
                new_item["Layer"] = {"S": layer}
                dynamodb_client.put_item(
                    TableName=DYNAMODB_PERMISSIONS_TABLE, Item=new_item
                )
            item_keys = {
                key: value for key, value in item.items() if key in ["PK", "SK"]
            }
            dynamodb_client.delete_item(
                TableName=DYNAMODB_PERMISSIONS_TABLE, Key=item_keys
            )

    print("-- Finished migrating the protected domain permissions")

def migrate_tables(layer: str, glue_client, athena_adapter: AthenaAdapter):
    print("- Starting to migrate the tables")
    tables = fetch_all_tables(glue_client, layer)

    for table in tables:
        print(f"-- Migrating the table: {table['Name']}")

        partition_indexes = glue_client.get_partition_indexes(
            DatabaseName=GLUE_DB,
            TableName=table["Name"],
        )
        if partition_indexes:
            athena_adapter.query_sql_async(
                f"MSCK REPAIR TABLE `f{layer}_{table['Name']}`;"
            )
        glue_client.delete_table(Name=table["Name"], DatabaseName=GLUE_DB)

def migrate_crawlers(glue_client, resource_client):
    """
    Steps:
    1. Fetch all of the crawlers
    2. Delete the crawlers
    """
    print("- Starting to migrate the crawlers")
    crawlers = fetch_all_crawlers(resource_client)

    for crawler in crawlers:
        glue_client.delete_crawler(Name=crawler["Name"])

    print("- Finished migrating the crawlers")

def fetch_all_tables(glue_client, layer):
    paginator = glue_client.get_paginator("get_tables")
    page_iterator = paginator.paginate(DatabaseName=GLUE_DB)
    return [
        item
        for page in page_iterator
        for item in page["TableList"]
        if not item["Name"].startswith(f"{layer}_")
    ]

def fetch_all_crawlers(resource_client):
    paginator = resource_client.get_paginator("get_resources")
    page_iterator = paginator.paginate(ResourceTypeFilters=["glue:crawler"])
    resources = [
        item for page in page_iterator for item in page["ResourceTagMappingList"]
    ]
    crawlers = [
        {**resource, **{"Name": resource["ResourceARN"].split(":crawler/")[-1]}}
        for resource in resources
        if f":crawler/{RESOURCE_PREFIX}_crawler/" in resource["ResourceARN"]
    ]
    return crawlers

def migrate_schemas(layer, schema_service: SchemaService, glue_client):
    print("- Migrating the schemas")

    errors = []

    paginator = s3_client.get_paginator("list_objects_v2")
    page_iterator = paginator.paginate(
        Bucket=DATA_BUCKET, Prefix=f"data/{layer}/schemas"
    )
    files = [item for page in page_iterator for item in page["Contents"]]
    for file in files:
        # Read in and write to dynamodb with layer
        print(f"-- Migrating the schema: {file['Key']}")
        res = s3_client.get_object(Bucket=DATA_BUCKET, Key=file["Key"])
        json_object = json.loads(res["Body"].read().decode("utf-8"))
        json_object["metadata"] = {"layer": layer, **json_object["metadata"]}

        schema = Schema.parse_obj(json_object)
        old_table_name = f"{schema.metadata.domain}_{schema.metadata.dataset}_{schema.metadata.version}"
        try:
            old_table = glue_client.get_table(DatabaseName=GLUE_DB, Name=old_table_name)
            glue_column_types = {
                col["Name"]: col["Type"]
                for col in old_table["Table"]["StorageDescriptor"]["Columns"]
                + old_table["Table"]["PartitionKeys"]
            }

            for column in schema.columns:
                column.data_type = glue_column_types[column.name]

            try:
                schema_service.upload_schema(schema)
            except ConflictError:
                schema_service.update_schema(schema)

            s3_client.delete_object(Bucket=DATA_BUCKET, Key=file["Key"])

        except glue_client.exceptions.EntityNotFoundException:
            errors.append(
                f"-- Schema [{file['Key']}] could not be migrated because there is no corresponding table. Please delete the file manually and recreate it"
            )

    print("- Finished migrating the schemas")

    return errors

def migrate_files(layer: str, s3_client):
    print("- Migrating the files")
    # Data
    move_files_by_prefix(s3_client, "data/", f"data/{layer}/")

    # Raw data
    move_files_by_prefix(s3_client, "raw_data/", f"raw_data/{layer}/")
    print("- Finished migrating the files")

def move_files_by_prefix(s3_client, src_prefix: str, dest_prefix: str):
    print(f"-- Moving files from the prefix [{src_prefix}] to [{dest_prefix}]")
    paginator = s3_client.get_paginator("list_objects_v2")
    page_iterator = paginator.paginate(Bucket=DATA_BUCKET, Prefix=src_prefix)

    try:
        files = [item for page in page_iterator for item in page["Contents"]]
    except KeyError:
        print(f"-- There were no files in the prefix [{src_prefix}] to migrate")
        return None

    for file in files:
        src_key = file["Key"]
        # Don't move files if they are already in the destination - can happen if the script is run twice
        if not src_key.startswith(dest_prefix):
            dest_key = re.sub(f"^{src_prefix}", dest_prefix, src_key)
            copy_source = {"Bucket": DATA_BUCKET, "Key": src_key}
            s3_client.copy_object(
                Bucket=DATA_BUCKET,
                CopySource=copy_source,
                Key=dest_key,
            )

            s3_client.delete_object(Bucket=DATA_BUCKET, Key=src_key)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--layer",
        help="Specify the layer to migrate the resources to. Defaults to 'default'",
    )
    parser.add_argument(
        "--all-layers",
        help="Specify all the layers that will exist on this rAPId instance as strings separated by commas e.g 'raw,staging,presentation'. Defaults to 'default'",
    )
    args = parser.parse_args()
    if args.layer:
        layer = args.layer
    else:
        layer = "default"

    if args.all_layers:
        all_layers = args.all_layers.split(",")
    else:
        all_layers = ["default"]

    print(f"Migration to layer [{layer}] with {all_layers} starting")
    s3_client = boto3.client("s3")
    glue_client = boto3.client("glue", region_name=AWS_REGION)
    resource_client = boto3.client("resourcegroupstaggingapi", region_name=AWS_REGION)
    dynamodb_client = boto3.client("dynamodb", region_name=AWS_REGION)
    schema_service = SchemaService()
    athena_adapter = AthenaAdapter()
    main(
        layer,
        all_layers,
        s3_client,
        glue_client,
        resource_client,
        dynamodb_client,
        schema_service,
        athena_adapter,
    )
    print("Migration finished")


"""
A set of commonly-used residuals in batch estimation.

These residuals are
    - the PriorResidual, to assign a prior estimate on the state,
    - the ProcessResidual, which uses a navlie `ProcessModel` to compute an error between
      a predicted state and the actual state,
    - a MeasurementResidual, which uses a navlie `Measurement` to compare
      a true measurement to the measurement predicted by the `MeasurementModel`.

"""

from abc import ABC, abstractmethod
from typing import Hashable, List, Tuple
from navlie.types import State, ProcessModel, Measurement, Input
import numpy as np

class Residual(ABC):
    """
    Abstract class for a residual to be used in batch estimation.

    Each residual must implement an evaluate(self, states) method,
    which returns an error and Jacobian of the error with
    respect to each of the states.

    Each residual must contain a list of keys, where each key corresponds to a
    variable for optimization.
    """

    def __init__(self, keys: List[Hashable]):
        # If the hasn't supplied a list, make a list
        if isinstance(keys, list):
            self.keys = keys
        else:
            self.keys = [keys]

    @abstractmethod
    def evaluate(
        self,
        states: List[State],
        compute_jacobian: List[bool] = None,
    ) -> Tuple[np.ndarray, List[np.ndarray]]:
        """
        Evaluates the residual and Jacobians.

        Parameters
        ----------
        states : List[State]
            List of states for optimization.
        compute_jacobian : List[bool], optional
            optional flag to compute Jacobians, by default None

        Returns
        -------
        Tuple[np.ndarray, List[np.ndarray]]
            Returns the error and a list of Jacobians.
        """
        # TODO: seems more appropriate to receive states as a dict with
        # corresponding state names than as a list.
        pass

    def jacobian_fd(self, states: List[State], step_size=1e-6) -> List[np.ndarray]:
        """
        Calculates the model jacobian with finite difference.

        Parameters
        ----------
        states : List[State]
            Evaluation point of Jacobians, a list of states that
            the residual is a function of.

        Returns
        -------
        List[np.ndarray]
            A list of Jacobians of the measurement model with respect to each of the input states.
            For example, the first element of the return list is the Jacobian of the residual
            w.r.t states[0], the second element is the Jacobian of the residual w.r.t states[1], etc.
        """
        jac_list: List[np.ndarray] = [None] * len(states)

        # Compute the Jacobian for each state via finite difference
        for state_num, X_bar in enumerate(states):
            e_bar = self.evaluate(states)
            size_error = e_bar.ravel().size
            jac_fd = np.zeros((size_error, X_bar.dof))

            for i in range(X_bar.dof):
                dx = np.zeros((X_bar.dof, 1))
                dx[i, 0] = step_size
                X_temp = X_bar.plus(dx)
                state_list_pert: List[State] = []
                for state in states:
                    state_list_pert.append(state.copy())

                state_list_pert[state_num] = X_temp
                e_temp = self.evaluate(state_list_pert)
                jac_fd[:, i] = (e_temp - e_bar).flatten() / step_size

            jac_list[state_num] = jac_fd

        return jac_list

class PriorResidual(Residual):
    """
    A generic prior error.
    """

    def __init__(
        self,
        keys: List[Hashable],
        prior_state: State,
        prior_covariance: np.ndarray,
    ):
        super().__init__(keys)
        self._cov = prior_covariance
        self._x0 = prior_state
        # Precompute square-root of info matrix
        self._L = np.linalg.cholesky(np.linalg.inv(self._cov))

    def evaluate(
        self,
        states: List[State],
        compute_jacobians: List[bool] = None,
    ) -> Tuple[np.ndarray, List[np.ndarray]]:
        """
        Evaluates the prior error of the form

            e = x.minus(x0),

        where :math:`\mathbf{x}` is our operating point and
        :math:`\mathbf{x}_0` is a prior guess.
        """
        x = states[0]
        error = x.minus(self._x0)
        # Weight the error
        error = self._L.T @ error
        # Compute Jacobian of error w.r.t x
        if compute_jacobians:
            jacobians = [None]

            if compute_jacobians[0]:
                jacobians[0] = self._L.T @ x.minus_jacobian(self._x0)
            return error, jacobians

        return error

class ProcessResidual(Residual):
    """
    A generic process residual.

    Can be used with any :class:`navlie.types.ProcessModel`.
    """

    def __init__(
        self,
        keys: List[Hashable],
        process_model: ProcessModel,
        u: Input,
    ):
        super().__init__(keys)
        self._process_model = process_model
        self._u = u

    def evaluate(
        self,
        states: List[State],
        compute_jacobians: List[bool] = None,
    ) -> Tuple[np.ndarray, List[np.ndarray]]:
        """
        Evaluates the process residual.

        An input :math:`\mathbf{u}` is used to propagate the state
        :math:\mathbf{x}_{k-1}` through the process model, to generate
        :math:`\hat{\mathbf{x}}_{k}`. This operation is written as

        .. math::
            \hat{\mathbf{x}}_k = \mathbf{f}(\mathbf{x}_{k-1}, \mathbf{u}, \Delta t).

        An error is then created as

            e = x_k.minus(x_k_hat),

        where :math:`\mathbf{x}_k` is our current operating point at time :math:`t_k`.
        """
        x_km1 = states[0]
        x_k = states[1]
        dt = x_k.stamp - x_km1.stamp

        # Evaluate the process model, compute the error
        x_k_hat = self._process_model.evaluate(x_km1.copy(), self._u, dt)
        e = x_k.minus(x_k_hat)

        # Scale the error by the square root of the info matrix
        L = self._process_model.sqrt_information(x_km1, self._u, dt)
        e = L.T @ e

        # Compute the Jacobians of the residual w.r.t x_km1 and x_k
        if compute_jacobians:
            jac_list = [None] * len(states)
            if compute_jacobians[0]:
                jac_list[0] = -L.T @ self._process_model.jacobian(
                    x_km1, self._u, dt
                )
            if compute_jacobians[1]:
                jac_list[1] = L.T @ x_k.minus_jacobian(x_k_hat)

            return e, jac_list

        return e

class MeasurementResidual(Residual):
    """
    A generic measurement residual.

    Can be used with any :class:`navlie.Measurement`.
    """

    def __init__(self, keys: List[Hashable], measurement: Measurement):
        super().__init__(keys)
        self._y = measurement

    def evaluate(
        self,
        states: List[State],
        compute_jacobians: List[bool] = None,
    ) -> Tuple[np.ndarray, List[np.ndarray]]:
        """
        Evaluates the measurement residual.

        The error is computed as

        .. math::
            \mathbf{e} = \mathbf{y} - \mathbf{g} (\mathbf{x}).

        The Jacobian of the residual with respect to the state
        is then the negative of the measurement model Jacobian.
        """
        # Extract state
        x = states[0]

        # Compute predicted measurement
        y_check = self._y.model.evaluate(x)
        e = self._y.value.reshape((-1, 1)) - y_check.reshape((-1, 1))

        # Weight error by square root of information matrix
        L = self._y.model.sqrt_information(x)
        e = L.T @ e

        if compute_jacobians:
            jacobians = [None] * len(states)

            if compute_jacobians[0]:
                jacobians[0] = -L.T @ self._y.model.jacobian(x)
            return e, jacobians

        return e


#-----------------------------------------------------
# Mimas: conference submission and review system
# (c) Allan Kelly 2016-2020 http://www.allankelly.net
# Licensed under MIT License, see LICENSE file
# -----------------------------------------------------

# System imports

# Application imports

# Local imports
import submissionrecord
from submissionrecord import retrieve_conference_submissions

def is_submitted(conf_key, speaker_email, talk_title):
    for s in submissionrecord.retrieve_conference_submissions(conf_key):
        if speaker_email == s.email():
            if talk_title == s.title():
                return True

    return False

def count_submissions(conf_key, speaker_key):
    subs = retrieve_conference_submissions(conf_key)
    count = 0
    for s in subs:
        if s.talk.parent() == speaker_key:
            count = count + 1

    return count


import shutil
from concurrent.futures import ThreadPoolExecutor
from concurrent.futures import ProcessPoolExecutor
import time

def copy_file(src, dest):
	shutil.copy2(src, dest)

def main():

	t1 = time.time()
	copy_file('./filename.txt', './new_dest/new_file.txt_1')
	t2 = time.time()
	print(f'Without threading took {t2-t1} seconds')

	t3 = time.time()
	executor = ThreadPoolExecutor(5)
	future = executor.submit(copy_file, './filename.txt', './new_dest/new_file.txt_2')
	t4 = time.time()
	print(f'ThreadPoolExecutor took {t4-t3} seconds')

	t5 = time.time()
	executor = ProcessPoolExecutor(5)
	future = executor.submit(copy_file, './filename.txt', './new_dest/new_file.txt_3')
	t6 = time.time()
	print(f'ProcessPoolExecutor took {t6-t5} seconds')

if __name__ == '__main__':
	main()