from typing import Dict, List, Optional

class Display:
    def __init__(self, mapping: Dict[str, str]) -> None:
        self.map = mapping

        self.numbers = {}
        self.numbers[0] = set(
            self.map["a"]
            + self.map["b"]
            + self.map["c"]
            + self.map["d"]
            + self.map["e"]
            + self.map["f"]
        )
        self.numbers[1] = set(self.map["b"] + self.map["c"])
        self.numbers[2] = set(
            self.map["a"]
            + self.map["b"]
            + self.map["d"]
            + self.map["e"]
            + self.map["g"]
        )
        self.numbers[3] = set(
            self.map["a"]
            + self.map["b"]
            + self.map["c"]
            + self.map["d"]
            + self.map["g"]
        )
        self.numbers[4] = set(
            self.map["b"] + self.map["c"] + self.map["f"] + self.map["g"]
        )
        self.numbers[5] = set(
            self.map["a"]
            + self.map["c"]
            + self.map["d"]
            + self.map["f"]
            + self.map["g"]
        )
        self.numbers[6] = set(
            self.map["a"]
            + self.map["c"]
            + self.map["d"]
            + self.map["e"]
            + self.map["f"]
            + self.map["g"]
        )
        self.numbers[7] = set(self.map["a"] + self.map["b"] + self.map["c"])
        self.numbers[8] = set(
            self.map["a"]
            + self.map["b"]
            + self.map["c"]
            + self.map["d"]
            + self.map["e"]
            + self.map["f"]
            + self.map["g"]
        )
        self.numbers[9] = set(
            self.map["a"]
            + self.map["b"]
            + self.map["c"]
            + self.map["d"]
            + self.map["f"]
            + self.map["g"]
        )

    def convert_to_int(self, output: List[str]) -> int:
        displayed_number = ""
        for num in output:
            num = set(list(num))
            for key, value in self.numbers.items():
                if num == value:
                    displayed_number += str(key)
        return int(displayed_number)

def get_one(input: list) -> Optional[str]:
    for segment in input:
        if len(segment) == 2:
            return segment
    return None

def get_four(input: list) -> Optional[str]:
    for segment in input:
        if len(segment) == 4:
            return segment
    return None

def get_seven(input: list) -> Optional[str]:
    for segment in input:
        if len(segment) == 3:
            return segment
    return None

def get_eight(input: list) -> Optional[str]:
    for segment in input:
        if len(segment) == 7:
            return segment
    return None

def observations_len_5(input: List[str]) -> List[str]:
    """
    Get all observations that could be 5, 3, or 2.

    Parameters
    ----------
    input : List[str]
        All of the observations.

    Returns
    -------
    List[str]
        The options with a length of 5.
    """
    possibles = []
    for segment in input:
        if len(segment) == 5:
            possibles.append(segment)

    return possibles

def solve_display(input: List[List[str]]) -> Dict[str, str]:
    top = {"a", "b", "c", "d", "e", "f", "g"}
    topl = {"a", "b", "c", "d", "e", "f", "g"}
    topr = {"a", "b", "c", "d", "e", "f", "g"}
    middle = {"a", "b", "c", "d", "e", "f", "g"}
    bottoml = {"a", "b", "c", "d", "e", "f", "g"}
    bottomr = {"a", "b", "c", "d", "e", "f", "g"}
    bottom = {"a", "b", "c", "d", "e", "f", "g"}

    # Solve top if 1 and 7 present
    seven = get_seven(input[0])
    one = get_one(input[0])
    four = get_four(input[0])
    if seven and one and four:
        seven = set(list(seven))
        one = set(list(one))

        # Top Solved
        top = seven.difference(one)

        # Right side narrowed down
        topr = set(list(one))
        bottomr = set(list(one))

        # Narrow down topl and middle
        four = set(list(four))
        middle = four.difference(one)
        topl = four.difference(one)

        bottom = bottom.difference(top, topr, bottomr, middle, topl)
        bottoml = bottoml.difference(top, topr, bottomr, middle, topl)

        # Solve 5 and bottoml, bottomr, bottom
        options = []
        # Get possibles for 5, 3, 2
        for option in observations_len_5(input[0]):
            options.append(set(list(option)))

        for option in options:
            # If true, the option is number 5
            if (
                len(bottomr.intersection(option)) == 1
                and len(bottom.intersection(option)) == 1
                and len(topl.intersection(option)) == 2
            ):
                diff = option.difference(top, topl, middle, bottom)
                bottomr = diff
                topr = topr.difference(bottomr)
                bottom = option.difference(top, topl, middle, bottomr)
                bottoml = bottoml.difference(bottom)
                break

        # Get possibles for 5, 3, 2
        for option in observations_len_5(input[0]):
            options.append(set(list(option)))

        for option in options:
            # If true, option is number 2
            if (
                len(option.intersection(middle)) == 1
                and len(option.intersection(bottomr)) == 0
                and not (
                    # Not Number 5
                    len(bottomr.intersection(option)) == 1
                    and len(bottom.intersection(option)) == 1
                    and len(topl.intersection(option)) == 2
                )
            ):
                middle = option.difference(top, topr, bottoml, bottom)
                topl = topl.difference(middle)

        mapping = {
            "a": "".join(top),
            "b": "".join(topr),
            "c": "".join(bottomr),
            "d": "".join(bottom),
            "e": "".join(bottoml),
            "f": "".join(topl),
            "g": "".join(middle),
        }

        return mapping

def read_file(filename) -> List[List[str]]:
    with open(filename, "r") as contents:
        readings = []
        for line in contents:
            data = line.strip()
            data = data.split(" | ")
            observations = data[0].split()
            output = data[1].split()
            readings.append([observations, output])

    return readings

def get_occurances(data) -> int:
    occurances = 0

    for line in data:
        for observation in line[1]:
            if (
                len(observation) == 3
                or len(observation) == 7
                or len(observation) == 4
                or len(observation) == 2
            ):
                occurances += 1

    return occurances

def part_one(filename: str) -> int:
    readings = read_file(filename)
    return get_occurances(readings)

def part_two(filename: str) -> int:
    sum_output = 0
    readings = read_file(filename)
    for line in readings:
        mapping = solve_display(line)
        display = Display(mapping)
        sum_output += display.convert_to_int(line[1])
    return sum_output

if __name__ == "__main__":
    filename = "data/08.data"
    print("Part One")
    count = part_one(filename)
    print(f"In the output values, 1, 4, 7, 8 appear:  {count} times")

    print("Part Two")
    print(f"The sum of the output values: {part_two(filename)}")


class Solution:
    def validTree(self, n: int, edges: List[List[int]]) -> bool:
        adjList = defaultdict(list)
        for u, v in edges:
            adjList[u].append(v)
            adjList[v].append(u)

        seen = set()
        def dfs(node, parent):
            if node in seen:
                return False
            seen.add(node)
            for nei in adjList[node]:
                if nei == parent:
                    continue
                if not dfs(nei, node):
                    return False
            return True

        return dfs(0, -1) and n == len(seen)


class BankAccount:
    def __init__ (self, int_rate, balance):
        self.int_rate = int_rate
        self.balance = balance

class User:
    def __init__ (self, name, email, int, bal):
        self.name = name
        self.email = email
        self.account = BankAccount(int, bal)

garett = User("Garett", "hawaii.edu", 0.02, 1000)

print(garett.account.balance)


tableData = [["apples", "oranges", "cherries", "banana"],
            ["Alice", "Bob", "Carol", "David"],
            ["dogs", "cats", "moose", "goose"]]
#initialise 3 objects that will contain the 3 lists of the object tableData, return an object of type list
#list1 = []
#list2 = []
#list3 = []

#put the 3 lists of tableData in each object we are created, return objectcs of type list
list1,list2, list3 = tableData

#condition to browse the three lists at the same time, return an object of type bool
for item1,item2,item3 in zip(list1,list2,list3):

    #print the result like in a table on the screen, return an object of type NoneType
    print("{:>9}{:>6}{:>7}".format(item1,item2,item3))


import os

import pytest

BASE_DIR = os.path.dirname(__file__)

@pytest.fixture
def voila_args_extra():
    return ["--VoilaConfiguration.strip_sources=False", "--VoilaExecutor.timeout=240"]

async def test_no_strip_sources(http_server_client, base_url):
    response = await http_server_client.fetch(base_url)
    assert response.code == 200
    html_text = response.body.decode("utf-8")
    assert "Hi Voilà" in html_text
    assert "print" in html_text, "the source code should *NOT* be stripped"


# -*- coding: utf-8 -*-
from sys import stdin
stdin = open("../../stdin.txt")

N = int(stdin.readline())
A = [int(i) for i in stdin.readline().split()]

deg_gcd = -1
res = 0

for i in range(2, 1001):
    k = sum(a % i == 0 for a in A)
    if res < k:
        res = k
        deg_gcd = i

print(deg_gcd)


from __future__ import print_function
from past.builtins import cmp
from builtins import range
from builtins import object
import sys,os
os.chdir(os.path.dirname(os.path.abspath(__file__))) # add working directory
sys.path.append(os.path.abspath("../util")) # add relative path

from advutils.threader import api, Designator, IterDecouple

if __name__ == "__main__":

    def worker(q):
        while not q.empty():
            next_job = q.get()
            print('Processing job:', next_job.description)

    class Job(object):
        def __init__(self, priority, description):
            self.priority = priority
            self.description = description
            print('New job:', description)
        def __cmp__(self, other):
            return cmp(self.priority, other.priority)

    #api.spawn = True
    #q = PriorityQueue()
    q = Designator()
    # https://pymotw.com/2/Queue/
    q.put( Job(10, 'Low-level job 1') )
    q.put( Job(1, 'Important job 1') )
    q.put( Job(10, 'Low-level job 2') )
    q.put( None)
    q.put( Job(3, 'Mid-level job') )
    q.put( Job(1, 'Important job 2') )
    q.put( Job(10, 'Low-level job 3') )
    """
    print len(q)
    for i in q:
        print i.description
    print len(q)
    """
    import multiprocessing as m
    import threading as th

    #processes = [m.Process(target=worker,args=(q,)) for i in xrange(3)]
    processes = [th.Thread(target=worker,args=(q,)) for i in range(3)]
    for p in processes:
        p.start()
    for p in processes:
        p.join()
    #print len(q)

if __name__ == '__main__':
    from multiprocessing import freeze_support
    freeze_support()

    def handler_timeout(signum, frame):
        raise Exception("Function taking too long")

    #import signal
    #signal.signal(signal.SIGALRM, handler_timeout)

    def demo_decouple_iter():
        """

        :return:
        """
        import time, random

        class SimulateProcess(object):
            def __init__(self, value, time_processing):
                self.value = value
                self.time_processing = time_processing
                self._creation_time = time.time()

            def process(self):
                time.sleep(self.time_processing)
                return self

            __call__ = process

            def __str__(self):
                return str(self.value)

        def simulate_iterable(max_time=1, no_itmes = 10):
            """
            Simulates iterable that gives data un-evenly in time,
            fast or really slow in some cases.

            :param max_time: maximum random time to simulate process
            :param no_itmes: number of processes to create
            :return: iterator
            """
            for order in range(no_itmes):
                tp = random.random() * max_time
                yield SimulateProcess(order, tp), tp

        def simulate_consumer(myiter, call = False, stop_in= None, test_name = None):
            """
            Simulates iterator consumption. The idea is to simulate a process
            that wants to get data as fast as possible, faster is better.

            :param myiter: iterator of SimulateProcess instances
            :param buffsize: buffer size
            :return: (No_items, time_ideal, time_normal_for, time_test_for,
                    time_spared_normal, time_loosed_ideal)
            """
            if test_name is None:
                test_name = type(myiter)
            name = " ({})".format(test_name)
            print("Testing for loop {}".format(name))

            worker_times = [0] # time of each worker, 0 is added to simulate empty
            times = [] # time for each epoch
            time_test_for = time.time()  # real process time
            time_epoch_start = time.time() # epoch start time
            for i, (task, time_processing) in enumerate(myiter):
                if stop_in is not None and i == stop_in:
                    myiter.close()
                if call:
                    task()
                time_epoch_end = time.time() # epoch finish time
                time_epoch = time_epoch_end - time_epoch_start # epoch time
                worker_times.append(time_processing) # append real worker time
                times.append(time_epoch) # append for loop epoch time
                print("value: {}, epoch time: {}, worker time: {}"
                      "".format(task,time_epoch,time_processing))
                time_epoch_start = time_epoch_end # update epoch start time

            No_items = len(times) # Number of items
            time_test_for = time.time() - time_test_for
            time_ideal = max(worker_times) # ideal process time
            time_normal_for = sum(worker_times) # not decoupled time
            time_spared_normal = time_normal_for - time_test_for # spared time from normal for loop
            time_loosed_ideal = time_test_for - time_ideal # loosed time from ideal
            print("processed items: ", No_items)
            print("ideal process time: ", time_ideal)
            print("normal for loop time: ", time_normal_for)
            print("tested for loop time{}: ".format(name), time_test_for)
            print("time spared (from normal for): ", time_spared_normal)
            print("loosed time (from ideal):", time_loosed_ideal)
            return (No_items, time_ideal, time_normal_for, time_test_for,
                    time_spared_normal, time_loosed_ideal)

        api.spawn = False
        myiter = list(simulate_iterable())
        stop_in = 3
        decoupled = IterDecouple(myiter, buffsize=0)
        decoupled.call_func = lambda x: (x[0](), x[1])
        decoupled.buffsize = 0 # unlimited

        start_before = 2
        print("\ntest: detached for loop is started {} seconds before\n"
              "".format(start_before))
        decoupled.start()
        time.sleep(start_before)
        (No_items, time_ideal, time_normal_for, time_test_for,
        time_spared_normal, time_loosed_ideal) = simulate_consumer(decoupled, stop_in = stop_in)
        if stop_in is None:
            assert No_items == len(myiter)
            assert time_ideal < time_test_for < time_normal_for

        processes = 5
        print("\ntest: detached for loop is started in normal for loop with "
              "{} processes\n".format(processes))
        decoupled.processes = processes
        (No_items, time_ideal, time_normal_for, time_test_for,
        time_spared_normal, time_loosed_ideal) = simulate_consumer(decoupled, stop_in = stop_in)
        if stop_in is None:
            assert No_items == len(myiter)
            assert time_ideal < time_test_for < time_normal_for

        print("\ntest: detached for loop is started {} seconds before in "
              "normal for loop with {} processes\n".format(start_before, processes))
        decoupled.processes = processes
        decoupled.start()
        time.sleep(start_before)
        (No_items, time_ideal, time_normal_for, time_test_for,
        time_spared_normal, time_loosed_ideal) = simulate_consumer(decoupled, stop_in = stop_in)
        if stop_in is None:
            assert No_items == len(myiter)
            assert time_test_for < time_normal_for # the ideal should be 0

        # disable processes
        decoupled.processes = None

        print("\ntest: detached for loop is started in normal for loop\n")
        (No_items, time_ideal, time_normal_for, time_test_for,
        time_spared_normal, time_loosed_ideal) = simulate_consumer(decoupled, stop_in = stop_in)
        if stop_in is None:
            assert No_items == len(myiter)

        print("\ntest: simple list\n")
        (No_items, time_ideal, time_normal_for, time_test_for,
        time_spared_normal, time_loosed_ideal) = simulate_consumer(myiter, call=True) # tests: simple list
        if stop_in is None:
            assert No_items == len(myiter)

        print("\ntest: empty iteration\n")
        (No_items, time_ideal, time_normal_for, time_test_for,
        time_spared_normal, time_loosed_ideal) = simulate_consumer(IterDecouple([]))

        """
        # tests repeatability and that processes do not block
        i = 0
        while True:
            signal.alarm(5)  # timeout of 5 seconds
            i+=1
            print("Try",i)
            try:
                simulate_consumer([]) # tests: empty iteration
            except Exception as e:
                signal.alarm(0)
                raise e
        """

    demo_decouple_iter()


from typing import Iterable, Tuple

import ruamel.yaml

from plover.steno_dictionary import StenoDictionary
from plover.steno import normalize_steno, STROKE_DELIMITER

class YAMLDictionary(StenoDictionary):
    '''
    Stenography dictionary for YAML files.

    Format:

    Translation 1:
    - STROKE_1
    - STROKE_2
    Translation 2:
    - STROKE_1

    etc.
    '''

    def _load(self, filename: str):
        '''
        Populates the dictionary entries for Plover from a file.

        :param filename: The file path of the dictionary to load.
        '''

        self.update(YAMLDictionary.load_yaml_file(filename))

    def _save(self, filename: str):
        '''
        Saves off the current dictionary state in Plover to a file.

        :param filename: The file path of the dictionary to save to.
        '''

       # Group dictionary by value, sorted alphabetically
        data = {}

        for strokes, translation in sorted(self._dict.items(), key=lambda x: x[1].casefold()):
            # Need to join the multi-stroke entries into one stroke string first
            stroke = STROKE_DELIMITER.join(strokes)
            data.setdefault(translation, []).append(stroke)
            data[translation] = sorted(data[translation])

        # Write out the data
        yaml = ruamel.yaml.YAML(typ='safe')
        yaml.allow_unicode = True
        yaml.default_flow_style = False
        yaml.indent(sequence=4, offset=2)

        with open(filename, 'w', encoding='utf-8') as out_file:
            yaml.dump(data, out_file)

    @staticmethod
    def load_yaml_file(filename: str) -> Iterable[Tuple[Tuple[str], str]]:
        '''
        Loads a YAML dictionary file and provides an iterable to its
        stroke to translation mappings.

        :param filename: The file path of the YAML dictionary to load.
        :return: An iterable that provides tuples of stroke tuple, translation.
        '''

        # Load the data. Can't use the round-trip loader for performance reasons
        yaml = ruamel.yaml.YAML(typ='safe')

        with open(filename, 'r', encoding='utf-8') as in_file:
            data = yaml.load(in_file)

        # Provide tuples of stroke tuple, translation
        for translation, strokes in data.items():
            for stroke in strokes:
                yield (normalize_steno(stroke), translation)


with open("calories.txt", 'r') as cals:
    cals = list([x.replace('\n', '') for x in cals])
    cals.append('')
    backpack=list()
    count=0
    for line in cals:
        if line!='':
            count = count + float(line)
        else:
            backpack.append(count)
            count=0

sum(sorted(backpack,reverse=True)[0:3])


from collections import namedtuple
from copy import deepcopy
from string import ascii_uppercase

def parse_input(filename: str = "input") -> list:
    with open(filename, encoding="utf-8") as _:
        return _.read().splitlines()

stacks = [[] for _ in range(20)]

it = iter(parse_input("input.txt"))

# Parse input stacks
while True:
    line = next(it)
    if line.startswith(" 1"):
        break
    for i, c in enumerate(line):
        if c in ascii_uppercase:
            real_index = (i - 1) // 4
            stacks[real_index].append(c)

for i in range(len(stacks) -1, -1, -1):
    if stacks[i]:
        del stacks[i+1:]
        break

next(it)

MoveCommand = namedtuple("MoveCommand", ("count", "src", "dest"))

def parse_commands():
    cmd = []
    for _line in it:
        parts = _line.split(" ")
        count = int(parts[1])
        src = int(parts[3]) - 1
        dest = int(parts[5]) - 1
        cmd.append(MoveCommand(count, src, dest))
    return cmd

commands = parse_commands()

stacks_orig = deepcopy(stacks) # Saving for part 2

# Part 1
def exec_move(cmd):
    for _ in range(cmd.count):
        stacks[cmd.dest].insert(0, stacks[cmd.src].pop(0))

for _cmd in commands:
    exec_move(_cmd)

print("".join(i[0] for i in stacks))

stacks = stacks_orig

# Part 2
def exec_move2(cmd):
    temp = []
    for _ in range(cmd.count):
        temp.append(stacks[cmd.src].pop(0))
    stacks[cmd.dest] = temp + stacks[cmd.dest]

for _cmd in commands:
    exec_move2(_cmd)

print("".join(i[0] for i in stacks))


import json
import socket
from threading import Thread
import time
from libraries.udp_server_multi_client import UDPServerMultiClient

class Gateway():

    __address: tuple[str, int]
    __client: dict = {
        "socket": socket.socket,
        "address": tuple[str, int],
        "conn": socket.socket,
        "is_connected": False
    }
    __buffer_size: int = 1024
    __drone_server: UDPServerMultiClient
    __is_client_connected: bool = False

    def __init__(self, gateway_ip: str, gateway_port: int, max_drones: int) -> None:
        self.__drone_server = UDPServerMultiClient(
            gateway_ip, gateway_port, max_drones)
        self._setup_gateway_client(gateway_ip, gateway_port)

    def _setup_gateway_client(self, gateway_ip: str, gateway_port: int) -> None:
        """
        Setup the connection between the gateway and the client.
        """
        self.__address = (gateway_ip, gateway_port)
        self.__client["socket"] = socket.socket(
            family=socket.AF_INET, type=socket.SOCK_STREAM)
        self.__client["socket"].bind(self.__address)

    def __connect_client(self) -> None:
        self.__client["socket"].listen()
        self.__client["conn"], self.__client["address"] = self.__client["socket"].accept()
        if (not self.__is_client_connected):
            print(f"Connected to Client: {self.__client['address']}")
            self.__is_client_connected = True

    def main_loop(self) -> None:
        """
        Listen if there are orders to be shipped.
        """
        self.__connect_client()
        drone_server_thread = Thread()
        client_thread = Thread()

        try:
            while True:
                # creating the drone listener thread
                if self.__is_client_connected:
                    if not drone_server_thread.is_alive():
                        drone_server_thread = Thread(
                            target=self.__drone_server.thread_request_handle)
                        drone_server_thread.daemon = True
                        drone_server_thread.start()
                        time.sleep(0.5)

                # handling the client
                if not client_thread.is_alive():
                    client_thread = Thread(self.client_handling())
                    client_thread.daemon = True
                    client_thread.start()

        except ConnectionResetError:
            print(f"Client {self.__client['address']} disconnected!")
            self.__is_client_connected = True

    def client_handling(self) -> None:
        """
        Handles the client
        """
        self.send_drone_information()

        data = self.__client["conn"].recv(self.__buffer_size)
        if not data:
            return
        order = eval(data.decode("utf-8"))
        if not self._check_order(order):
            self._send_message_to_client("Error: wrong order syntax")
        elif not "update" in order:
            print(f"Clinet request: {order}")
            self._process_order(order)

    def send_drone_information(self) -> bool:
        """
        Send the drones id that are free to the client.
        """
        free_drones = self.__drone_server.get_free_drones()
        drones = []
        for i in free_drones:
            drones.append(i["id"])

        try:
            self.__client["conn"].send(str(drones).encode())
            return True
        except:
            print(f"Client {self.__client['address']} disconnected!")
            self.__is_client_connected = False
            return False

    def _send_message_to_client(self, message: str) -> None:
        """
        Send a message to the client.
        """
        self.__client["conn"].send(message.encode())

    def _process_order(self, order: dict) -> None:
        """
        Process the order and send the drone to the order destination.
        """
        self.__drone_server.thread_send_message(
            order["order_destination"], self.__drone_server.get_drone_address_by_id(order["drone_id"]))

    def _check_order(self, order: dict) -> bool:
        """
        Check if the order is valid and if the drone is available.
        """
        if "update" in order:
            return True

        # check if the order syntax is valid
        if not all(keys in order for keys in ("drone_id", "order_destination")):
            print("Wrong order syntax")
            return False

        # check if the drone exists
        if order["drone_id"] not in self.__drone_server.get_drones():
            print("The selected drone does not exist")
            return False

        # check if the drone is free
        drone_id_list = []
        for i in self.__drone_server.get_free_drones():
            drone_id_list.append(i["id"])
        if order["drone_id"] not in drone_id_list:
            print("The selected drone is not free")
            return False

        return True

    def close_client_connection(self) -> None:
        """
        Close the connection between the gateway and the client.
        """
        self.__client["socket"].close()

if __name__ == '__main__':
    with open('setup.json') as json_file:
        data = json.load(json_file)
        gateway = Gateway("", int(data['gateway']["port"]), data["max_drones"])
        gateway.main_loop()

        gateway.close_client_connection()


import keyword
import string

from django.shortcuts import render, redirect
from .models import *
# from .serializers import ServicePriceSerializers, RoomCategorySerializers, OrdersSerialiser
from rest_framework.views import APIView
from rest_framework.response import Response
from .forms import *
from rest_framework import generics

# class RoomCategoryAPIView(generics.CreateAPIView):
#     # queryset = OrderItem.objects.all()
#     # serializer_class = OrdersSerialiser

# class RoomCategoryAPIView(APIView):
#     def get(self, request):
#         return Response({
#             'categoryroom': RoomCategorySerializers(RoomCategory.objects.all(), many=True).data
#         })

#     def post(self, request):
#         services: list = request.POST.getlist('services')
#         for item in services:
#             # {name: "ser-1", price: 35},
#             # {name: "ser-2", price: 25}
#             pass
#         data = OrdersSerialiser(data=request.data)
#         if data.is_valid():2
#             return Response({
#                 'status': "Post qo'shildi",
#                 'data': data.errors
#             })
#         data.save()
#         return Response({
#             'status': 'success',
#             'data': OrdersSerialiser(data.instance).data
#         })

# class ServicePriceAPIView(APIView):
#     def get(self, request):
#         return Response({
#             'servicecategoryprice': ServicePriceSerializers(Service.objects.all(), many=True).data
#         })

#     def post(self, request):
#         data = OrdersSerialiser(data=request.data)
#         if data.is_valid():
#             return Response({
#                 'status': "Post qo'shildi",
#                 'data': data.errors
#             })
#         data.save()
#         return Response({
#             'status': 'success',
#             'data': OrdersSerialiser(data.instance).data
#         })

def home(request):
    caruselimg = CaruselImage.objects.filter()
    form = OrderModelForm()
    if request.method == 'POST':
        form = OrderModelForm(request.POST)
        # print("QWERTYTREAWERT------------", form)
        if form.is_valid():
            print('AAAAAAAAA------', form)
            form.save()
            return redirect('myprint:home')
        else:
            print("TTTTTTTTTTTT------------>>>>>>>", form.errors)
            form = OrderModelForm()
            print("SSSSSSSSSSSSSS------------>>>>>>>", form.errors)
    context = {
        'form': form,
        'caruselimg': caruselimg,
    }
    return render(request, 'main/index.html', context=context)

def product_detail(request, pk):
    detailCarusel = CaruselDetail.objects.filter(caruselimage_id=pk)
    context = {
        'detailCarusel': detailCarusel
    }
    return render(request, 'main/product-detail.html', context=context)

def about(request):
    return render(request, 'main/about.html')

def contact(request):
    form = ContactModelForm()
    if request.method == "POST":
        form = ContactModelForm(request.POST)
        if form.is_valid():
            form.save()
            return redirect('myprint:home')
        else:
            form = ContactModelForm()
    context = {
        'form': form
    }
    return render(request, 'main/contact.html', context=context)

def gallary(request):
    proj = Room2.objects.filter()
    context = {
        'proj': proj,
    }
    return render(request, 'main/gallary.html', context=context)

def gallary_details(request, pk):
    project = Project.objects.filter(gallary_id=pk)
    context = {
        'project': project,
    }
    return render(request, 'main/gallary-details.html', context)

def guests(request):

    return render(request, 'main/guests.html')

# def services(request):
#     return render(request, 'main/services.html')

# def services(request):
#     cat = RoomCategory.objects.all()
#     pr = Price.objects.all()
#     if request.method == "POST":
#         check = request.POST.getlist('checks[]')
#         option = request.POST.getlist('option[]')
#         print("WWWWWWWWWWW_______>>>>>>>>>", check)
#         print("OOOOOOOOOOO_______>>>>>>>>>", option)
#         # req_obj = Orders.objects.create(nameroom=option, nameservice=check)
#         # req_obj.save()
#         return redirect('myprint:services')
#     context = {
#         'cat': cat,
#         'pr': pr
#     }
#     return render(request, 'main/services.html', context=context)

def product_detail(request, pk):
    detailCarusel = CaruselDetail.objects.filter(carusel_id=pk)
    context = {
        'detailCarusel': detailCarusel
    }
    return render(request, 'main/product-details.html', context=context)

# def services(request):
#     card = CardServices.objects.all()
#     cat = Room.objects.all()
#     pr = Service.objects.all()
#     if request.method == "POST":
#         services = request.POST.getlist('checks[]')
#         house = request.POST.get('option[]').split("-")
#         print("aaaaaaaaaaaa-------------", house)
#         house_name = house[0]
#         house_price = int(house[1])
#         for service in services:
#             service = service.split("-")
#             service_name = service[0]
#             service_price = service[1]
#             new_order = Orders.objects.create(roomname=house_name, roomprice=house_price, servicename=service_name, serviceprice=service_price)
#             new_order.save()
#         return redirect('myprint:services')
#     context = {
#         'cat': cat,
#         'card': card,
#         'pr': pr
#     }
#     return render(request, 'main/services.html', context=context)

# def services(request):
#     card = CardServices.objects.all()
#     cat = Room.objects.all()
#     pr = Service.objects.all()
#     if request.method == "POST":
#         services = request.POST.getlist('checks[]')
#         house = request.POST.get('option[]').split("-")
#         print(">>>>>>>>>>>>>>>>>>>>>>>>", house)
#         print("aaaaaaaaaaaa-------------", services)
#         house_name = house[0]
#         house_price = int(house[1])
#         total = 0
#         for ser in services:
#             print(">>>>>>>>>>>>>>>>>>>>>", ser)
#             ser = ser.split("-")
#             service_name = ser[0]
#             service_price = int(ser[1])
#             total += service_price
#             order = Order.objects.create(roomname=house_name, roomprice=house_price, total=total)
#             order.save()
#             order.service.add(ser)
#         return redirect('myprint:services')
#     context = {
#         'cat': cat,
#         'card': card,
#         'pr': pr
#     }
#     return render(request, 'main/services.html', context=context)

# def services(request):
#     card = CardServices.objects.all()
#     cat = Room.objects.all()
#     pr = Service.objects.all()
#     if request.method == "POST":
#         services = request.POST.getlist('checks[]')
#         house = request.POST.get('option[]').split("-")
#         print(">>>>>>>>>>>>>>>>>>>>>>>>", house)
#         print("aaaaaaaaaaaa-------------", services)
#         house_name = house[0]
#         house_price = int(house[1])
#         service = Service.objects.create(ser)
#         # service_name = services[0]
#         # service_price = int(services[1])
#         order = Order.objects.create(roomname=house_name, roomprice=house_price)
#         order.service.add()
#         order.save()
#         return redirect('myprint:services')
#     context = {
#         'cat': cat,
#         'card': card,
#         'pr': pr
#     }
#     return render(request, 'main/services.html', context=context)

def info_order(request):
    info = Orders.objects.all()
    total = 0

    context={'info': info}
    return render(request, 'service-info.html', context)

# def services(request):
#     card = CardServices.objects.all()
#     cat = Room.objects.all()
#     servic = Service.objects.all()
#     if request.method == "POST":
#         services = request.POST.getlist('checks[]')
#         house = request.POST.get('option[]').split("-")
#         total = 0
#         house_name = house[0]
#         house_price = int(house[1])
#         service_name = services[0]
#         service_price = int(services[1])
#         tt = house_price + service_price
#         tts = Total.objects.create(total = tt)
#         tts.save()

#         order = Orders.objects.create(roomname=house_name, roomprice=house_price, servicename=service_name, serviceprice=service_price, total_price=total)
#         order.save()
#         return redirect('myprint:home')
#     context = {
#         'cat': cat,
#         'card': card,
#         'servic': servic
#     }
#     return render(request, 'main/services.html', context=context)

def services(request):
    card = CardServices.objects.all()
    cat = Room.objects.all()
    servic = Service.objects.all()
    # form = AllForm()
    if request.method == "POST":
    #     services = request.POST.getlist('checks[]')
    #     house = request.POST.get('option[]').split("-")
    #     # name = AllForm('name')
    #     phone_number = AllForm('phone_number')
    #     house_name = house[0]
    #     house_price = int(house[1])
    #     house = Orders.objects.create(roomname=house_name, roomprice=house_price, servicename=services)
    #     form.save()

        return redirect('myprint:contact')
    context = {
        'cat': cat,
        'servic': servic,
        'card': card,
    }
    return render(request, 'main/services.html', context=context)


import random
import logging
import numpy as np

import torch
from spacy.lang.zh.stop_words import STOP_WORDS
from sklearn.metrics import f1_score, accuracy_score
from transformers import (
    AdamW,
    get_linear_schedule_with_warmup,
    get_cosine_schedule_with_warmup
)

from category_id_map import lv2id_to_lv1id

def setup_device(args):
    args.device = 'cuda' if torch.cuda.is_available() else 'cpu'
    args.n_gpu = torch.cuda.device_count()

def setup_seed(args):
    random.seed(args.seed)
    np.random.seed(args.seed)
    torch.manual_seed(args.seed)

def setup_logging():
    logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s',
                        datefmt='%Y-%m-%d %H:%M:%S',
                        level=logging.INFO)
    logger = logging.getLogger(__name__)

    return logger

def build_optimizer(args, model):
    # Prepare optimizer and schedule (linear warmup and decay)
    no_decay = ['bias', 'LayerNorm.weight']
    optimizer_grouped_parameters = [
        {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],
         'weight_decay': args.weight_decay},
        {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
    ]
    optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)
    if args.lr_scheduler == 'cosine':
        scheduler = get_cosine_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps,
                                                    num_training_steps=args.max_steps)
    else:
        scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps,
                                                    num_training_steps=args.max_steps)

    return optimizer, scheduler

def evaluate(predictions, labels):
    # prediction and labels are all level-2 class ids

    lv1_preds = [lv2id_to_lv1id(lv2id) for lv2id in predictions]
    lv1_labels = [lv2id_to_lv1id(lv2id) for lv2id in labels]

    lv2_f1_micro = f1_score(labels, predictions, average='micro')
    lv2_f1_macro = f1_score(labels, predictions, average='macro')
    lv1_f1_micro = f1_score(lv1_labels, lv1_preds, average='micro')
    lv1_f1_macro = f1_score(lv1_labels, lv1_preds, average='macro')
    mean_f1 = (lv2_f1_macro + lv1_f1_macro + lv1_f1_micro + lv2_f1_micro) / 4.0

    eval_results = {'lv1_acc': accuracy_score(lv1_labels, lv1_preds),
                    'lv2_acc': accuracy_score(labels, predictions),
                    'lv1_f1_micro': lv1_f1_micro,
                    'lv1_f1_macro': lv1_f1_macro,
                    'lv2_f1_micro': lv2_f1_micro,
                    'lv2_f1_macro': lv2_f1_macro,
                    'mean_f1': mean_f1}

    return eval_results

def log_uniform(low, high):
    log_rval = np.random.uniform(np.log(low), np.log(high))
    rval = float(np.exp(log_rval))
    return rval

def remove_stopwords(string, nlp):
    doc = nlp(string)
    new_string = []
    for token in doc:
        if token.text not in STOP_WORDS:
            new_string.append(token.text)
    return ''.join(new_string)


# -*- coding: utf-8 -*-
"""
Created on Mon Jun 26 20:35:19 2023

@author: uendel
"""

import os
from console import fg, bg, fx
from random import randint

#%% CONSTANTES PARA SAÍDA DE TEXTO DO TERMINAL
TERMINAL_SIZE = os.get_terminal_size

ERRO = fg.yellow + fx.bold + bg.red
AVISO = fg.red + fx.bold + bg.yellow
OK = fg.green + fx.bold
INFO = fg.cyan + fx.bold

CLEAR_LINE_MSG = f"\r{' ' * TERMINAL_SIZE()[0]}"

#%% Configurações do terminal
def max_cols():
  return TERMINAL_SIZE()[0] * 2

def max_lines():
  return TERMINAL_SIZE()[1]

# clear line
def cll(size = max_cols()):
  print(f"\r{' ' * size}", end="")

#%% Funções que retornam mensagens formatadas no terminal
def print_erro(msg, end = ''):
  cll()
  print("{:>210}".format(f"\r\U000026D4 {ERRO('ERRO')} {msg}"),
        end='\U000026D4 \r\n' if end == '' else f'\U000026D4 {end}')

def print_aviso(msg, end = ''):
  cll()
  print("{:80}".format(f"\r\U000026A0 {AVISO('AVISO')} {msg}"),
        end='\U000026A0 \r' if end == '' else f'\U000026A0 {end}')

def print_ok(msg="OK", end=""):
  cll()
  print("{:>210}".format(f"{OK(msg)}"), end=end)

def print_info(msg="INFO", end=""):
  cll()
  print("{:<120}".format(f"{INFO(msg)}"), end=end)

#%% GAUGES AND BARS
def gauge(i, max = 100, steps = 20):
  percent = round((i / max) * 100, 4)
  step = int(steps * percent // 100)
  return f"{i}/{max} {percent:0.2f}% {'■' * step:⬚<{steps}}"

def slide_bar(i, steps = 10, fill_char = '■', step_char = '⬚', slide = True):
  bar = list(step_char * steps)
  if slide:
    bar[i % steps] = fill_char
  else:
    bar[0:(i % steps) + 1] = "■" * ((i % steps) + 1)
    #bar[0:(i + 1) % steps] = fill_char * (i % steps)

  bar = "".join(bar)
  return f"{bar:{steps}}" #f"{bar:<{steps}}".format(bar)

# for i in range(11):
#     caixa = list("⬚" * 10)
#     caixa[0:i % 10 + 1] = "■" * (i%10+1)
#     caixa = "".join(caixa)
#     print(f'{caixa}', end=f"{i}")
#     time.sleep(0.5)

propellers = {
  "slash":['|','/','—','\\'],
  "dotbar":['░', '▒', '▓', '█'], # ASSCII: 176, 177, 178, 219
  "vslice": ['_','▄','■', '▀', '¯'], # ASCII: _, 220, 254, 223, 254, 238
  "vbar": ['▁','▂','▃','▄','▅','▆','▇','█','■','▀','¯'], # i % 10
  'hbar': ['▏','▎','▍','▋','▊','▉'], # i % 5
  'sonar': ['◯','⬤','●','•','○','◌'], # i % 3
  "dash": ['_','‗','═','¯'], # i % 4
  "squares": ['◳','◲','◱','◰'],
  "pipes": ['╗','╝','╚','╔'],
  "pizza": ['◷','◶','◵','◴'],
  "circles": ['◦','○','◌','◯','◌','○'],
  "slide_bar": slide_bar,
              }

def propeller():
  return propellers[list(propellers.keys())[randint(0, len(propellers)-1)]]


import findspark
findspark.init()

from pyspark import SparkContext, SparkConf
import time
import sys
import os
import random
import math
import json
from itertools import combinations
from copy import deepcopy

startTime = time.time()

conf = SparkConf().setMaster("local[*]")         .set("spark.executor.memory", "4g")         .set("spark.driver.memory", "4g")
sc = SparkContext(conf=conf).getOrCreate()

def summarize(points_cluster_list):
    summary = list()
    for cluster in points_cluster_list:
        cluster_dict = dict() # {N:n, SUM:sum, SUMSQ:sumsq}
        cluster_dict['N'] = len(cluster)
        SUM, SUMSQ = getSUMandSUMSQ(cluster)
        cluster_dict['SUM'] = SUM
        cluster_dict['SUMSQ'] = SUMSQ
        summary.append(cluster_dict)
    return summary

def getSUMandSUMSQ(cluster):
    cluster_points = [pair[0] for pair in cluster]
    for idx, point in enumerate(cluster_points):
        sq_point = tuple([digit**2 for digit in point])
        if idx==0:
            SUM_result = point
            SUMSQ_result = sq_point
        else:
            SUM_result = [sum(x) for x in zip(SUM_result, point)]
            SUMSQ_result = [sum(x) for x in zip(SUMSQ_result, sq_point)]
    return tuple(SUM_result), tuple(SUMSQ_result)

def groupPointsByCluster(k, cluster_result):
    points_cluster_list = list()
    for kth in range(k):
        l = list(filter(lambda pair:pair[1]==kth, cluster_result))
        if len(l)!=0:
            points_cluster_list.append(l)
    points_cluster_list = sorted(points_cluster_list, key=lambda cluster:len(cluster))
    return points_cluster_list

def kmeans(k, points_list, max_iter, initialization, n_init=1):
    best_centroids = dict()
    best_inertia = float('inf')
    for n in range(n_init):
        centroids_dict = dict() # map centroid tuple to unique and index
        if initialization=='random':
            dictinct_points = list(set(points_list))
            selected_cent = set()
            for cluster_idx in range(k):
                random_idx = random.randrange(0, len(dictinct_points))
                while random_idx in selected_cent:
                    random_idx = random.randrange(0, len(dictinct_points))
                selected_cent.add(random_idx)
                centroids_dict[dictinct_points[random_idx]] = cluster_idx
            # centroids is the list of initial centorids
        elif initialization=='farthest':
            #initialize the initial centroids by farthest distance from each other
            curr_sum_distance = 0
            closest_distance = 0
            for kth_centroid in range(k): #first centroid is still randomly picked
                if kth_centroid == 0:
                    random_idx = random.randrange(0, len(points_list))
                    centroids_dict[points_list[random_idx]] = kth_centroid
                else:
                    curr_sum_distance = 0
                    # compute sum of distance to the current centroids to each other point
                    closest_centroid = points_list[0] # in case there is only one point in points_list
                    for point in points_list:
                        if point not in centroids_dict.keys():
                            for curr_cent in centroids_dict.keys():
                                curr_sum_distance += euclideanDist(curr_cent, point)
                            if curr_sum_distance > closest_distance:
                                closest_distance = curr_sum_distance
                                closest_centroid = point
                            curr_sum_distance = 0
                    centroids_dict[closest_centroid] = kth_centroid
        else:
            return 'initialization method not recognized'

        convergence = False
        curr_iter = 0
        while not convergence and curr_iter<max_iter:
            # assign points to clusters
            cluster_list = list() # [(point, centroid_idx),(point, centroid_idx), ...]
            for point in points_list:
                #this point should be assigned to return value centroid
                centroid = findCentroid(point, centroids_dict.keys())
                centroid_idx = centroids_dict[centroid]
                cluster_list.append((point, centroid_idx))

            # update centroids
            centroids_comp = list()
            centroids_dict_items = list(centroids_dict.items())

            for old_centroid, idx in centroids_dict_items:
                idx_cluster = list(filter(lambda pair: pair[1]==idx, cluster_list))
                # a cluster might be empty due to bad initialization
                if len(idx_cluster) == 0:
                    # simply assign a random point to that cluster
                    points = [points_list[random.randrange(0, len(points_list))]]
                else:
                    # list of points in this cluster
                    points = [pair[0] for pair in idx_cluster]

                # calculate new centroid
                new_centroid = calculateNewCentroid(points, len(old_centroid))
                # update centroids_dict
                if new_centroid != old_centroid:
                    # new centroid has same cluster index
                    # add new centroid
                    centroids_dict[new_centroid] = idx
                    # delete old centroid
                    del centroids_dict[old_centroid]

                centroids_comp.append((old_centroid, new_centroid))

            # test convergence
            curr_iter += 1
            convergence = all([euclideanDist(pair[0], pair[1])==0 for pair in centroids_comp])
        # after convergence, calculate interia
        curr_inertia = 0
        for point in points_list:
            #this point should be assigned to return value centroid
            centroid = findCentroid(point, centroids_dict.keys())
            curr_inertia += euclideanDist(point, centroid) ** 2
        if curr_inertia < best_inertia:
            best_inertia = curr_inertia
            best_centroids = deepcopy(centroids_dict)

    cluster_list = list() # [(point, centroid_idx),(point, centroid_idx), ...]
    for point in points_list:
        #this point should be assigned to return value centroid
        centroid = findCentroid(point, best_centroids.keys())
        centroid_idx = best_centroids[centroid]
        cluster_list.append((point, centroid_idx))
    return cluster_list, list(best_centroids.keys())

def calculateNewCentroid(clusterPoints, dimensions):
    coord_list = list()
    for d in range(dimensions):
        coord_list.append([pt[d] for pt in clusterPoints]) # get d-th coordinates of all points
    return tuple([sum(dim)/len(dim) for dim in coord_list])

def euclideanDist(pt, cent):
    sumsq = 0
    for d in range(len(pt)):
        sumsq += (pt[d] - cent[d]) ** 2
    return math.sqrt(sumsq)

def findCentroid(pt, centroids):
    reuslt_list = list()
    for centroid in centroids:
        ED = euclideanDist(pt, centroid) #Euclidean distance
        reuslt_list.append((ED, centroid))
    return sorted(reuslt_list, key=lambda pair:pair[0])[0][1]# return the centroid with min ED to pt

def updateSummary(summary, idx, point):
    summary[idx]['N'] += 1
    summary[idx]['SUM'] = tuple([sum(x) for x in zip(summary[idx]['SUM'], point)])
    sq_point = tuple([digit**2 for digit in point])
    summary[idx]['SUMSQ'] = tuple([sum(x) for x in zip(summary[idx]['SUMSQ'], sq_point)])

def mahalanobisDist(point, summary):
    md = 0
    N = summary['N']
    for d in range(len(point)):
        d_SUM = summary['SUM'][d]
        d_SUMSQ = summary['SUMSQ'][d]
        d_centroid = d_SUM / N
        sd = math.sqrt((d_SUMSQ / N) - (d_SUM / N) ** 2)
        if sd==0:
            normalization = 0
        else:
            normalization = ((point[d] - d_centroid) / sd) ** 2
        md += normalization
    return math.sqrt(md)

def mergeCS(CS_summary):
    merged_CS_idx_set = set()
    new_CS_list = list()

    MD_pair_list = list()
    # generate combinations of CS based on their indices
    CS_idx_comb = list(combinations(range(len(CS_summary)), 2))
    for CS_idx_tuple in CS_idx_comb:
        summary1 = CS_summary[CS_idx_tuple[0]]
        summary2 = CS_summary[CS_idx_tuple[1]]
        MD, thres = getNearness(summary1, summary2)
        if MD < thres:
            MD_pair_list.append((MD, CS_idx_tuple))
    MD_pair_list = sorted(MD_pair_list, key=lambda pair:pair[0])

    for MD_pair in MD_pair_list:
        MD = MD_pair[0]
        CS_idx_tuple = MD_pair[1]
        # if none of these two CS have been merged before
        if CS_idx_tuple[0] not in merged_CS_idx_set and CS_idx_tuple[1] not in merged_CS_idx_set:
            # merge these two CS into one
            summary1 = CS_summary[CS_idx_tuple[0]]
            summary2 = CS_summary[CS_idx_tuple[1]]
            N = summary1['N'] + summary2['N']
            SUM = tuple([sum(zipped) for zipped in zip(summary1['SUM'], summary2['SUM'])])
            SUMSQ = tuple([sum(zipped) for zipped in zip(summary1['SUMSQ'], summary2['SUMSQ'])])
            new_CS = {'N':N, 'SUM':SUM, 'SUMSQ':SUMSQ}
            new_CS_list.append(new_CS) # append merged CS
            merged_CS_idx_set.add(CS_idx_tuple[0])
            merged_CS_idx_set.add(CS_idx_tuple[1])

    # get the list of CS indices that are not merged
    unmerged_CS_idx_list = list(set(range(len(CS_summary))) - merged_CS_idx_set)

    # append the not merged CS back to CS list
    for unmerged_CS_idx in unmerged_CS_idx_list:
        new_CS_list.append(CS_summary[unmerged_CS_idx])

    return new_CS_list

def getNearness(CS1, CS2):
    # nearness is defined as the average MD between a centroid of a cluster to another cluster and vice versa
    cent1 = tuple([d/CS1['N'] for d in CS1['SUM']])
    cent2 = tuple([d/CS2['N'] for d in CS2['SUM']])
    CS1cent2CS2_MD = mahalanobisDist(cent1, CS2)
    CS2cent2CS1_MD = mahalanobisDist(cent2, CS1)
    threshold = alpha_threshold * math.sqrt(len(cent1))
    return (CS1cent2CS2_MD + CS2cent2CS1_MD)/2, threshold

def generateIntermediateResult(round_num, DS_summary, CS_summary, RS_list):
    nof_point_discard = str(sum([DS['N'] for DS in DS_summary]))
    nof_point_compression = str(sum([CS['N'] for CS in CS_summary]))
    line = str(round_num) + ',' + str(len(DS_summary)) + ','            + nof_point_discard + ',' + str(len(CS_summary)) + ','            + nof_point_compression + ',' + str(len(RS_list))
    return line

input_folder_path = r'C:\Users\11921\Downloads\data\test3'
k = 10
output_path_cluster = r'points_assignment.json'
output_path_intermediate_result = r'intermediate_result.csv'

csv_files_list = os.listdir(input_folder_path)
large_num_centroids = 3 * k
max_iterations = 10
very_few = 500
alpha_threshold = 2
sample_ratio = 0.7
kmeans_initial_cent = 'random' #random
generateCSRS = 'complement' #'outliers', method of generating CS and RS, either from complement points or from outliers
cache_datapoints = True
merge_CS = False
all_datapoints = list()

intermediate_result_list = list()
for round_idx, csv in enumerate(sorted(csv_files_list)):
    '''
    Step 1, 7. load data
    '''
    if cache_datapoints:
        full_csv_path = input_folder_path + "\\" + csv
        raw_points = (sc.textFile(full_csv_path)  # read a data chunk
                        .map(lambda line:line.split(',')) # split the raw data string by comma
                        .map(lambda pt:[float(coord) for coord in pt]) # cast the str coordinates to float
                        .map(lambda pt:(str(int(pt[0])), tuple(pt[1:])))  # form tuple of idx and coordinates
                        .collect()
                       )
        points_list = [pair[1] for pair in raw_points]
        all_datapoints.extend(raw_points)

    else:
        full_csv_path = input_folder_path + "\\" + csv
        points_list = (sc.textFile(full_csv_path)  # read a data chunk
                        .map(lambda line:line.split(',')) # split the raw data string by comma
                        .map(lambda pt:[float(coord) for coord in pt]) # cast the str coordinates to float
                        .map(lambda pt: tuple(pt[1:])) # keep only the point coordinates
                        .collect()
                       )

    if round_idx == 0: # first round, run k-means on a subset to initialize
        # sample the points list to make it smaller
        subset_points_list = random.sample(points_list, int(len(points_list) * sample_ratio))
        comp_points_list = list(set(points_list) - set(subset_points_list))# remaining points of the first load of data

        if generateCSRS == 'outliers':
            '''
            Step 2. run k-means on a subset of data points (to generate outliers and inliers)
            '''
            # cluster_result: [((point1), cluster_index), ((point2), cluster_index), ...]
            # centroids: [(centroid1), (centroid2), ...]
            cluster_result, _ = kmeans(large_num_centroids, subset_points_list, max_iterations, kmeans_initial_cent)

            # put together the points that have been clustered into a same cluster
            points_cluster_list = groupPointsByCluster(large_num_centroids, cluster_result)
            del cluster_result

            '''
            Step 3. move clusters with few points to outliers, otherwise to inliers
            '''
            outliers_list = list()
            inliers_list = list()
            for cluster in points_cluster_list:
                if len(cluster) <= very_few:
                    outliers_list.append(cluster)
                else:
                    inliers_list.append(cluster)
            del points_cluster_list
            outlier_points_list = [pair[0] for cluster in outliers_list for pair in cluster]

            '''
            Step 4. run kmeans again on inliers to cluster inlier data to k clusters. Use k clusters as DS and summarize
            '''
            inlier_points_list = [pair[0] for cluster in inliers_list for pair in cluster]
            inliers_points_result, _ = kmeans(k, inlier_points_list, max_iterations, kmeans_initial_cent)

            # put together the points that have been clustered into a same cluster
            inliers_cluster_result = groupPointsByCluster(k, inliers_points_result)
            DS_summary = summarize(inliers_cluster_result)
            # discard the points
            del inliers_list # original inliers points with index
            del inlier_points_list # original inliers points
            del inliers_points_result # cluster result of inliers points
            del inliers_cluster_result # cluster result of inliers points grouped by cluster index

            '''
            Step 5. run kmeans on outliers, create CS and RS from the result, keep only the summary of CS
            '''
            outliers_points_result, _ = kmeans(large_num_centroids, outlier_points_list, max_iterations, kmeans_initial_cent)
            outliers_cluster_result = groupPointsByCluster(large_num_centroids, outliers_points_result)
            CS_list, RS_list = list(), list()
            for outliers_cluster in outliers_cluster_result:
                if len(outliers_cluster) > 1:
                    CS_list.append(outliers_cluster)
                else:
                    RS_list.append(outliers_cluster)
            RS_list = [point[0] for cluster in RS_list for point in cluster]
            CS_summary = summarize(CS_list)
            del CS_list, outliers_cluster_result, outlier_points_list

            '''
            Step 6,7,8,9,10. Load the remaining data of the first round, assign each of them to DS, CS or RS
            '''
            for point in comp_points_list:
                min_DS_MD = float('inf')
                min_CS_MD = float('inf')
                found_qualified_DS_point = False
                for curr_DS_idx, DS_cluster in enumerate(DS_summary):
                    if DS_cluster['N'] != 0:
                        curr_DS_MD = mahalanobisDist(point, DS_cluster)
                        if curr_DS_MD < alpha_threshold * math.sqrt(len(point)):
                            if curr_DS_MD < min_DS_MD:
                                min_DS_MD = curr_DS_MD
                                DS_idx = curr_DS_idx
                                found_qualified_DS_point = True
                if found_qualified_DS_point:
                    updateSummary(DS_summary, DS_idx, point)
                else:
                    found_qualified_CS_point = False
                    for curr_CS_idx, CS_cluster in enumerate(CS_summary):
                        curr_CS_MD = mahalanobisDist(point, CS_cluster)
                        if curr_CS_MD < alpha_threshold * math.sqrt(len(point)):
                            if curr_CS_MD < min_CS_MD:
                                min_CS_MD = curr_CS_MD
                                CS_idx = curr_CS_idx
                                found_qualified_CS_point = True
                    if found_qualified_CS_point:
                        updateSummary(CS_summary, CS_idx, point)
                    else:
                        RS_list.append(point)
        elif generateCSRS == 'complement':
            #generate DS
            subset_points_result, _ = kmeans(k, subset_points_list, max_iterations, kmeans_initial_cent)

            # put together the points that have been clustered into a same cluster
            subset_points_cluster_result = groupPointsByCluster(k, subset_points_result)
            DS_summary = summarize(subset_points_cluster_result)
            # discard the points
            del subset_points_result
            del subset_points_cluster_result

            #generate CS/RS
            comp_points_result, _ = kmeans(large_num_centroids, comp_points_list, max_iterations, kmeans_initial_cent)
            comp_points_cluster_result = groupPointsByCluster(large_num_centroids, comp_points_result)

            CS_list = list()
            RS_list = list()
            CS_summary = list()
            for cluster in comp_points_cluster_result:
                if len(cluster) > 1:
                    CS_list.append(cluster)
                elif len(cluster) == 1:
                    RS_list.append(cluster)
            CS_summary = summarize(CS_list)
            RS_list = [point[0] for cluster in RS_list for point in cluster]
            del comp_points_result, comp_points_cluster_result, CS_list

    else: # not first round, Repeat step 6 to 12.
        '''
        Step 8,9,10. Assign each of the data points to DS, CS or RS
        '''
        for point in points_list: # go through each point
            min_DS_MD = float('inf')
            min_CS_MD = float('inf')
            found_qualified_DS_point = False
            for curr_DS_idx, DS_cluster in enumerate(DS_summary): # go through each DS cluster
                curr_DS_MD = mahalanobisDist(point, DS_cluster)
                if curr_DS_MD < alpha_threshold * math.sqrt(len(point)):
                    if curr_DS_MD < min_DS_MD:
                        min_DS_MD = curr_DS_MD
                        DS_idx = curr_DS_idx
                        found_qualified_DS_point = True
            if found_qualified_DS_point:
                updateSummary(DS_summary, DS_idx, point)
            else:
                if len(CS_summary) > 0:
                    found_qualified_CS_point = False
                    for curr_CS_idx, CS_cluster in enumerate(CS_summary): # go through each CS cluster
                        curr_CS_MD = mahalanobisDist(point, CS_cluster)
                        if curr_CS_MD < alpha_threshold * math.sqrt(len(point)):
                            if curr_CS_MD < min_CS_MD:
                                min_CS_MD = curr_CS_MD
                                CS_idx = curr_CS_idx
                                found_qualified_CS_point = True
                    if found_qualified_CS_point:
                        updateSummary(CS_summary, CS_idx, point)
                    else:
                        RS_list.append(point)
                else:
                    RS_list.append(point)

    # if CS_summary is empty, run this
    if len(CS_summary) == 0:
        '''
        Step 11. Run kmeans on current RS list, split clusters into CS and RS.
        generate new CS summary and add it to CS summary, keep remaining RS list
        '''
        RS_points_result, _ = kmeans(large_num_centroids, RS_list, max_iterations, kmeans_initial_cent)
        RS_cluster_result = groupPointsByCluster(large_num_centroids, RS_points_result)

        RS_list = list() # delete the old RS_list
        for RS_cluster in RS_cluster_result:
            if len(RS_cluster) > 1:
                new_CS_summary = summarize([RS_cluster])[0] # only one CS cluster in the list
                CS_summary.append(new_CS_summary)
            elif len(RS_cluster) == 1: # RS_cluster might be empty when RS has less points than centroids
                RS_list.append(RS_cluster[0][0]) # keep only the points coordinates

        del RS_points_result # remove points
        del RS_cluster_result

    '''
    Step 12. Merge CS clusters that are close enough. The nearness is calculated based on the average of
    Mahalanobis Distance of one cluster centroid to another CS cluster
    '''
    if merge_CS:
        CS_summary = mergeCS(CS_summary)

    '''
    Last round, merge CS clusters into closest DS cluster
    '''
    if round_idx == len(sorted(csv_files_list)) - 1: #
        if len(CS_summary) > 0:
            for CS in CS_summary:
                min_ED_CSDS = float('inf')
                target_DS_idx = 0
                CS_cent = tuple([d/CS['N'] for d in CS['SUM']])
                for DS_idx, DS in enumerate(DS_summary): # go through each DS, found the cloest one
                    DS_cent = tuple([d/DS['N'] for d in DS['SUM']])
                    curr_ED_CSDS = euclideanDist(CS_cent, DS_cent)
                    if curr_ED_CSDS < min_ED_CSDS:
                        min_ED_CSDS = curr_ED_CSDS
                        target_DS_idx = DS_idx
                # merge current CS to DS with min_ED_CSDS
                DS_summary[target_DS_idx]['N'] = DS_summary[target_DS_idx]['N'] + CS['N']
                new_SUM = tuple([sum(zipped) for zipped in zip(DS_summary[target_DS_idx]['SUM'], CS['SUM'])])
                new_SUMSQ = tuple([sum(zipped) for zipped in zip(DS_summary[target_DS_idx]['SUMSQ'], CS['SUMSQ'])])
                DS_summary[target_DS_idx]['SUM'] = new_SUM
                DS_summary[target_DS_idx]['SUMSQ'] = new_SUMSQ
            CS_summary = []

        '''
        for RS_point in RS_list:
            min_ED_RSDS = float('inf')
            target_DS_idx = 0
            for DS_idx, DS in enumerate(DS_summary): # go through each DS, found the cloest one
                DS_cent = tuple([d/DS['N'] for d in DS['SUM']])
                curr_ED_RSDS = euclideanDist(RS_point, DS_cent)
                if curr_ED_RSDS < min_ED_RSDS:
                    min_ED_RSDS = curr_ED_RSDS
                    target_DS_idx = DS_idx
            # merge current CS to DS with min_ED_CSDS
            updateSummary(DS_summary, target_DS_idx, RS_point)
        RS_list = []
        '''

    '''
    Step 13. Output the intermediate result
    By now, we have DS_summary, CS_summary and RS_list
    '''
    intermediate_result = generateIntermediateResult(round_idx, DS_summary, CS_summary, RS_list)
    intermediate_result_list.append(intermediate_result)
del CS_summary # all points info are now in the DS_summary

'''
Assign each point to corresponding DS cluster
'''
result_dict = dict()
if cache_datapoints:
    for idx_point_pair in all_datapoints:
        min_ED = float('inf')
        pt_idx = str(idx_point_pair[0])
        point = idx_point_pair[1]
        DS_idx = 0
        if point in RS_list:
            result_dict[pt_idx] = -1
        else:
            for curr_DS_idx, DS in enumerate(DS_summary):
                DS_cent = tuple([d/DS['N'] for d in DS['SUM']])
                curr_ED = euclideanDist(point, DS_cent)
                if curr_ED < min_ED:
                    min_ED = curr_ED
                    DS_idx = curr_DS_idx
            result_dict[pt_idx] = DS_idx
    del RS_list
else:
    for csv in sorted(csv_files_list):
        full_csv_path = input_folder_path + "\\" + csv
        points_list = (sc.textFile(full_csv_path)  # read a data chunk
                        .map(lambda line:line.split(',')) # split the raw data string by comma
                        .map(lambda pt:[float(coord) for coord in pt]) # cast the str coordinates to float
                        .map(lambda pt:(str(int(pt[0])), tuple(pt[1:]))) # form tuple of idx and coordinates
                        .collect()
                       )
        for idx_point_pair in points_list:
            min_ED = float('inf')
            pt_idx = str(idx_point_pair[0])
            point = idx_point_pair[1]
            DS_idx = 0
            for curr_DS_idx, DS in enumerate(DS_summary):
                DS_cent = tuple([d/DS['N'] for d in DS['SUM']])
                curr_ED = euclideanDist(point, DS_cent)
                if curr_ED < min_ED:
                    min_ED = curr_ED
                    DS_idx = curr_DS_idx
            result_dict[pt_idx] = DS_idx

output_cluster = open(output_path_cluster, 'w')
output_cluster.write(json.dumps(result_dict))
output_cluster.close()

output_inter = open(output_path_intermediate_result, 'w')
header = 'round_id,nof_cluster_discard,nof_point_discard,nof_cluster_compression,nof_point_compression,nof_point_retained\n'
output_inter.write(header)
for intermediate_result in intermediate_result_list:
    output_inter.write(intermediate_result + '\n')
output_inter.close()

print('Duration:', str(time.time()-startTime))


import os

import joblib

from etl.data_loader import DataLoader
from etl.feature_engineer import TitanicFeatureEngineer
from model.dummy_classifier import DummyModel
from model.gradient_boosting import GradientBoostingModel
from model.kfold_cross_validator import KFoldValidator
from model.random_forest import RandomForestModel

class Train:
    def __init__(self, algorithm):
        self.algorithm = algorithm

    def train_model(self, train_file_path, test_file_path):
        if self.algorithm == "Dummy":
            model = DummyModel().get_model()
        elif self.algorithm == "RandomForest":
            model = RandomForestModel().get_model()
        elif self.algorithm == "GradientBoosting":
            model = GradientBoostingModel().get_model()
        else:
            raise ValueError(f"Invalid algorithm name: {self.algorithm}")

        titanic_dl = DataLoader()
        titanic_dl.load_data(train_file_path, test_file_path)
        titanic_fe = TitanicFeatureEngineer(
            titanic_dl.get_train_df(), titanic_dl.get_test_df()
        )
        titanic_fe.feature_engineering()
        titanic_fe.feature_selection("RFE")
        X, y = titanic_fe.get_features_and_target()

        validator = KFoldValidator(model)
        (
            mean_roc_auc,
            std_roc_auc,
            mean_accuracy,
            std_accuracy,
        ) = validator.calculate_metrics(X, y)

        model.fit(X, y)

        joblib.dump(
            model,
            f"{os.getcwd()}/model/models/{str(type(model).__name__).lower()}.joblib",
        )

        print("------------------------")
        print("Training stage completed")
        print("------------------------")
        print("Mean ROC-AUC:", mean_roc_auc)
        print("Standard Deviation of ROC-AUC:", std_roc_auc)
        print("Mean Accuracy:", mean_accuracy)
        print("Standard Deviation of Accuracy:", std_accuracy)


import cv2
import numpy as np

image_index = 0

def getImage():
    img = cv2.imread('images/image2.jfif')
    return img

def showImage(img):
    global image_index
    cv2.imshow(f'Image#{image_index}', img)
    image_index += 1

def getMask(img):
    mask = np.zeros(
        shape=img.shape[:2],
        dtype='uint8'
    )
    return mask

def getCircleMask(img, mask):
    circle_mask = cv2.circle(
        img=mask,
        center=(img.shape[1]//2, img.shape[0]//2),
        radius=70,
        color=255,
        thickness=-1
    )
    return circle_mask

def getMaskedImage(img, mask):
    masked_image = cv2.bitwise_and(
        src1=img,
        src2=img,
        mask=mask
    )
    return masked_image

def main():
    img = getImage()
    showImage(img)

    mask = getMask(img)
    showImage(mask)

    circle_mask = getCircleMask(img, mask)
    showImage(circle_mask)

    masked_image = getMaskedImage(img, mask)
    showImage(masked_image)

    cv2.waitKey(0)

if __name__ == "__main__":
    main()


import collections
import re

# Intervalo de palabras que se incluirán en el diccionario
b = 0   # Inicio del intervalo
n = 40000 # Final del intervalo
sorted_vocab = []

vocab_file_path = "./corpora/quijote.txt"
tokenizer = re.compile("\W+")
with open(vocab_file_path, "r", encoding='utf-8') as fr:
    c= collections.Counter(tokenizer.split(fr.read().lower()))
    if '' in c:
        del c['']
    reversed_c = [(freq, word) for (word, freq) in c.items()]
    sorted_reversed = sorted(reversed_c, reverse=True)
    sorted_vocab = [word for (freq, word) in sorted_reversed]

with open("./corpora/quijote_"+str(b)+"_"+str(n)+".txt", "w", encoding='utf-8') as new_dict:
    new_dict.write(" ".join(sorted_vocab[b:n]))


import heapq
import tarfile
import os

class Beam(object):

	def __init__(self, beam_width, init_beam=None):
		if init_beam is None:
			self.heap = list()
		else:
			self.heap = init_beam

		heapq.heapify(self.heap)
		self.beam_width = beam_width

	def add(self, to_add):
		heapq.heappush(self.heap, to_add)

		if len(self.heap) > self.beam_width:
			heapq.heappop(self.heap)

	def __iter__(self):
		return iter(self.heap)

	def complete(self):
		return sum([int(c) for (_, c, _, _) in self]) == self.beam_width

	def pop(self):
		return heapq.heappop(self.heap)

	def is_empty(self):
		return len(self.heap) == 0

	def get_best(self, n=1, largest=True):
		if len(self.heap) >= n:
			if largest:
				return heapq.nlargest(n, self)
			else:
				return heapq.nsmallest(n, self)
		return None

	def remove(self, to_remove):
		self.heap.remove(to_remove)
		heapq.heapify(self.heap)

class DataFileHandler(object):

	def __init__(self, filename):
		self.filename = filename
		self.tmp = 'tmp/'
		self.counter = 0

	def zipped(self):
		return tarfile.is_tarfile(self.filename)

	def unzip_file(self):
		"""Utiility method to unzip a file
		"""
		tar = tarfile.open(name=self.filename)
		filename = tar.getnames()[0]

		if not os.path.isdir(self.tmp):
			os.mkdir(self.tmp)

		tar.extractall(path=self.tmp)
		return os.path.join(self.tmp, filename)

	def read(self):
		if self.zipped():
			print("File is zipped - unzipping to tmp directory")
			readfile = self.unzip_file()
		else:
			readfile = self.filename

		with open(readfile, 'r', encoding='utf8') as reader:
			for line in reader.readlines():
				(word, labels) = line.strip().split('\t')
				labels = labels.split(' - ')
				self.counter += 1
				yield(word, labels)

	def cleanup(self):
		if self.zipped():
			for root, dirs, files in os.walk(self.tmp, topdown=False):
				for name in files:
					os.remove(os.path.join(root, name))
				for name in dirs:
					os.rmdir(os.path.join(root, name))
			os.rmdir(self.tmp)


class SubclassDict(dict):
    '''
    A special dict that returns the associated Django database model
    if the queried item is a subclass of any of the keys
    '''
    def __getitem__(self, name):
        try:
            return super(self.__class__, self).__getitem__(name)
        except KeyError:
            for inst, model in list(self.items()):
                if issubclass(name, inst):
                    return model
        raise KeyError


import pandas as pd
import numpy as np
import os

folder = r'C:\Users\sergknut\Documents\Parse_Shafa\df_test'
folder = os.getcwd()
files = os.listdir(folder)
files.sort()
df = pd.DataFrame()

def create_new_df(file):
    dat, username = file[:8], file[9:-4]
    df_new = pd.read_csv(file, delimiter='|')
    df_new['File'] = file
    df_new['User'] = username
    df_new['Date'] = dat
    df_new['Current Price'] = df_new['Current Price'].astype('str')
    df_new['Current Price'] = df_new['Current Price'].str.extract(r'(\d+)').astype('float')
    df_new['Old Price'] = df_new['Old Price'].astype('str')
    df_new['Old Price'] = df_new['Old Price'].str.extract(r'(\d+)').astype('float')
    df_new['Date'] = pd.to_datetime(df_new['Date'], format='%d%m%Y')
    return df_new

def compare_dfs(df1, df2):
    comp = df1[['Link', 'User']].merge(df2[['Link', 'User']],indicator = True, how='left', left_on='Link', right_on='Link').loc[lambda x : x['_merge']=='left_only']
    rez_df = df1[df1['Link'].isin(comp['Link'])]
    return rez_df

for file in files:
    if file[-4:] != '.csv' or file[:4] == 'sales':
        continue
    df_new = create_new_df(file)
    print(f'Append {file}')
    df = pd.concat((df, df_new))

dfs = dict(tuple(df.groupby('Date')))
dates = list(dfs.keys())
sales = pd.DataFrame()
new_items =  pd.DataFrame()

for i, dat in enumerate(dates):
    if i == 0:
        continue
    df1 = dfs[dates[i-1]]
    df2 = dfs[dates[i]]

    filt = np.intersect1d(df1.User.unique(), df2.User.unique())

    sales = pd.concat((sales,compare_dfs(df1,df2)))
    sales = sales[sales.User.isin(filt)]
    new_items = pd.concat((new_items,compare_dfs(df2,df1)))
    new_items = new_items[new_items.User.isin(filt)]

sales['Flow'] = 'Sales'
new_items['Flow'] = 'New Items'
sales.Date += pd.Timedelta(days=1)
report = pd.concat((sales, new_items))

#sales.to_json('sales.json', orient='records')
sales.to_csv('sales.txt', sep='\t', index=False)
new_items.to_csv('new_items.txt', sep='\t', index=False)
report.to_csv('report.txt', sep='\t', index=False)


import cv2 as cv
from matplotlib import pyplot as plt
import numpy as np
from typing import List, Tuple
from model.digitrecognizer import DigitRecognizer
from board import Board
import logging

# TODO Think how to configure logger in one place and passed it to objects
# with proper name

def set_logger(logging_level):
    streamHandler = logging.StreamHandler()
    streamHandler.setLevel(logging_level)
    streamHandler.setFormatter(
        logging.Formatter('%(name)s[%(levelname)s]: %(message)s'))
    logger = logging.getLogger(__name__)
    logger.setLevel(logging_level)
    logger.addHandler(streamHandler)
    return logger

logger = set_logger(logging.DEBUG)

def prepare_model():
    recognizer = DigitRecognizer(logging.DEBUG)
    recognizer.load_weights()
    recognizer.evaluate_model()
    logger.info("Digit recognizer is ready!")
    return recognizer

def main(filename: str) -> None:
    recognizer = prepare_model()
    board = Board(logging.DEBUG)
    img_gray = cv.imread(filename, cv.IMREAD_GRAYSCALE)
    img_color = cv.imread(filename, cv.IMREAD_UNCHANGED)
    if img_gray is not None and img_color is not None:
        logger.info(f"Image {filename} successfuly read!")
        cropped_image = find_lines(img_gray, img_color)
        # plot_image(cropped_image, 'gray')
        boxes = resize_images_to_mnist(get_single_boxes(cropped_image))
        for row in range(9):
            row_numbers = []
            for col in range(9):
                # plot_image(boxes[row+col], 'gray')
                if doesBoxContainDigit(boxes[row+col]):
                    cv.imwrite(f"input_images/SingleBox_{row}-{col}.png",
                               boxes[row+col])
                    reformated_box = boxes[row+col].reshape(1, 28, 28).astype(
                        'float32') / 255.0
                    prediction = recognizer.predict(reformated_box)
                else:
                    prediction = 0
                row_numbers.append(prediction)
            board.add_row(row_numbers)
        board.pretty_print()
    else:
        logger.error(f"Couldn't open image {filename} !")

def doesBoxContainDigit(image) -> bool:
    threshold: float = 230.0
    mean = cv.mean(image)
    # print(f"Mean of image: {mean}")
    return mean[0] < threshold

def threshold_image(image):
    max_val: int = 255
    return cv.adaptiveThreshold(image, max_val, cv.ADAPTIVE_THRESH_GAUSSIAN_C,
                                cv.THRESH_BINARY, 11, 2)

def find_lines(image_grayscale, image_color):
    def restructure_array(lines) -> List:
        return [line[0].tolist() for line in lines]

    def find_boundaries(lines: List) -> List:
        result = []
        lines.sort(key=lambda points: points[0])
        result.append(lines[0])
        result.append(lines[-1])
        lines.sort(key=lambda points: points[3])
        result.append(lines[0])
        result.append(lines[-1])
        return result

    def draw_boundary(boundaries: List, image_color) -> None:
        color = (0, 255, 0)
        width = 2
        for boundary in boundaries:
            x1, y1, x2, y2 = boundary
            logger.debug(f"Drawn boundaries: ({x1}, {y1}) -> ({x2}, {y2})")
            cv.line(image_color, (x1, y1), (x2, y2), color, width)
        return image_color

    def crop_image(boundaries: List):
        x1 = boundaries[0][0]
        x2 = boundaries[1][0]
        y1 = boundaries[2][1]
        y2 = boundaries[3][1]
        logger.debug(f"Crop image to [{x1}:{x2}, {y1}:{y2}]")
        return image_grayscale[y1:y2, x1:x2]

    lowThreshold: int = 200
    highThreshold: int = 220
    rho: int = 1
    theta: int = np.pi/180
    threshold: int = 200
    minLineLength: int = 400
    maxLineGap: int = 8
    edges = cv.Canny(image_grayscale, lowThreshold, highThreshold,
                     apertureSize=3)
    lines = cv.HoughLinesP(edges, rho, theta, threshold, minLineLength,
                           maxLineGap)
    if lines is None:
        logger.error("Didn't found any lines!")
        return

    # draw_boundary(find_boundaries(restructure_array(
    #     lines)), image_color)
    # plot_image(cv.cvtColor(image_color, cv.COLOR_BGR2RGB), None)
    return crop_image(find_boundaries(restructure_array(lines)))

def get_single_boxes(image) -> List:
    """
    Slices image to 81 boxes in which should be one box from sudoku board

    Arguments:
        image {np.array} -- openCV's image

    Returns:
        List -- list of 81 single box images.
    """
    width, height = image.shape
    box_width = width // 9
    box_height = height // 9

    single_boxes = []
    for x in range(9):
        for y in range(9):
            start_x: int = x * box_width
            start_y: int = y * box_height
            single_boxes.append(image[start_x:(start_x + box_width),
                                      start_y:(start_y + box_height)])
    return single_boxes

def resize_images_to_mnist(image_list: List) -> List:
    """
    Resizes each image in list to 28x28px.

    Arguments:
        image_list {List} -- list of openCV's images

    Returns:
        List -- list of openCV's images in different size.
    """
    mnist_dim: Tuple[int, int] = (28, 28)
    result = []
    for image in image_list:
        result.append(cv.resize(image, mnist_dim, interpolation=cv.INTER_AREA))
    return result

def plot_image(image, color: str) -> None:
    """
    Shows image in color.

    Arguments:
        image {np.array} -- image read with openCV
        color {str} -- string that defines color palete, taken from cmap param.

    Returns:
        None
    """
    plt.imshow(image, cmap=color)
    plt.show()

if __name__ == "__main__":
    main("input_images/SimpleBoard_1.PNG")


#!/usr/bin/env python
# coding: utf-8

# In[50]:

import argparse
import time

import numpy as np
import pandas as pd
from sklearn.preprocessing import LabelEncoder, OneHotEncoder
from sklearn.model_selection import train_test_split
import warnings

import torch
from torch.autograd import Variable
from torch.utils.data import DataLoader
import torchvision
import torch.nn as nn
import torch.nn.functional as F

import torch.utils.data as data
import matplotlib.pyplot as plt
from matplotlib.pyplot import imshow
from matplotlib import ticker
from scipy.signal import savgol_filter

get_ipython().run_line_magic('matplotlib', 'inline')
get_ipython().run_line_magic('config', "InlineBackend.figure_format = 'svg'")

# In[25]:

### READ DATA ###
data = pd.read_csv('adult.csv')
print(data.info())

# In[26]:

print(data.shape)
print(data.columns)
print(data[:5])
income = pd.Index([">50K", "<=50K"])
data.income.value_counts()

# In[27]:

### DATA CLEANING AND CHECK ###
col_names = data.columns
num_rows = data.shape[0]

for feature in col_names:
    count = data[feature].isin(["?"]).sum() #sum the "?"s in the column
#     print("?s in", feature, ":", count)

# Get rid of samples with"?"
    warnings.simplefilter(action='ignore', category=FutureWarning)
    data = data[data[feature] != "?"]

data.income.value_counts()

# In[28]:

for feature in col_names:
    count = data[feature].isin(["?"]).sum() #sum the "?"s in the column
#     print("?s in", feature, ":", count)

categorical_feats = ['workclass', 'race', 'education', 'marital-status', 'occupation',
                    'relationship', 'gender', 'native-country', 'income']

#Check each possible value of a categorical feature and how often it occurs
for feature in categorical_feats:
    print("-----",feature,"-----")
    print(data[feature].value_counts()) # No.of each distinct feature/feature column

    n = len(pd.unique(data[feature]))

# In[29]:

### ENCODE DATA ###
# 1. Encode categorical features, drop the income
label_encoder = LabelEncoder()
cts_feats = ['age','fnlwgt','educational-num','capital-gain','capital-loss','hours-per-week']
cat_data = data.drop(columns = cts_feats)
cat_data_encoded = cat_data.apply(label_encoder.fit_transform)
labels = cat_data_encoded['income'].values.reshape(45222, 1)  # labels in columns
cat_data_encoded = cat_data_encoded.drop(columns = 'income')

one_hot_encoder = OneHotEncoder()
cat_one_hot = one_hot_encoder.fit(cat_data_encoded.values)
cat_one_hot = one_hot_encoder.transform(cat_data_encoded.values).toarray() #98 features

# 2. Normalize continuous features
cts_data = data.drop(columns=categorical_feats)
cts_mean = cts_data.mean(axis = 0)
cts_std = cts_data.std(axis = 0)
cts_data = cts_data.sub(cts_mean,axis=1)
cts_data = cts_data.div(cts_std,axis=1)

X = np.concatenate([cts_data, cat_one_hot, labels], axis = 1)
# X = np.concatenate([cts_data, cat_one_hot], axis = 1)
X_high = X[X[:,104] == 1] #11208
X_low = X[X[:,104] == 0]
print(X_high.shape)
print(X_low.shape)

X_male = X[X[:,62] == 1]
X_female = X[X[:,62] == 0]
print(X_male.shape)

# In[30]:

# Define Adult Dataset class
class AdultDataset(torch.utils.data.Dataset):
    def __init__(self, X, y):
        self.X = X
        self.y = y

    def __len__(self):
        return len(self.X)

    def __getitem__(self, index):
        # generates one sample of data
        return torch.from_numpy(self.X[index]),self.y[index]

# In[31]:

# Set the protion of different labels, Encode the training data into a required class
def load_data(portion: int):
    #portion of the rich
    n = 5500
    n_high = int(n/100*portion)
    n_low = n - n_high
    sample_train = None

    np.random.shuffle(X_high)
    np.random.shuffle(X_low)
    sample_train = np.concatenate((X_high[:n_high], X_low[:n_low]), 0)

    train_X = sample_train[:,:104]
    train_y = sample_train[:,104:105].reshape(n)
    train_data = AdultDataset(train_X, train_y)

#     np.random.shuffle(X)
#     sample_test = X[1000:1200]
#     test_X = sample_test[:,:104]
#     test_y = sample_test[:,104:105].reshape(200)
#     test_data = AdultDataset(test_X, test_y)

#     train_loader = DataLoader(dataset=train_data,batch_size=32,shuffle=True)
#     test_loader = DataLoader(dataset=test_data,batch_size=32,shuffle=True)
    return train_data

# In[32]:

# Define the architecture of the network
class net(nn.Module):
    def __init__(self):
        super(net, self).__init__()
        self.fc1 = nn.Linear(104, 32)
        self.fc2 = nn.Linear(32, 16)
        self.fc3 = nn.Linear(16, 8)
        self.fc4 = nn.Linear(8, 1)

    def forward(self, x):
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = F.relu(self.fc3(x))
#         return torch.sigmoid(self.fc4(x))
        return self.fc4(x)

# In[33]:

#Define the accuracy on the test set
def accuracy(model, val_loader):
    model.eval()
    total_corr = 0
    for i,data in enumerate(val_loader):
        inputs,labels = data
        y_pred = model(inputs.float())
        for i in range (len(labels)):
            #<=50k is encoded to 0 and > 50k encoded to 1
            if (y_pred[i].item() > 0.5):
                r = 1
            else:
                r = 0
            if (r == labels[i].item()):
                total_corr += 1

    return float(total_corr)/len(val_loader.dataset)

# In[35]:

x_high = X_high[:500,:104]
y_high = X_high[:500,104:105]
high = AdultDataset(x_high, y_high)
high_loader = DataLoader(dataset = high, batch_size=32, shuffle=True)

x_low = X_low[:500,:104]
y_low = X_low[:500,104:105]
low = AdultDataset(x_low, y_low)
low_loader = DataLoader(dataset = low, batch_size=32, shuffle=True)

# In[65]:

# Initialize a network and specify the loss function and optimizer|binary cross entrophy and Adam optimizer
def get_model_paras(portion: int):
    print('Portion: '+ str(portion))
    lr = 0.001 #learning rate
    epochs = 30

    train_dataset, test_dataset = torch.utils.data.random_split(load_data(portion = portion), [5000, 500])
    train_loader = DataLoader(dataset=train_dataset,batch_size=32,shuffle=True)
    test_loader = DataLoader(dataset=test_dataset,batch_size=32,shuffle=True)
#     train_loader, test_loader = load_data(portion = portion)

    model = net()
    #CrossEntropyLoss takes batch_size*class_number tensor as pred, batch_size 1D tensor as target
    criterion = nn.BCEWithLogitsLoss()
#     criterion = nn.BCELoss()
    optimizer = torch.optim.Adam(model.parameters(), lr, weight_decay=1e-2)
    model.train()
    loss_list = []

    for epoch in range(epochs):
            running_loss = 0
            for data, target in train_loader:
                optimizer.zero_grad()
                output = model(data.float()).squeeze()
                loss = criterion(output.float(), target.float())
                loss.backward()
                optimizer.step()
                running_loss += loss.item()

            loss_list.append(running_loss)
    plt.plot(loss_list)
    plt.xlabel('epochs')
    plt.ylabel('Crossentropy Loss')
    plt.show()

    layers = []
    i = 0
    for name, param in model.named_parameters():
        print(name)
        print(param.size())
        if(i%2 == 0):
            layers.append(param.data)
        else:
            k = int((i-1)/2)
            layers[k] = torch.cat((layers[k], param.data.unsqueeze(1)), 1)
        i += 1

#     acc = accuracy(model, test_loader)
# #     print('Accuracy on test set: {}'.format(acc))

#     acc_high = accuracy(model, high_loader)
# #     print('Accuracy on high test set: {}'.format(acc_high))
#     acc_low = accuracy(model, low_loader)
# #     print('Accuracy on low test set: {}'.format(acc_low))
# #     print('------------------------------------------')
    return acc, acc_high, acc_low

# In[66]:

a,b,c = get_model_paras(portion = 10)

# In[46]:

acc_l = []
high_acc = []
low_acc = []
for i in range(100):
    acc, acc_high, acc_low = (get_model_paras(portion = i))
    acc_l.append(acc)
    high_acc.append(acc_high)
    low_acc.append(acc_low)

# In[60]:

x = list(range(100))
for i,k in enumerate(x):
    x[i]=k/100
plt.plot(x,acc_l, label = 'Test set with the same data distribution')
plt.plot(x,high_acc, label = 'Test set with high income')
plt.plot(x,low_acc, label = 'Test set with low income')
plt.gca().xaxis.set_major_formatter(ticker.PercentFormatter(xmax=1, decimals=1))
plt.legend(loc = 'best')
plt.xlabel('Portion')
plt.ylabel('Accuracy on different test sets')
plt.savefig('Accuracy on different test sets', dpi=500, bbox_inches='tight')
plt.show()

# In[65]:

for i in range (2000):
    torch.save(get_model_paras(portion = i%100), 'meta_train/'+ str(i) +'.pt')
    ## e.g. if the index is 218, the percentage should be 18%

# In[ ]:


from keras.models import Model
from keras.layers import Input, Lambda
import numpy as np

# 定義 Lambda 要執行的函數
def minus(inputs):
    x, y = inputs
    return (x-y)

if __name__ == "__main__":
    # Step 1. 定義模型
    a = Input(shape=(2,))
    b = Input(shape=(2,))

    # Lambda layer
    minus_layer = Lambda(minus, name='minus')([a, b])

    # Model
    model = Model(inputs=[a, b], outputs=minus_layer)

    # Step 2. 測試模型
    v0 = np.array([8, 4])
    v1 = np.array([5, 2])

    # output: [[3, 2]]
    print(model.predict([v0.reshape(1, 2), v1.reshape(1, 2)]))