'''
Created on Oct 20, 2016

@author: christian
'''
import regex as re
import string

from andromeda.config.constants import Constants

class Tokenizer:

    def __init__(self):
        self._raw_string = ''
        self._char_map = []
        self._init_char_map()

    def _init_char_map(self):
        self._char_map = []
        for c in self._raw_string:
            self._char_map.append((Constants.TOKENIZER__TOKEN_CLASS__UNKNOWN, 0))

    def _set_token_class(self, start_pos, end_pos, token_class, token_class_nr=0, overwrite=False):
        for pos in range(start_pos, end_pos):
            if self._char_map[pos][0] == Constants.TOKENIZER__TOKEN_CLASS__UNKNOWN or overwrite == True:
                self._char_map[pos] = (token_class, token_class_nr)

    def tokenize(self, s):
        self._raw_string = s.strip()
        self._init_char_map()

        self._match_whitespaces(Constants.TOKENIZER__TOKEN_CLASS__WHITESPACE)
        #self._match_unicode_strings(Contstants.TOKENIZER__TOKEN_CLASS__UNICODE)
        self._match_urls(Constants.TOKENIZER__TOKEN_CLASS__URL)
        self._match_emails(Constants.TOKENIZER__TOKEN_CLASS__EMAIL)
        self._match_special_twitter_concepts('@', Constants.TOKENIZER__TOKEN_CLASS__SPECIAL_TERM_USER)
        self._match_special_twitter_concepts('#', Constants.TOKENIZER__TOKEN_CLASS__SPECIAL_TERM_TOPIC)
        self._match_html_entities(Constants.TOKENIZER__TOKEN_CLASS__HTML_ENTITY)
        #self._match_time_formats(Contstants.TOKENIZER__TOKEN_CLASS__TIME_FORMAT)
        self._match_numbers(Constants.TOKENIZER__TOKEN_CLASS__NUMBER)
        self._match_brackets()
        self._match_quotes()
        self._match_emoticons(Constants.TOKENIZER__TOKEN_CLASS__EMOTICON)
        #self._match_laughter(Contstants.TOKENIZER__TOKEN_CLASS__LAUGHTER)
        self._match_character_substrings('-=\+<>', 2, Constants.TOKENIZER__TOKEN_CLASS__CHARACTER_SUBSTRING)
        self._match_repeated_chars(Constants.TOKENIZER__PUNCTUATION_MARKS__TERMINAL_POINTS, Constants.TOKENIZER__TOKEN_CLASS__PUNCTUATION_TERMINAL_POINTS_REPEATED)
        self._match_repeated_chars(Constants.TOKENIZER__PUNCTUATION_MARKS__PAUSING_POINTS, Constants.TOKENIZER__TOKEN_CLASS__PUNCTUATION_PAUSING_POINTS_REPEATED)
        self._match_punctuation_marks(Constants.TOKENIZER__PUNCTUATION_MARKS__TERMINAL_POINTS, Constants.TOKENIZER__TOKEN_CLASS__PUNCTUATION_TERMINAL_POINTS)
        self._match_punctuation_marks(Constants.TOKENIZER__PUNCTUATION_MARKS__PAUSING_POINTS, Constants.TOKENIZER__TOKEN_CLASS__PUNCTUATION_PAUSING_POINTS)
        self._match_alphanumeric_words(Constants.TOKENIZER__TOKEN_CLASS__ALPHANUM)
        #self._match_negation_abbreviation(Contstants.TOKENIZER__TOKEN_CLASS__ALPHANUM_NEGATION_ABBREVIATION)
        self._match_selected_characters('\'&@*+', Constants.TOKENIZER__TOKEN_CLASS__ALPHANUM)
        self._match_unicode_characters(Constants.TOKENIZER__TOKEN_CLASS__UNICODE)
        return self._generate_token_list()

    def _match_html_entities(self, token_class):
        p = re.compile('&[^\s]*;')
        for m in p.finditer(self._raw_string):
            self._set_token_class(m.span()[0], m.span()[1], token_class)

    def _match_unicode_strings(self, token_class):
        token_class_nr = 0
        for size in [8,4]:
            p = re.compile(r'(\\[U])(\w{%s})' % size, re.IGNORECASE)
            for m in p.finditer(self._raw_string):
                token_class_nr += 1
                self._set_token_class(m.start(1), m.end(2), token_class, token_class_nr)

    def _match_urls(self, token_class):
        p = re.compile('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+#]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+[^ \<\.;:"\'\)\]\}]')
        for m in p.finditer(self._raw_string):
            self._set_token_class(m.span()[0], m.span()[1], token_class)

    def _match_emails(self, token_class):
        p = re.compile('([a-z0-9!#$%&\'*+\/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&\'*+\/=?^_`{|}~-]+)*(@|\sat\s)(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?(\.|\sdot\s))+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?)')
        for m in p.finditer(self._raw_string, re.UNICODE):
            self._set_token_class(m.span()[0], m.span()[1], token_class)

    def _match_whitespaces(self, token_class):
        p = re.compile(' ')
        for m in p.finditer(self._raw_string):
            self._set_token_class(m.span()[0], m.span()[1], token_class)

    def _match_repeated_chars(self, character_list, token_class):
        pattern_core = '|'.join(character_list)
        p = re.compile('['+pattern_core+']{2,}')
        for m in p.finditer(self._raw_string):
            self._set_token_class(m.span()[0], m.span()[1], token_class)

    def _match_character_substrings(self, character_list, min_length, token_class):
        core_pattern = ''.join(character_list)
        p = re.compile('['+core_pattern+']{2,}')
        for m in p.finditer(self._raw_string):
            self._set_token_class(m.span()[0], m.span()[1], token_class)

    def _match_quotes(self):
        p = re.compile('(?<=\W)"([^"'+Constants.TOKENIZER__PUNCTUATION_MARKS__PAUSING_POINTS+Constants.TOKENIZER__PUNCTUATION_MARKS__TERMINAL_POINTS+']*)"(?=\W)')
        for m in p.finditer(self._raw_string):
            self._set_token_class(m.span()[0], m.span()[0] + 1, Constants.TOKENIZER__TOKEN_CLASS__QUOTE_OPEN)
            self._set_token_class(m.span()[1] - 1, m.span()[1], Constants.TOKENIZER__TOKEN_CLASS__QUOTE_CLOSE)
        p = re.compile('(?<=\W)\'([^\':;\.,]*)\'(?=\W)')
        for m in p.finditer(self._raw_string):
            self._set_token_class(m.span()[0], m.span()[0] + 1, Constants.TOKENIZER__TOKEN_CLASS__QUOTE_OPEN)
            self._set_token_class(m.span()[1] - 1, m.span()[1], Constants.TOKENIZER__TOKEN_CLASS__QUOTE_CLOSE)

    def _match_brackets(self):
        bracket_pairs = [('(', ')'), ('[', ']'), ('{', '}'), ('<', '>')]
        for bracket_pair in bracket_pairs:
            p = re.compile('\\'+bracket_pair[0]+'([^\\'+bracket_pair[0]+'\\'+bracket_pair[1]+']*)\\'+bracket_pair[1])
            for m in p.finditer(self._raw_string):
                self._set_token_class(m.span()[0], m.span()[0] + 1, Constants.TOKENIZER__TOKEN_CLASS__BRACKET_OPEN)
                self._set_token_class(m.span()[1] - 1, m.span()[1], Constants.TOKENIZER__TOKEN_CLASS__BRACKET_CLOSE)

    def _match_punctuation_marks(self, character_list, token_class):
        pattern_core = '|'.join(character_list)
        p = re.compile(r'(['+pattern_core+'])')
        for m in p.finditer(self._raw_string):
            self._set_token_class(m.start(1), m.end(1), token_class)

    def _match_alphanumeric_words(self, token_class):
        p = re.compile('([a-zA-Z0-9]+)(?:[-[a-zA-Z0-9]|\']*([a-zA-Z0-9])?)?')
        for m in p.finditer(self._raw_string):
            self._set_token_class(m.span()[0], m.span()[1], token_class)

    def _match_special_twitter_concepts(self, start_char, token_class):
        p = re.compile(start_char+'([A-Za-z_]+[A-Za-z0-9_]+)')
        for m in p.finditer(self._raw_string):
            self._set_token_class(m.span()[0], m.span()[1], token_class)

    def _match_numbers(self, token_class):
        p = re.compile('(?<=\s)([+-]?\d+(?:.\d)?\d*)(?=\W)')
        for m in p.finditer(self._raw_string):
            self._set_token_class(m.span()[0], m.span()[1], token_class)

    def _match_time_formats(self, token_class):
        p = re.compile(r'(?:(?:([01]?\d|2[0-3]):)?([0-5]?\d):)?([0-5]?\d)\s?(am|pm|a\.m\.|p\.m\.)?')
        for m in p.finditer(self._raw_string):
            self._set_token_class(m.span()[0], m.span()[1], token_class)

    def _match_laughter(self, token_class):
        pattern_core = '|'.join(Contstants.TOKENIZER__EMOTEXTS_CORE_PATTERNS_LIST)
        p = re.compile(r'\b[a-z]*(' + pattern_core + r'){2,}[a-z]*\b', re.IGNORECASE)
        for m in p.finditer(self._raw_string):
            self._set_token_class(m.span()[0], m.span()[1], token_class)

    def _match_emoticons(self, token_class):
        # Generic patterns
        p = re.compile("(^|\s+)([%s]?)([%s])([%s]?)([%s]+)([,\.]|$|\s+)" % tuple(map(re.escape, [Constants.TOKENIZER__EMOTICONS_GENERIC_PATTERN_TOP, Constants.TOKENIZER__EMOTICONS_GENERIC_PATTERN_EYES, Constants.TOKENIZER__EMOTICONS_GENERIC_PATTERN_NOSES, Constants.TOKENIZER__EMOTICONS_GENERIC_PATTERN_MOUTHS])))
        for m in p.finditer(self._raw_string, overlapped=True):
            self._set_token_class(m.start(2), m.end(5), token_class)
        # Generic patterns (mirrored orientation)
        p = re.compile("(^|\s+)([%s]+)([%s]?)([%s])([%s]?)([,\.]|$|\s+)" % tuple(map(re.escape, [Constants.TOKENIZER__EMOTICONS_GENERIC_PATTERN_MOUTHS, Constants.TOKENIZER__EMOTICONS_GENERIC_PATTERN_NOSES, Constants.TOKENIZER__EMOTICONS_GENERIC_PATTERN_EYES, Constants.TOKENIZER__EMOTICONS_GENERIC_PATTERN_TOP])))
        for m in p.finditer(self._raw_string, overlapped=True):
            self._set_token_class(m.start(2), m.end(5), token_class)
        # Fixed patterns
        p = re.compile('|'.join(map(re.escape, Constants.TOKENIZER__EMOTICONS_FIXED_PATTERNS)))
        for m in p.finditer(self._raw_string, overlapped=True):
            self._set_token_class(m.span()[0], m.span()[1], token_class, overwrite=True)

    def _match_unicode_characters(self, token_class):
        token_class_nr = 0
        p = re.compile("\W", re.UNICODE)
        for m in p.finditer(self._raw_string):
            c = self._raw_string[m.span()[0]:m.span()[1]]
            if c not in string.printable:
                token_class_nr += 1
                self._set_token_class(m.span()[0], m.span()[1], token_class, token_class_nr)

    def _match_selected_characters(self, character_list, token_class):
        pattern_core = '|'.join(character_list)
        p = re.compile('['+pattern_core+']{1}')
        for m in p.finditer(self._raw_string):
            self._set_token_class(m.span()[0], m.span()[1], token_class)

    def _match_negation_abbreviation(self, token_class):
        p = re.compile('([A-Za-z0-9_]+)([^n])([n]+\'[t]+)' , re.IGNORECASE)
        for m in p.finditer(self._raw_string):
            self._set_token_class(m.start(3), m.end(3), token_class, overwrite=True)

    def __match_time_formats(self, token_class):
        p = re.compile(r'(?:(?:([01]?\d|2[0-3]):)?([0-5]?\d):)?([0-5]?\d)\s?(am|pm|a\.m\.|p\.m\.)?')
        for m in p.finditer(self._raw_string):
            self._set_token_class(m.span()[0], m.span()[1], token_class)

    def _generate_token_list(self):
        token_list = []
        token = ''
        current_token_class = -1
        current_token_class_nr = -1
        for pos, val in enumerate(self._char_map):
            token_class = val[0]
            token_class_nr = val[1]

            if token_class != current_token_class or token_class_nr != current_token_class_nr or token_class == Constants.TOKENIZER__TOKEN_CLASS__NON_ASCII:
                if not token.isspace() and current_token_class >= 0:
                    token_list.append({ Constants.TOKENIZER__TOKEN_FEATURE__BASE :
                                            { Constants.TOKENIZER__TOKEN_FEATURE__TOKEN: token,
                                              Constants.TOKENIZER__TOKEN_FEATURE__CLASS: current_token_class,
                                              Constants.TOKENIZER__TOKEN_FEATURE__FIRST_CHARACTER_INDEX: (pos - len(token)),
                                              Constants.TOKENIZER__TOKEN_FEATURE__CHARACTER_COUNT: len(token)
                                            }
                                      })
                token = ''
                current_token_class = token_class
                current_token_class_nr = token_class_nr
            token += self._raw_string[pos]

        if token != '':
            token_list.append({ Constants.TOKENIZER__TOKEN_FEATURE__BASE :
                                    { Constants.TOKENIZER__TOKEN_FEATURE__TOKEN: token,
                                      Constants.TOKENIZER__TOKEN_FEATURE__CLASS: current_token_class,
                                      Constants.TOKENIZER__TOKEN_FEATURE__FIRST_CHARACTER_INDEX: (pos+1 - len(token)),
                                      Constants.TOKENIZER__TOKEN_FEATURE__CHARACTER_COUNT: len(token)
                                    }
                              })
        return token_list

if __name__ == "__main__":

    s = '@DemoUser Watched Spectre yesterday HAHAHAHA It was good, but not the "masterpiece" everyone had expected :('
    #s = '@DemoUser Can I do it? \U0001F61E \U0001F52B'
    #s = 'peoples\' consumers \'decide on what they really want\': SINGAPORE An elderly customer bought... http://t.co/dwSxPnUbJJ #singapore'
    #s = 'new \'Your Signature Emoji\' feature (http://bla.com)'
    s = ':\'( :\'( \'test\' yep'
    #s = ':\'( :\'( "test" yes'
    s = '\U0001F602\U0001F602Woooow\u2026Deadpool\U0001F602'.decode('unicode-escape')

    tokenizer = Tokenizer()
    token_list = tokenizer.tokenize(s)


import datetime, time
import os
from difflib import SequenceMatcher

#size of the item length
#decided in the stringMargin method
sizeX = 0
# predefined width for the box that will contain items
DEFAULT_SIZE_X=50
sizeY = 8
# takes an input of list of contents that need to be printed in a box
# creates an artificial spaces around the item to fit it in the box
# if the list is less than 8 elemnts it will add empty string items
#if any item in the list is longer than 50 the box is resized to fit it
# to keep the box at a standard height
def stringMargin(listOfItemsForBox):

    #resize the height of the box
    listOfItemsForBox.append("")
    while len(listOfItemsForBox) < sizeY:
       listOfItemsForBox.append("")

    # refere to the outer sizeX
    global sizeX

    sizeX = len(max(listOfItemsForBox, key=len)) + 10
    if sizeX < DEFAULT_SIZE_X :
        sizeX = DEFAULT_SIZE_X
    # the final output of the function
    modifiedList = []
    for item in listOfItemsForBox :
        # calculates needed length
        needed_length = sizeX - len(item)
         # a standard right intended
        rightIndent = 6
        if item == listOfItemsForBox[0]:
            rightIndent = 3
        leftIndent = needed_length - rightIndent
        leftMargin = len(item) + leftIndent

         # creates left margin
        item = item.ljust(leftMargin)
        rightMargin = len(item) + rightIndent
         # adds right indent spaces into the item
        item = item.rjust(rightMargin, " ")
        modifiedList.append(item)
    return modifiedList

# function to generate a box
#takes input for the actual content of the box
#outputs a box
def boxGenerator(listOfContent):
	# clears the terminal before the box so the box stays on top
    os.system('clear')

    #takes the input list to stringMargin() to modify it
    listOfContent = stringMargin(listOfContent)

    # prints a corner then stays in the same line
    print(u'\u250F', end="", flush=True)
    for i in range(0, sizeX):
    	# prints the top-bar
        print(u'\u2501', end="", flush=True)

	# prints a corner
    print(u'\u2513')

    #  the body of the box
    for i in range(0, len(listOfContent)):
        # prints a vertical pipe
        print(u'\u2503', end="", flush=True)

        #prints the actual content with spaces around it
        print(listOfContent[i], end="", flush=True)

        # prints a vertical pipe
        print(u'\u2503')

    # prints a T style corner
    print(u'\u2523', end="", flush=True)

    # prints the bottom bar
    for i in range(0, sizeX):
    	# prints a horizontal bar
        print(u'\u2501', end="", flush=True)

    # prints a corner
    print(u'\u251B')

    #prints a corner
    print(u'\u2517', end="", flush=True)

    # prints a vertical line
    for i in range(0, 15):
        print(u'\u2501', end="", flush=True)

#cleans the sentence from special characters
def sentenceModification(sentence):
    sentence = sentence.lstrip().rstrip().lower()
    for chars in sentence:
        if not chars.isalpha() and chars !=" ":
            sentence = sentence.replace(chars, "")
    return sentence

#dictionary that contains word counts
fileWordsInformation = {
    'total Number of words': 0,
    'correctly spelt words': 0,
    'incorrectly spelt words': 0,
    'ignored words': 0,
    'marked words': 0,
    'words added to dictionary': 0
}

# checks words in sentences
def checkWordsForSentences(sentence):
    #cleans the sentence from any special characters
    sentence = sentenceModification(sentence)
    #splits the sentence into list of words
    listofWords = sentence.split()
    fileWordsInformation["total Number of words"] = len(listofWords)
    fileWordsInformation["correctly spelt words"] = 0
    fileWordsInformation["incorrectly spelt words"] = 0

    #irretates through the list of words to find any matching in the dictionary
    #prints the word is spelt correctly if it was found in the dictionary
    for word in listofWords:
        if word in EnglishWords:
            boxGenerator(["W O R D     F O U N D", "", "", word,
            "Spelt correctly"])
            #introduce a lag so the screen above does not disapper
            # if next screens come
            dummyInput = input("enter any key to go next")
            fileWordsInformation["correctly spelt words"] += 1
        else:
            boxGenerator(["W O R D   N O T  F O U N D", "", "", word,
             "Not found in dictionary"])
            dummyInput = input("enter any key to go next")
            fileWordsInformation["incorrectly spelt words"] += 1
    finalBoxList = ["F I N A L   R E S U L T", ""]

    #prints the fileWordsInformation which contains number of words,
    # spelt correctly not spelt correctly
    for x, y in fileWordsInformation.items():
        tempStr = "Number of " + x + " : " + str(y)
        finalBoxList.append(tempStr)
        #to not print the file related counts
        if len(finalBoxList) == 5:
            break
    #generate final information
    boxGenerator(finalBoxList)

#a funtion to spell check for words
#takes the input for the file and file name
#creates a file with the name 2+file name
#saves the spellchecked words to the new file
#offeres the user the chance to use other words
#instead of the incorrect ones
def checkWordsForFiles(sentence, file):
    sentence = sentenceModification(sentence)
    listofWords = sentence.split()
    fileWordsInformation["total Number of words"] = len(listofWords)
    correctedWords = ""
    resultFile = open("2" + file, "w")
    for word in listofWords:
        if word in EnglishWords:
            correctedWords += word + " "
            fileWordsInformation["correctly spelt words"] += 1
        # case the word is spelt incorrectly
        else:
            fileWordsInformation["incorrectly spelt words"] += 1
            listOfRaitos = []
            wordsFromRatios = []
            #saves the list of all ratios against the word from dictonary
            for matchedWord in EnglishWords:
                ratio = SequenceMatcher(None, matchedWord, word).ratio()
                listOfRaitos.append(ratio)
            desiredInput = False
            # insure correct input from the user
            while not desiredInput:
            	#takes the maximum ratio available to the word
                requiredWordIndex = listOfRaitos.index(max(listOfRaitos))
                desiredWord = EnglishWords[requiredWordIndex]
                boxGenerator(["W O R D   N O T   F O U N D", "", word, "", "did you mean", "", desiredWord])
                changeWordInput = input(" Enter [y] or [n]: ")
                print("\n")
                if changeWordInput == "y":
                    correctedWords += desiredWord + " "
                    desiredInput = True

                #in case the user wants to save the word to a dictionary
                #or mark as incorrect or ignore it
                elif changeWordInput == "n":
                    desiredInput = True
                    correctInput = False
                    # asks the user to add the word into dictionart
                    # ignore the word or mark it
                    # while loop is to ensure correct input from the user
                    while not correctInput:
                        boxGenerator(["W O R D   N O T   F O U N D", "", word, "", "1. Ignore the word.",
                                      "2. Mark the word as incorrect.", "3. Add word to dictionary."])
                        incorrectWordsHandling = input(" Enter choice: ")
                        print("\n")
                        if incorrectWordsHandling == "1":
                            correctedWords += "!" + word + "! "
                            fileWordsInformation["ignored words"] += 1
                            correctInput = True
                        elif incorrectWordsHandling == "2":
                            correctedWords += "?" + word + "? "
                            fileWordsInformation["marked words"] += 1
                            correctInput = True
                        elif incorrectWordsHandling == "3":
                            correctedWords += "*" + word + "* "
                            fileWordsInformation["words added to dictionary"] += 1
                            englishDict.write("\n" + word)
                            correctInput = True
    #prints the data into the corrected file
    resultFile.write(datetime.datetime.now().strftime("%d-%m-%Y %H:%M:%S\n"))
    resultFile.write("\n")
    finalBoxList = ["F I N A L   R E S U L T", ""]
    for x, y in fileWordsInformation.items():
        tempStr = "Number of " + x + " : " + str(y)
        temp2Str = tempStr + "\n"
        finalBoxList.append(tempStr)
        resultFile.write(temp2Str)
    resultFile.write("\n")
    #writes all the words after correction to the file
    resultFile.write(correctedWords)
    resultFile.close()

    #generates feedback with word counts, correct, incorrect,ignored...etc
    boxGenerator(finalBoxList)
    dummyInput = input(" enter any key to go next ")

# the main screen
terminated = False
englishDictReadOnly = open("EnglishWords.txt")
# creates a list to contain the words in the file
EnglishWords = englishDictReadOnly.read().split("\n")
englishDictReadOnly.close()
# reopens the file in append mode so it can be edited
englishDict = open("EnglishWords.txt", "a")
while not terminated:

	#generates the main screen
    boxGenerator(["S P E L L   C H E C K E R", "", "1. Check a file", "2. Check a sentence", "", "", "0.Quit"])
    usrInput = input(" Enter choice: ")

    # file mode
    if usrInput == "1":
        fileWrittenCorrect = False
        userFileNameInput = ""
        #validates input for file name
        while not fileWrittenCorrect:
        	# file mode main screen
            boxGenerator(["L O A D    F I L E", "", "Enter a file name", "then press [enter]"])
            if userFileNameInput == "":
                userFileNameInput = input(" Filename: ")
            startTime = time.clock()

            #checks if the file name exists
            if os.path.exists(userFileNameInput):
                boxGenerator(["L O A D    F I L E", "", "", "Correct File name", "", "Done"])
                checkWordsForFiles(open(userFileNameInput).read(), userFileNameInput)
                fileWrittenCorrect = True
                open(userFileNameInput).close()
                endTime = time.clock() - startTime
                #shows the time in microseconds
                boxGenerator(["C H E C K E D   F I L E", "", "TimeElapsed", str(endTime * 10 ** 6), "microseconds"])
            else:
                boxGenerator(["L O A D    F I L E", "", "Enter a file name or press q to quit", "then press [enter]"])
                userFileNameInput = input(" Filename: ")
                if (userFileNameInput == "q"):
                    break
	# sentence mode
    elif usrInput == "2":

    	# prints sentence screen
        boxGenerator(["S E N T E N C E   M O D E", "", "Enter a sentence", "then press [enter]"])
        checkWordsForSentences(input(" Enter sentence to spellcheck: "))
        usrInput = input(" Press q [enter] to quit or any other key [enter] to go again: ")
        # termenates the program
        if usrInput == "q":
            terminated = True

    # quits the program
    elif usrInput == "0" or usrInput == "q":
        terminated = True
        englishDict.close()


from src.Utils import Utils
import pygame

class PlayField:

    @staticmethod
    #This method creates an empty grid and add cells from blocked shapes
    def create_grid(locked_positions={}):  # locked_positions is a dictionary (keys=coordinates, values=rgb colors)
        grid = [[Utils.EMPTY_CELL_COLOR for x in range(Utils.COLUMNS)] for x in range(Utils.ROWS)]

        for row in range(Utils.ROWS):
            for col in range(Utils.COLUMNS):
                if (col, row) in locked_positions:
                    c = locked_positions[(col, row)] #vector of tuples
                    grid[row][col] = c
        return grid

    @staticmethod
    def check_lost(positions):
        for pos in positions:
            col, row = pos
            if row < 1:  # means that a piece is over the screen height
                return True
        return False

    @staticmethod
    def draw_text_middle(text, size, color, surface):
        pass

    @staticmethod
    def draw_grid(surface):
        sx = Utils.TOP_LEFT_X
        sy = Utils.TOP_LEFT_Y
        # Draw grid
        for row in range(Utils.ROWS):
            # horizontal lines
            pygame.draw.line(surface, Utils.GRID_COLOR, (sx, sy + row * Utils.BLOCK_SIZE),
                             (sx + Utils.PLAY_W, sy + row * Utils.BLOCK_SIZE))
        for col in range(Utils.COLUMNS):
            # vertical lines
            pygame.draw.line(surface, Utils.GRID_COLOR, (sx + col * Utils.BLOCK_SIZE, sy),
                             (sx + col * Utils.BLOCK_SIZE, sy + Utils.PLAY_H))
        return surface

    @staticmethod
    def clear_rows(grid, locked):
        # if a row is clear then shifts other row above down one

        row_full = 0
        for i in range(Utils.ROWS - 1, -1, -1): #from len(grid)-1 to 0 step -1 (from Rows to 0)
            row = grid[i] #grid contains (r,g,b) that is (0,0,0) if cell is empty (grid[i] is a row)
            if Utils.EMPTY_CELL_COLOR not in row:
                row_full += 1 #this increments if the row is full
                # add positions to remove from locked
                ind = i #row i is full
                for j in range(Utils.COLUMNS): #delete cells in the row that doesn't contain empty cell
                    try:
                        del locked[(j, i)] #delete row in blocked cell
                    except:
                        continue
        #now we need to shift all cells of locked, down of many rows as row_full
        if row_full > 0:
            for key in sorted(list(locked), key=lambda x: x[1])[::-1]:
                col, row = key
                if row < ind:
                    newKey = (col, row + row_full)
                    locked[newKey] = locked.pop(key) #add row in the top
        return locked, row_full

    @staticmethod
    def draw_next_shape(piece, surface):
        #This method draws the next piece in the right side of the screen
        sx = Utils.TOP_LEFT_X + Utils.PLAY_W + 50
        sy = Utils.TOP_LEFT_Y + Utils.PLAY_H / 2 - 150
        font = pygame.font.SysFont('comicsans', 30)
        label = font.render('Next Shape', 1, Utils.FONT_COLOR)
        surface.blit(label, (sx + 10, sy - 30))

        format = piece.shape[piece.rotation % len(piece.shape)] #select for every shape the right rotation
        #format = ['.....','...O.','..OO.','..OOO']
        for i, line in enumerate(format):
            row = list(line)
            for j, column in enumerate(row):
                if column == '0':
                    pygame.draw.rect(surface, piece.color, (sx + j*Utils.BLOCK_SIZE, sy + i*Utils.BLOCK_SIZE,
                                                            Utils.BLOCK_SIZE, Utils.BLOCK_SIZE), 0)
                    pygame.draw.rect(surface, Utils.GRID_COLOR, (sx + j * Utils.BLOCK_SIZE, sy + i * Utils.BLOCK_SIZE,
                                                            Utils.BLOCK_SIZE, Utils.BLOCK_SIZE), 1)

    @staticmethod
    def draw_window(surface, grid, score=0):

        #background image
        bg = pygame.image.load("../media/bg.jpg")
        surface.blit(bg, (0, 0))

        # surface.fill(Utils.BACKGROUND_COLOR)
        # Tetris Title
        font = pygame.font.SysFont('comicsans', 60)
        label = font.render('TETRIS', 1, Utils.FONT_COLOR)
        surface.blit(label, ((Utils.TOP_LEFT_X + Utils.PLAY_W / 2 - (label.get_width() / 2), 30)))
        # Draw border
        pygame.draw.rect(surface, (0, 0, 255), (Utils.TOP_LEFT_X, Utils.TOP_LEFT_Y, Utils.PLAY_W, Utils.PLAY_H), 3)

        # current score
        font = pygame.font.SysFont('comicsans', 30)
        label = font.render('Score: ' + str(score), 1, Utils.FONT_COLOR)
        surface.blit(label, (Utils.TOP_LEFT_X - 150, Utils.TOP_LEFT_Y + 110))

        #draw blocks
        for i in range(Utils.ROWS):
            for j in range(Utils.COLUMNS):
                pygame.draw.rect(surface, grid[i][j],(Utils.TOP_LEFT_X + j * Utils.BLOCK_SIZE, Utils.TOP_LEFT_Y + i * Utils.BLOCK_SIZE,
                                  Utils.BLOCK_SIZE, Utils.BLOCK_SIZE), 0)
        #draw grid
        surface = PlayField.draw_grid(surface)

        return surface


import logging
import itertools
import pandas as pd
from modelseedpy import MSGenome
from Bio.Seq import Seq

logger = logging.getLogger(__name__)

class KEGenome:
    def __init__(
        self,
        genome,
        assembly=None,
        contigs=None,
        rast=None,
        kb_re=None,
        load_arango=None,
        transform_rast=None,
    ):
        self.kb_re = kb_re
        self.seq_protein_store = kb_re.protein_store
        self.seq_dna_store = kb_re.dna_store
        self.genome = genome
        self.assembly = assembly
        self.rast = rast
        self.p_annotation = {}
        self.contig_data = contigs
        self.contigs = {}
        self.load_arango = load_arango
        self.transform_rast = transform_rast
        if self.contig_data:
            for contig in self.contig_data.features:
                self.contigs[contig.id] = [
                    contig.seq,
                    str(Seq(contig.seq).complement()),
                ]

        self.d_hash = {}
        self.p_hash = {}
        for o in self.genome.features:
            if o.dna_sequence:
                hash_d = self.seq_dna_store.store_sequence(o.dna_sequence)
                self.d_hash[o.id] = hash_d
            if o.protein_translation:
                hash_p = self.seq_protein_store.store_sequence(o.protein_translation)
                self.p_hash[o.id] = hash_p
            elif len(o.dna_sequence) % 3 == 0:
                try:
                    protein_translation = Seq(o.dna_sequence).translate()
                    hash_p = self.seq_protein_store.store_sequence(protein_translation)
                    self.p_hash[o.id] = hash_p
                except ValueError as e:
                    logger.debug(f"unable to translate {e}")

        self.loc_pos = {}
        self.gene_pairs = set()
        self.f_pairs = {}
        self.null_pairs = {}
        self.anno_pairs = {}
        self.null_annotation = {"rast/hypotheticalprotein", "rast/null", None}

    def run(self, batch_size=20, skip_seq_match=False):
        self.fetch_annotation(batch_size)
        self.loc_pos = self.get_loc_pos()
        self.gene_pairs = set()
        for contig_id in self.loc_pos:
            pairs = self.get_pairs(self.loc_pos[contig_id])
            self.gene_pairs |= pairs
        self.f_pairs = self.get_f_pairs(self.gene_pairs)
        self.null_pairs, self.anno_pairs = self.get_pp(self.f_pairs)

    @staticmethod
    def load_from_kbase(kbase, object_id, ws, kb_re, rast, load_arango, transform_rast):
        genome = kbase.get_from_ws(object_id, ws)
        assembly = kbase.get_from_ws(genome.assembly_ref)
        kbase.download_file_from_kbase2(
            assembly.fasta_handle_ref,
            "/home/fliu/kbase/cache/handle/" + assembly.fasta_handle_ref,
        )
        ms_contigs = MSGenome.from_fasta(
            "/home/fliu/kbase/cache/handle/" + assembly.fasta_handle_ref, split=" "
        )

        return KEGenome(
            genome, assembly, ms_contigs, rast, kb_re, load_arango, transform_rast
        )

    def _fetch_annotation_arango(self):
        aql = """
        FOR doc_protein IN re_seq_protein_has_rast_function
            FILTER doc_protein._from IN @h
            RETURN [doc_protein._from, doc_protein._to]
        """
        h_list = list({"re_seq_protein/" + x[1] for x in self.p_hash.items()})
        res = self.kb_re.db.AQLQuery(aql, rawResults=True, bindVars={"h": h_list})
        for h, annotation in res:
            if h not in self.p_annotation:
                self.p_annotation[h] = annotation

        return h_list

    def fetch_annotation(self, batch_size=20):
        h_list = self._fetch_annotation_arango()
        self.etl_rast(h_list, batch_size)
        self._fetch_annotation_arango()

        return self.p_annotation

    def etl_rast(self, h_list, batch_size=20):
        q = {}
        for i in h_list:
            if i not in self.p_annotation:
                q[i] = self.seq_protein_store.get_sequence(i.split("/")[-1])
                pass
            if len(q) >= batch_size:
                p_features = [
                    {"id": x[0], "protein_translation": x[1]} for x in q.items()
                ]
                res = self.rast.f(p_features)
                _rast_annotations = {}
                for o in res[0]["features"]:
                    _function_value = "null" if "function" not in o else o["function"]
                    _rast_annotations[o["id"].split("/")[-1]] = _function_value
                _nodes, _edges = self.transform_rast.transform(_rast_annotations)
                self.load_arango.load({"nodes": _nodes, "edges": _edges})
                q = {}
        if len(q) > 0:
            p_features = [{"id": x[0], "protein_translation": x[1]} for x in q.items()]
            res = self.rast.f(p_features)
            _rast_annotations = {}
            for o in res[0]["features"]:
                _function_value = "null" if "function" not in o else o["function"]
                _rast_annotations[o["id"].split("/")[-1]] = _function_value
            _nodes, _edges = self.transform_rast.transform(_rast_annotations)
            self.load_arango.load({"nodes": _nodes, "edges": _edges})

    def get_loc_pos(self, skip_seq_match=False):
        loc_pos = {}
        for f in self.genome.features:
            ll = locate_feature_dna_sequence_in_contig(f, self.contigs, skip_seq_match)
            contig_id = [x[0] for x in ll]
            if len(contig_id) == 1:
                contig_id = contig_id[0]
            else:
                raise ValueError("!!!" + str(contig_id))
            if contig_id not in loc_pos:
                loc_pos[contig_id] = {}
            if len(ll) == 0:
                print(ll, f.id)
                break
            else:
                p_min = None
                p_max = None
                for o in ll:
                    if p_min is None:
                        p_min = o[2]
                    elif p_min > o[2]:
                        p_min = o[2]
                    if p_min > o[3]:
                        p_min = o[3]
                    if p_max is None:
                        p_max = o[2]
                    elif p_max < o[2]:
                        p_max = o[2]
                    if p_max < o[3]:
                        p_max = o[3]
                if p_min >= p_max:
                    raise ValueError(
                        f"assertion fail for p_min < pmax: ({p_min}, {p_max})"
                    )
                if p_min not in loc_pos[contig_id]:
                    loc_pos[contig_id][p_min] = []
                loc_pos[contig_id][p_min].append([ll, p_min, p_max, [], []])

        return loc_pos

    def get_pairs(self, loc_pos):
        p_gene_id_pairs = set()
        s_pmin = sorted(loc_pos)

        def get_gene(d1):
            res = {}
            for loc in d1:
                loc_sites, p_min, p_max, member_l, member_r = loc
                for loc_site in loc_sites:
                    gene_id = loc_site[1]
                    if gene_id not in res:
                        res[gene_id] = []
                    res[gene_id].append(loc_site)
            return res

        for i in range(len(s_pmin) - 1):
            d1 = loc_pos[s_pmin[i]]
            d2 = loc_pos[s_pmin[i + 1]]
            g1 = get_gene(d1)
            g2 = get_gene(d2)
            somelists = [set(g1), set(g2)]
            for element in itertools.product(*somelists):
                p_gene_id_pairs.add(element)
            # print(set(g1), set(g2))

        return p_gene_id_pairs

    def get_f_pairs(self, p_gene_id_pairs):
        f_pairs = {}
        for g1, g2 in p_gene_id_pairs:

            if g1 in self.p_hash:
                f1 = self.p_annotation["re_seq_protein/" + self.p_hash[g1]]
            else:
                f1 = None
            if g2 in self.p_hash:
                f2 = self.p_annotation["re_seq_protein/" + self.p_hash[g2]]
            else:
                f2 = None

            f_pair = (f1, f2)
            if f_pair not in f_pairs:
                f_pairs[f_pair] = 0
            f_pairs[f_pair] += 1

        return f_pairs

    def get_pp(self, f_pairs1):
        null_pairs1 = {}
        anno_pairs1 = {}
        for p in f_pairs1:
            p1, p2 = p
            if p1 in self.null_annotation or p2 in self.null_annotation:
                null_pairs1[p] = f_pairs1[p]
            else:
                anno_pairs1[p] = f_pairs1[p]
        return null_pairs1, anno_pairs1

def locate_feature_dna_sequence_in_contig(f, contigs, skip_seq_match=False):
    ll = []
    contig_sub_strings = []

    for l in f.location:
        contig_id, start, direction, seq_size = l
        if contig_id in contigs:
            off_set = seq_size
            sub_str_start = start - 1
            sub_str_end = sub_str_start + off_set
            if direction == "+":
                contig_sub_string = contigs[contig_id][0][
                    sub_str_start : (sub_str_start + seq_size)
                ]
                contig_sub_strings.append(contig_sub_string)
                # if contig_sub_string == f.dna_sequence:
                ll.append(
                    [
                        contig_id,
                        f.id,
                        sub_str_start,
                        sub_str_start + seq_size,
                        False,
                        direction,
                        l,
                    ]
                )
                # print(f.id, contig_sub_string[:10], contig_sub_string[-10:])
                # print(f.id, f.dna_sequence[:10], f.dna_sequence[-10:])
            elif direction == "-":
                contig_sub_string = contigs[contig_id][1][
                    (sub_str_start - seq_size + 1) : (sub_str_start + 1)
                ]
                contig_sub_string = contig_sub_string[::-1]
                contig_sub_strings.append(contig_sub_string)
                ll.append(
                    [
                        contig_id,
                        f.id,
                        sub_str_start - seq_size + 1,
                        sub_str_start + 1,
                        True,
                        direction,
                        l,
                    ]
                )
                # print(f.id, contig_sub_string[:10], contig_sub_string[-10:])
                # print(f.id, f.dna_sequence[::-1][:10], f.dna_sequence[::-1][-10:])
            else:
                print("not sure what this means:", direction)

            # print(f.id, l, f.functions, contig_sub_string == f.dna_sequence)
        else:
            print("contig not found:", contig_id)
    if skip_seq_match or "".join(contig_sub_strings) == f.dna_sequence:
        return ll
    return []

class GenomeCluster:
    def __init__(self, genomes, gcf_to_ani, filename="./data/fastani_library.txt"):
        self.genomes = genomes
        self.ani_id_index = self.get_ani_id_index(filename)
        self.g_order = list(self.genomes)
        self.gcf_to_ani = gcf_to_ani

    @staticmethod
    def get_ani_id_index(filename="./data/fastani_library.txt"):
        df = pd.read_csv(filename, header=None, index_col=0)
        order = [row_id for row_id, d in df.iterrows()]
        ani_id_index = {}
        for i in range(len(order)):
            ani_id_index[order[i].split("/")[-1]] = i

        return ani_id_index

    def score(self, g1, g2, matrix):
        ke_genome1 = self.genomes[g1]
        ke_genome2 = self.genomes[g2]
        total_pairs = 0
        for p in ke_genome2.f_pairs:
            total_pairs += ke_genome2.f_pairs[p]

        common_pairs = 0
        for p in ke_genome2.anno_pairs:
            if p in ke_genome1.anno_pairs:
                c1 = ke_genome2.anno_pairs[p]
                c2 = ke_genome1.anno_pairs[p]
                if c1 >= c2:
                    common_pairs += c2
                else:
                    common_pairs += c1

        anno_1 = set()
        anno_2 = set()
        for a1, a2 in ke_genome2.f_pairs:
            anno_1.add(a1)
            anno_1.add(a2)
        for a1, a2 in ke_genome1.f_pairs:
            anno_2.add(a1)
            anno_2.add(a2)

        jaccard = len(set(anno_1 & anno_2)) / len(set(anno_1 | anno_2))
        i1 = self.ani_id_index[self.gcf_to_ani[g1][0]]
        i2 = self.ani_id_index[self.gcf_to_ani[g2][0]]
        ani_1_2 = matrix[i1][i2]
        ani_2_1 = matrix[i2][i1]
        # ani_a_b = gcf_to_ani
        return (
            total_pairs,
            common_pairs,
            common_pairs / total_pairs,
            jaccard,
            ani_1_2,
            ani_2_1,
        )

    def run(self, matrix_h5_filename="./data/ani_matrix.h5") -> dict:
        import h5py

        f = h5py.File(matrix_h5_filename, "r")
        matrix = f["matrix"]
        scores = {}
        for i1 in range(len(self.g_order)):
            g1 = self.g_order[i1]
            scores[g1] = {}
            for i2 in range(len(self.g_order) - i1):
                g2 = self.g_order[i2 + i1]
                scores[g1][g2] = self.score(g1, g2, matrix)
        f.close()
        return scores

    def write(self, scores, filename="./data/score_matrix.tsv"):
        with open(filename, "w") as fh:
            fh.write("\t")
            for i1 in range(len(self.g_order)):
                g1 = self.g_order[i1]
                fh.write(g1)
                fh.write("\t")
            fh.write("\n")
            for i1 in range(len(self.g_order)):
                g1 = self.g_order[i1]
                fh.write(g1)
                fh.write("\t")
                for i2 in range(len(self.g_order)):
                    g2 = self.g_order[i2]
                    if g1 in scores and g2 in scores[g1]:
                        s = scores[g1][g2]
                    else:
                        s = scores[g2][g1]
                    ani1 = s[4]
                    ani2 = s[5]
                    ani = s[2]
                    if g1 == g2:
                        ani = 0
                    fh.write(str(ani))
                    fh.write("\t")
                fh.write("\n")


import setuptools
import os
import re
import sys
import sysconfig
import platform
import subprocess
import numpy
from distutils.version import LooseVersion
from setuptools import setup, find_packages, Extension
from setuptools.command.build_ext import build_ext

'''
Modified from https://www.benjack.io/2017/06/12/python-cpp-tests.html
'''
class CMakeExtension(Extension):
    def __init__(self, name, sourcedir='',sources=[]):
        Extension.__init__(self, name, sources=[])
        self.sourcedir = os.path.abspath(sourcedir)

class CMakeBuild(build_ext):
    def run(self):
        try:
            out = subprocess.check_output(['cmake', '--version'])
        except OSError:
            raise RuntimeError(
                "CMake must be installed to build the following extensions: " +
                ", ".join(e.name for e in self.extensions))

        if platform.system() == "Windows":
            cmake_version = LooseVersion(re.search(r'version\s*([\d.]+)',
                                         out.decode()).group(1))
            if cmake_version < '3.1.0':
                raise RuntimeError("CMake >= 3.1.0 is required on Windows")

        for ext in self.extensions:
            self.build_extension(ext)

    def build_extension(self, ext):
        extdir = os.path.abspath(
            os.path.dirname(self.get_ext_fullpath(ext.name)))

        cmake_args = ['-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + extdir,
                      '-DPYTHON_EXECUTABLE=' + sys.executable]

        cfg = 'Debug' if self.debug else 'Release'
        build_args = ['--config', cfg]

        if platform.system() == "Windows":
            cmake_args += ['-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}'.format(
                cfg.upper(),
                extdir)]
            if sys.maxsize > 2**32:
                cmake_args += ['-A', 'x64']
            build_args += ['--', '/m']
        else:
            cmake_args += ['-DCMAKE_BUILD_TYPE=' + cfg]
            build_args += ['--', '-j2']
        cmake_args += ['-DNUMPY_DIR=' + numpy.get_include()]
        env = os.environ.copy()
        env['CXXFLAGS'] = '{} -DVERSION_INFO=\\"{}\\"'.format(
            env.get('CXXFLAGS', ''),
            self.distribution.get_version())
        if not os.path.exists(self.build_temp):
            os.makedirs(self.build_temp)
        subprocess.check_call(['cmake', ext.sourcedir] + cmake_args,
                              cwd=self.build_temp, env=env)
        subprocess.check_call(['cmake', '--build', '.'] + build_args,
                              cwd=self.build_temp)

with open("README.md", "r") as f:
    long_description = f.read()

setup(
    name="CudaVox",
    version="1.7.0",
    author="Ben Thorpe",
    author_email="b.j.thorpe@swansea.ac.uk",
    description="Python bindings for a C++ library to convert mesh models into Voxel images with OpenMp and CUDA",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url='https://github.com/bjthorpe/Cad2vox',
    classifiers=[
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: MIT License",
        "Operating System :: POSIX :: Linux"
    ],
    include_dirs=[numpy.get_include()],
    ext_modules=[CMakeExtension('CudaVox',sources=["python_bind.cpp"])],
    python_requires='>=3.6',
    cmdclass=dict(build_ext=CMakeBuild),
    zip_safe=False,
    install_requires=['numpy>=1.18','meshio','tifffile', 'pillow>=8.3', 'pandas', 'cmake>=3.22.0','pybind11'],
)


# import requests
# from bs4 import BeautifulSoup
# import json
#
# with open('data\\sport.csv', 'w', encoding='utf-8') as f:
#
#     url = 'https://sports.news.naver.com/wfootball/news/list.nhn?isphoto=N'
#     recvd = requests.get(url)
#     # print(recvd)
#     # print(recvd.text)
#     dic = json.loads(recvd.text)
#     # print(dic)
#     # print(dic['list'])
#
#     for i in dic['list']:
#         str = '{}::{}\n'.format(i['title'], i['subContent'])
#         # print(str)
#         f.write(str)

# import requests
# from bs4 import BeautifulSoup
# from fake_useragent import UserAgent
# import json
# import os
#
# # with open(os.path.join('data', 'money.csv'), 'w', encoding='utf-8') as f:
# with open(os.path.join('data', 'money.csv'), 'w') as f:
#
#     ua = UserAgent()
#     # print(ua.chrome)
#     # print(ua.ie)
#
#     headers = {'user-agent': ua.chrome, 'referer': 'https://finance.daum.net/'}
#
#
#     url = 'https://finance.daum.net/api/search/ranks?limit=10'
#     url2 = 'https://finance.daum.net/api/sectors?market=KOSPI&change=RISE&includedStockLimit=1&perPage=5'
#     url3 = 'https://finance.daum.net/api/sectors?market=KOSDAQ&change=RISE&includedStockLimit=1&perPage=5'
#
#     recvd = requests.get(url, headers=headers)
#     # print(recvd)
#     # print(recvd.text)
#
#     dic = json.loads(recvd.text)
#     for rank in dic['data']:
#         f.write('{},{},{}\n'.format(rank['rank'], rank['name'], rank['tradePrice']))
#
#     recvd =requests.get(url2, headers=headers)
#     dic = json.loads(recvd.text)
#     # print(dic)
#     for item in dic['data']:
#         print(item['sectorName'], item['includedStocks']['name'])

# 성탄절로 1000건 검색하여 제목(title)과 상세내용(description) blog.csv로 저장하세요

import os
import sys
import urllib.request
import json

with open('data\\blog.csv', 'w', encoding='utf-8') as f:

    client_id = "twrhEE4LU8HoKxIrMwzM"
    client_secret = "hsPSocfNRK"
    encText = urllib.parse.quote("성탄절")

    for i in range(1, 902, 100):
        url = "https://openapi.naver.com/v1/search/blog?start={}&display=100&query=".format(i) + encText # json 결과
        # url = "https://openapi.naver.com/v1/search/blog.xml?query=" + encText # xml 결과
        request = urllib.request.Request(url)
        request.add_header("X-Naver-Client-Id",client_id)
        request.add_header("X-Naver-Client-Secret",client_secret)
        response = urllib.request.urlopen(request)
        rescode = response.getcode()
        if(rescode==200):
            response_body = response.read()
            result = response_body.decode('utf-8')
        else:
            print("Error Code:" + rescode)

        dic = json.loads(result)
        for item in dic['items']:
            title = item['title']
            description = item['description']
            str = '{}::{}\n'.format(title, description)
            print(str)
            f.write(str)


""" Various sanity tests to check that filenames are handled corectly."""
import pytest
import cad2vox
import os

# use the test case folder as their working directory
@pytest.fixture(autouse=True)
def change_test_dir(request, monkeypatch):
    monkeypatch.chdir(request.fspath.dirname)

@pytest.fixture()
def cleanup():
    print("Starting Pytest")
    yield
    print("performing cleanup of output")
    if os.path.exists("greyscale.csv"):
        os.remove("greyscale.csv")

############ tests for unit length

@pytest.mark.parametrize("input_length",[5,"max","0.012",-0.00001])
def test_invalid_unit_length(input_length):
    # Test -ve or non-float values gives error
    with pytest.raises(TypeError):
        cad2vox.voxelise("inputs/AMAZE_Sample.med","outputs/AMAZE_Sample.tiff",unit_length=input_length)
    assert TypeError

@pytest.mark.parametrize("gridsize",[-5,"max","0.012",0.00001])
def test_invalid_gridsize(gridsize):
    # Test negative or non int value for gridsize gives error
    with pytest.raises(TypeError):
        cad2vox.voxelise("inputs/AMAZE_Sample.med","outputs/AMAZE_Sample.tiff",gridsize=gridsize)
    assert TypeError

def test_set_both():
    # Test negative or non int value for gridsize gives error
    with pytest.raises(TypeError):
        cad2vox.voxelise("inputs/AMAZE_Sample.med","outputs/AMAZE_Sample.tiff",gridsize=[100,100,100],unit_length=[0.0001,0.0001,0.0001])
    assert TypeError


from server.server import Server
HOST = '127.0.0.1'
PORT = 8888
if __name__ == '__main__':
    while True:
        server = Server(HOST, PORT)
        try:
            server.connect_client()
            server.setup()
            server.handle_rtsp_requests()
        except ConnectionError as e:
            ## server.server_state = server.STATE.TEARDOWN
            print(f"Connection reset: {e}")


import torch as th

# --------------------------------------------------------------
#    knn_avg
# --------------------------------------------------------------

def knn_avg (x, k=20, memory_limit=5e8):

    more_batches_are_present = False
    if x.dim() == 3: more_batches_are_present = True
    elif x.dim() == 2: x = x[None,:,:]
    elif x.dim() == 1: x = x[None,:,None]
    else: raise Exception("x must be 1D x[n], or 2D x[n,d], or 3D (batches) x[b,n,d]")
    b, n, d = x.shape
    k = _get_k(k, n, d)
    # b x n x n tensors could be too big. Batches of c1 x c2 x n are safer.
    c1, c2 = _get_c1_c2(b, n, memory_limit)

    sum_log_kth_dist = th.zeros(b, device=x.device)
    b_batch = 0
    while b_batch < b:
        n_batch = 0
        while n_batch < n:
            x_batch = x[b_batch:b_batch+c1, n_batch:n_batch+c2] # x_batch[c1,c2,d]
            dist, _ = th.cdist(x_batch, x[b_batch:b_batch+c1]).sort(dim=2) # dist[c1,c2,n] (n-sorted)
            del _
            sum_log_kth_dist[b_batch:b_batch+c1] = sum_log_kth_dist[b_batch:b_batch+c1] + \
                                                            th.sum(th.log(dist[:,:,1:k+1] + 1e-40), dim=(1,2))
            del dist
            n_batch += c2
        b_batch += c1
    h = d/n/k*sum_log_kth_dist # + const indipendent from x
    if more_batches_are_present: return h
    else: return h[0]

# --------------------------- utils ---------------------------

def _get_k (k, n, d):

    if k < 1: k = 1
    if k >= n: k = n - 1
    return k

def _get_c1_c2 (b, n, memory_limit):

    size_of_float32 = 4 # bytes
    c2 = int(memory_limit / size_of_float32 / n)
    if c2 < 1: c2 = 1
    if c2 > n:
        c2 = n
        c1 = int(memory_limit / size_of_float32 / n / c2)
        if c1 < 1: c1 = 1
        if c1 > b: c1 = b
    else: c1 = 1
    return c1, c2


import torch
import nimblephysics as nimble

def main():
  # Set up the world
  world = nimble.simulation.World()
  world.setGravity([0, -9.81, 0])
  world.setTimeStep(0.01)

  # Set up the box
  osim: nimble.biomechanics.OpenSimFile = nimble.models.RajagopalHumanBodyModel()
  skel: nimble.dynamics.Skeleton = osim.skeleton
  world.addSkeleton(skel)

  # Set up initial conditions for optimization
  bodyNode: nimble.dynamics.BodyNode = osim.markersMap['C7'][0]
  markerOffset: torch.Tensor = torch.zeros((3), requires_grad=True)
  bodyScale: torch.Tensor = torch.zeros((3), requires_grad=True)
  with torch.no_grad():
    markerOffset[:] = torch.from_numpy(osim.markersMap['C7'][1])
    bodyScale[:] = torch.from_numpy(bodyNode.getScale())

  while True:
    # Position is the first half of the state vector
    closestPoint: torch.Tensor = nimble.get_marker_dist_to_nearest_vertex(
        bodyNode, markerOffset, bodyScale)
    print('closest point dist: '+str(closestPoint))

    # Our loss is just the distance to the origin at the final step
    loss = torch.square(closestPoint - 1.0)
    print('loss: '+str(loss))

    # Manually update weights using gradient descent. Wrap in torch.no_grad()
    # because weights have requires_grad=True, but we don't need to track this
    # in autograd.
    loss.backward()
    with torch.no_grad():
      learning_rate = 0.01
      markerOffset -= learning_rate * markerOffset.grad
      markerOffset.grad = None
      bodyScale -= learning_rate * bodyScale.grad
      bodyScale.grad = None

if __name__ == "__main__":
  main()


"""
[incr tsdb()] Test Suites
"""

from typing import (
    Union, Iterable, Sequence, Tuple, List, Dict, Any,
    Iterator, Optional, IO, overload, Callable, cast as typing_cast
)
from pathlib import Path
import tempfile
from datetime import datetime
import logging
import collections
import itertools

from delphin import util
from delphin import tsdb
from delphin import interface
# Default modules need to import the PyDelphin version
from delphin.__about__ import __version__  # noqa: F401

logger = logging.getLogger(__name__)

##############################################################################
# Module variables

_default_task_selectors = {
    'parse': ('item', 'i-input'),
    'transfer': ('result', 'mrs'),
    'generate': ('result', 'mrs'),
}

#############################################################################
# Exceptions

class ITSDBError(tsdb.TSDBError):
    """Raised when there is an error processing a [incr tsdb()] profile."""

##############################################################################
# Processor Interface

Transaction = List[Tuple[str, tsdb.ColumnMap]]

class FieldMapper:
    """
    A class for mapping between response objects and test suites.

    If *source* is given, it is the test suite providing the inputs
    used to create the responses, and it is used to provide some
    contextual information that may not be present in the response.

    This class provides two methods for mapping responses to fields:

    * :meth:`map` -- takes a response and returns a list of (table,
      data) tuples for the data in the response, as well as
      aggregating any necessary information

    * :meth:`cleanup` -- returns any (table, data) tuples resulting
      from aggregated data over all runs, then clears this data

    And one method for mapping test suites to responses:

    * :meth:`collect` -- yield :class:`~delphin.interface.Response`
      objects by collecting the relevant data from the test suite

    In addition, the :attr:`affected_tables` attribute should list
    the names of tables that become invalidated by using this
    FieldMapper to process a profile. Generally this is the list of
    tables that :meth:`map` and :meth:`cleanup` create rows for,
    but it may also include those that rely on the previous set
    (e.g., treebanking preferences, etc.).

    Alternative [incr tsdb()] schemas can be handled by overriding
    these three methods and the __init__() method. Note that
    overriding :meth:`collect` is only necessary for mapping back from
    test suites to responses.

    Attributes:
        affected_tables: list of tables that are affected by the
            processing
    """
    def __init__(self, source: Optional[tsdb.Database] = None):
        # the parse keys exclude some that are handled specially
        self._parse_keys = '''
            ninputs ntokens readings first total tcpu tgc treal words
            l-stasks p-ctasks p-ftasks p-etasks p-stasks
            aedges pedges raedges rpedges tedges eedges ledges sedges redges
            unifications copies conses symbols others gcs i-load a-load
            date error comment
        '''.split()
        self._result_keys = '''
            result-id time r-ctasks r-ftasks r-etasks r-stasks size
            r-aedges r-pedges derivation surface tree mrs
        '''.split()
        self._run_keys = '''
            run-comment platform protocol tsdb application environment
            grammar avms sorts templates lexicon lrules rules
            user host os start end items status
        '''.split()
        self._parse_id = -1
        self._runs: Dict[int, Dict[str, Any]] = {}
        self._last_run_id = -1

        self.affected_tables = '''
            run parse result rule output edge tree decision preference
            update fold score
        '''.split()

        self._i_id_map: Dict[int, int] = {}
        if source:
            pairs = typing_cast(List[Tuple[int, int]],
                                source.select_from(
                                    'parse',
                                    ('parse-id', 'i-id'),
                                    cast=True))
            self._i_id_map.update(pairs)

    def map(self, response: interface.Response) -> Transaction:
        """
        Process *response* and return a list of (table, rowdata) tuples.
        """
        transaction: Transaction = []

        patch = self._map_parse(response)
        parse_id = patch['parse-id']
        assert isinstance(parse_id, int)
        transaction.append(('parse', patch))

        result: interface.Result
        for result in response.results():
            patch = self._map_result(result, parse_id)
            transaction.append(('result', patch))

        for edge in response.get('chart', []):
            patch = self._map_edge(edge, parse_id)
            transaction.append(('edge', patch))

        if 'run' in response:
            run_id = response['run'].get('run-id', -1)
            # check if last run was not closed properly
            if run_id not in self._runs and self._last_run_id in self._runs:
                last_run = self._runs[self._last_run_id]
                if 'end' not in last_run:
                    last_run['end'] = datetime.now()
            self._runs[run_id] = response['run']
            self._last_run_id = run_id

        return transaction

    def _map_parse(self, response: interface.Response) -> tsdb.ColumnMap:
        patch: tsdb.ColumnMap = {}
        # custom remapping, cleanup, and filling in holes
        keys = response.get('keys', {})
        if 'i-id' in keys:
            patch['i-id'] = keys['i-id']
        elif 'parse-id' in keys and keys['parse-id'] in self._i_id_map:
            patch['i-id'] = self._i_id_map[keys['parse-id']]
        else:
            patch['i-id'] = -1
        i_id = patch['i-id']
        assert isinstance(i_id, int)  # for type-checker's benefit, mainly
        self._parse_id = max(self._parse_id + 1, i_id)
        patch['parse-id'] = self._parse_id
        patch['run-id'] = response.get('run', {}).get('run-id', -1)
        if 'tokens' in response:
            patch['p-input'] = response['tokens'].get('initial')
            patch['p-tokens'] = response['tokens'].get('internal')
            if 'ninputs' not in response:
                toks = response.tokens('initial')
                if toks is not None:
                    response['ninputs'] = len(toks.tokens)
            if 'ntokens' not in response:
                toks = response.tokens('internal')
                if toks is not None:
                    response['ntokens'] = len(toks.tokens)
        if 'readings' not in response and 'results' in response:
            response['readings'] = len(response['results'])
        # basic mapping
        for key in self._parse_keys:
            if key in response:
                patch[key] = response[key]
        return patch

    def _map_result(self,
                    result: interface.Result,
                    parse_id: int) -> tsdb.ColumnMap:
        patch: tsdb.ColumnMap = {'parse-id': parse_id}
        if 'flags' in result:
            patch['flags'] = util.SExpr.format(result['flags'])
        for key in self._result_keys:
            if key in result:
                patch[key] = result[key]
        return patch

    def _map_edge(self,
                  edge: tsdb.ColumnMap,
                  parse_id: int) -> tsdb.ColumnMap:
        edge['parse-id'] = parse_id
        daughters = edge.get('e-daughters')
        if daughters:
            edge['e-daughters'] = util.SExpr.format(daughters)
        else:
            edge['e-daughters'] = None
        alternates = edge.get('e-alternates')
        if alternates:
            edge['e-alternates'] = util.SExpr.format(alternates)
        else:
            edge['e-alternates'] = None
        return edge

    def cleanup(self) -> Transaction:
        """
        Return aggregated (table, rowdata) tuples and clear the state.
        """
        inserts = []

        if self._last_run_id != -1:
            last_run = self._runs[self._last_run_id]
            if 'end' not in last_run:
                last_run['end'] = datetime.now()

            for run_id in sorted(self._runs):
                run = self._runs[run_id]
                d = {'run-id': run.get('run-id', -1)}
                for key in self._run_keys:
                    if key in run:
                        d[key] = run[key]
                inserts.append(('run', d))

        # reset for next task
        self._parse_id = -1
        self._runs = {}
        self._last_run_id = -1

        return inserts

    def collect(self, ts: 'TestSuite') -> Iterator[interface.Response]:
        """
        Map from test suites to response objects.

        The data in the test suite must be ordered.

        .. note::

           This method stores the 'item', 'parse', and 'result' tables
           in memory during operation, so it is not recommended when a
           test suite is very large as it may exhaust the system's
           available memory.
        """

        # type checking this function is a mess; it needs a better fix

        def get_i_id(row: 'Row') -> int:
            i_id = row['i-id']
            assert isinstance(i_id, int)
            return i_id

        def get_parse_id(row: 'Row') -> int:
            parse_id = row['parse-id']
            assert isinstance(parse_id, int)
            return parse_id

        parse_map: Dict[int, List[Dict[str, tsdb.Value]]] = {}
        rows  = typing_cast(Sequence['Row'], ts['parse'])
        for key, grp in itertools.groupby(rows, key=get_i_id):
            parse_map[key] = [dict(zip(row.keys(), row)) for row in grp]

        result_map: Dict[int, List[Dict[str, tsdb.Value]]] = {}
        rows  = typing_cast(Sequence['Row'], ts['result'])
        for key, grp in itertools.groupby(rows, key=get_parse_id):
            result_map[key] = [dict(zip(row.keys(), row)) for row in grp]

        for item in ts['item']:
            d: Dict[str, tsdb.Value] = dict(zip(item.keys(), item))
            i_id = d['i-id']
            assert isinstance(i_id, int)
            for parse in parse_map.get(i_id, []):
                response = interface.Response(d)
                response.update(parse)
                parse_id = parse['parse-id']
                assert isinstance(parse_id, int)
                response['results'] = result_map.get(parse_id, [])
                yield response

##############################################################################
# Test items and test suites

class Row(tsdb.Record):
    """
    A row in a [incr tsdb()] table.

    The third argument, *field_index*, is optional. Its purpose is to
    reduce memory usage because the same field index can be shared by
    all rows for a table, but using an incompatible index can yield
    unexpected results for value retrieval by field names
    (`row[field_name]`).

    Args:
        fields: column descriptions; an iterable of
            :class:`tsdb.Field` objects
        data: raw column values
        field_index: mapping of field name to its index in *fields*;
            if not given, it will be computed from *fields*
    Attributes:
        fields: The fields of the row.
        data: The raw column values.
    """

    __slots__ = 'fields', 'data', '_field_index'

    def __init__(self,
                 fields: tsdb.Fields,
                 data: Sequence[tsdb.Value],
                 field_index: Optional[tsdb.FieldIndex] = None):
        if len(data) != len(fields):
            raise ITSDBError(
                'number of columns ({}) != number of fields ({})'
                .format(len(data), len(fields)))
        if field_index is None:
            field_index = tsdb.make_field_index(fields)
        self.fields = fields
        self.data = tuple(tsdb.format(f.datatype, val)
                          for f, val in zip(fields, data))
        self._field_index = field_index

    def __repr__(self) -> str:
        return '<{} object ({}) at {}>'.format(
            type(self).__name__,
            ', '.join(map(repr, self)),
            id(self))

    def __str__(self) -> str:
        return tsdb.join(self, self.fields)

    def __eq__(self, other) -> bool:
        if not hasattr(other, '__iter__'):
            return NotImplemented
        return tuple(self) == tuple(other)

    def __len__(self) -> int:
        return len(self.fields)

    def __iter__(self) -> Iterator[tsdb.Value]:
        datatypes = tuple(field.datatype for field in self.fields)
        for datatype, raw_value in zip(datatypes, self.data):
            yield tsdb.cast(datatype, raw_value)

    @overload
    def __getitem__(self, key: int) -> tsdb.Value:
        ...

    @overload  # noqa: F811
    def __getitem__(self, key: slice) -> Tuple[tsdb.Value]:
        ...

    @overload  # noqa: F811
    def __getitem__(self, key: str) -> tsdb.Value:
        ...

    def __getitem__(self, key):  # noqa: F811
        if isinstance(key, slice):
            fields = self.fields[key]
            raw_values = self.data[key]
            return tuple(tsdb.cast(field.datatype, raw)
                         for field, raw in zip(fields, raw_values))
        else:
            if isinstance(key, str):
                index = self._field_index[key]
            else:
                index = key
            field = self.fields[index]
            raw_value = self.data[index]
            return tsdb.cast(field.datatype, raw_value)

    def keys(self) -> List[str]:
        """
        Return the list of field names for the row.

        Note this returns the names of all fields, not just those with
        the `:key` flag.
        """
        return [f.name for f in self.fields]

Rows = Sequence[Row]

class Table(tsdb.Relation):
    """
    A [incr tsdb()] table.

    Args:
        dir: path to the database directory
        name: name of the table
        fields: the table schema; an iterable of :class:`tsdb.Field`
            objects
        encoding: character encoding of the table file
    Attributes:
        dir: The path to the database directory.
        name: The name of the table.
        fields: The table's schema.
        encoding: The character encoding of table files.
    """

    def __init__(self,
                 dir: util.PathLike,
                 name: str,
                 fields: tsdb.Fields,
                 encoding: str = 'utf-8') -> None:
        self.dir = Path(dir).expanduser()
        self.name = name
        self.fields: Sequence[tsdb.Field] = fields
        self._field_index = tsdb.make_field_index(fields)
        self.encoding = encoding
        try:
            tsdb.get_path(self.dir, name)
        except tsdb.TSDBError:
            # file didn't exist as plain-text or gzipped, so create it
            path = self.dir.joinpath(name)
            path.write_text('')
        self._rows: List[Optional[Row]] = []
        # storing the open file for __iter__ let's Table.close() work
        self._file: Optional[IO[str]] = None

        # These two numbers are needed to track if changes to the
        # table are only additions or if they remove/alter existing
        # rows. The first is the number of rows in the file and the
        # second is the index of the first unwritten row
        self._persistent_count = 0
        self._volatile_index = 0

        self._sync_with_file()

    @property
    def _in_transaction(self) -> bool:
        num_recs = len(self._rows)
        return (num_recs > self._persistent_count
                or self._volatile_index < self._persistent_count)

    def _sync_with_file(self) -> None:
        """Clear in-memory structures so table is synced with the file."""
        self._rows = []
        i = -1
        with tsdb.open(self.dir,
                       self.name,
                       encoding=self.encoding) as lines:
            for i, line in enumerate(lines):
                self._rows.append(None)
        self._persistent_count = i + 1
        self._volatile_index = i + 1

    def __iter__(self) -> Iterator[Row]:
        if self._file is not None:
            self._file.close()
        fh: IO[str] = tsdb.open(self.dir, self.name,
                                encoding=self.encoding)
        self._file = fh

        for _, row in self._enum_rows(fh):
            yield row

    @overload
    def __getitem__(self, index: int) -> Row:
        ...

    @overload  # noqa: F811
    def __getitem__(self, index: slice) -> Rows:
        ...

    def __getitem__(self, index):  # noqa: F811
        if isinstance(index, slice):
            return self._iterslice(index)
        else:
            return self._getitem(index)

    def _iterslice(self, slice: slice) -> List[Row]:
        """Yield rows from a slice index."""
        with tsdb.open(self.dir, self.name, encoding=self.encoding) as fh:
            rows = [row for _, row in self._enum_rows(fh, slice)]
            if slice.step is not None and slice.step < 0:
                rows = list(reversed(rows))
            return rows

    def _getitem(self, index: int) -> Row:
        """Get a single non-slice index."""
        row = self._rows[index]
        if row is None:
            # need to handle negative indices manually
            if index < 0:
                index = len(self._rows) + index
            with tsdb.open(self.dir,
                           self.name,
                           encoding=self.encoding) as lines:
                for i, line in enumerate(lines):
                    if i == index:
                        row = Row(self.fields,
                                  tsdb.split(line),
                                  field_index=self._field_index)
                        break
        if row is None:
            raise ITSDBError('could not retrieve row {}'.format(index))
        return row

    @overload
    def __setitem__(self, index: int, value: Row) -> None:
        ...

    @overload  # noqa: F811
    def __setitem__(self, index: slice, value: Iterable[Row]) -> None:
        ...

    def __setitem__(self, index, value):  # noqa: F811
        # first normalize the arguments for slices and regular indices
        if isinstance(index, slice):
            values = list(value)
        else:
            if index < 0:
                index = len(self._rows) + index
            self._rows[index]  # check for IndexError
            values = [value]
            index = slice(index, index + 1)
        # now prepare the rows for being in a table
        for i, row in enumerate(values):
            values[i] = Row(self.fields,
                            row,
                            field_index=self._field_index)
        self._rows[index] = values
        self._volatile_index = min(
            self._volatile_index,
            min(index.indices(len(self._rows))[:2])
        )

    def __len__(self) -> int:
        return len(self._rows)

    def close(self) -> None:
        """Close the table file being iterated over, if open."""
        if self._file is not None:
            self._file.close()
        self._file = None

    def column_index(self, name: str) -> int:
        """Return the tuple index of the column with name *name*."""
        return self._field_index[name]

    def get_field(self, name: str) -> tsdb.Field:
        """Return the :class:`tsdb.Field` object with column name *name*."""
        return self.fields[self.column_index(name)]

    def clear(self) -> None:
        """Clear the table of all rows."""
        self._rows.clear()
        self._volatile_index = 0

    def append(self, row: tsdb.Record) -> None:
        """
        Add *row* to the end of the table.

        Args:
            row: a :class:`Row` or other iterable containing
                column values
        """
        self.extend([row])

    def extend(self, rows: Iterable[tsdb.Record]) -> None:
        """
        Add each row in *rows* to the end of the table.

        Args:
            row: an iterable of :class:`Row` or other iterables
                containing column values
        """
        for row in rows:
            if not isinstance(row, Row):
                row = Row(self.fields,
                          row,
                          field_index=self._field_index)
            self._rows.append(row)

    def update(self, index: int,
               data: tsdb.ColumnMap) -> None:
        """
        Update the row at *index* with *data*.

        Args:
            index: the 0-based index of the row in the table
            data: a mapping of column names to values for replacement

        Examples:
            >>> table.update(0, {'i-input': '...'})
        """
        if not isinstance(index, int):
            raise TypeError(type(index).__name__)
        values = list(self[index])
        for key, value in data.items():
            field_index = self._field_index[key]
            values[field_index] = value
        self[index] = Row(self.fields,
                          values,
                          field_index=self._field_index)

    def select(self, *names: str, cast: bool = True) -> Iterator[tsdb.Record]:
        """
        Select fields given by *names* from each row in the table.

        If no field names are given, all fields are returned.

        If *cast* is `False`, simple tuples of raw data are returned
        instead of :class:`Row` objects.

        Yields:
            Row
        Examples:
            >>> next(table.select())
            Row(10, 'unknown', 'formal', 'none', 1, 'S', 'It rained.', ...)
            >>> next(table.select('i-id'))
            Row(10)
            >>> next(table.select('i-id', 'i-input'))
            Row(10, 'It rained.')
            >>> next(table.select('i-id', 'i-input', cast=False))
            ('10', 'It rained.')
        """
        indices = tuple(map(self._field_index.__getitem__, names))
        fields = tuple(map(self.fields.__getitem__, indices))
        field_index = tsdb.make_field_index(fields)
        with tsdb.open(self.dir, self.name, encoding=self.encoding) as fh:
            for _, row in self._enum_rows(fh):
                data = tuple(row.data[i] for i in indices)
                if cast:
                    yield Row(fields, data, field_index=field_index)
                else:
                    yield data

    def _enum_rows(self,
                   fh: IO[str],
                   _slice: Optional[slice] = None
                   ) -> Iterator[Tuple[int, Row]]:
        """Enumerate on-disk and in-memory rows."""
        if _slice is None:
            _slice = slice(None)
        indices = range(*_slice.indices(len(self._rows)))
        fields = self.fields
        field_index = self._field_index

        file_exhausted = False
        for i, row in enumerate(self._rows):
            # always read next line until EOF to keep in sync
            if not file_exhausted:
                line: Optional[str] = None
                try:
                    line = next(fh)
                except StopIteration:
                    file_exhausted = True
            # now skip if it's not a requested index
            if i not in indices:
                continue
            # proceed only if we have a row in memory or on disk
            if row is None:
                if line is not None:
                    row = Row(fields,
                              tsdb.split(line),
                              field_index=field_index)
                else:
                    continue
            yield (i, row)

class TestSuite(tsdb.Database):
    """
    A [incr tsdb()] test suite database.

    Args:
        path: the path to the test suite's directory
        schema (dict, str): the database schema; either a mapping of
            table names to lists of :class:`Fields <Field>` or a path
            to a relations file; if not given, the relations file
            under *path* will be used
        encoding: the character encoding of the files in the test suite
    Attributes:
        schema (dict): database schema as a mapping of table names to
            lists of :class:`Field` objects
        encoding (str): character encoding used when reading and
            writing tables
    """

    def __init__(self,
                 path: Optional[util.PathLike] = None,
                 schema: Optional[tsdb.SchemaLike] = None,
                 encoding: str = 'utf-8') -> None:
        # Virtual test suites use a temporary directory
        if path is None:
            self._tempdir = tempfile.TemporaryDirectory()
            path = Path(self._tempdir.name)
        else:
            path = Path(path).expanduser()
            path.mkdir(exist_ok=True)  # can fail if path is a file

        # Ensure test suite directory has a relations file
        if not path.joinpath(tsdb.SCHEMA_FILENAME).is_file():
            if schema is None:
                raise ITSDBError(
                    '*schema* argument is required for new test suites')
            elif isinstance(schema, (str, Path)):
                schema = tsdb.read_schema(schema)
            tsdb.write_schema(path, schema)

        super().__init__(path, autocast=False, encoding=encoding)
        self._data: Dict[str, Table] = {}

    @property
    def in_transaction(self) -> bool:
        """Return `True` is there are uncommitted changes."""
        data = self._data
        return any(data[name]._in_transaction
                   for name in self.schema
                   if name in data)

    def __getitem__(self, name: str) -> Table:
        """Return a Table given its name in the schema."""
        if name not in self.schema:
            raise ITSDBError('table not defined in schema: {}'.format(name))
        # if the table is None it is invalidated; reload it
        if name not in self._data:
            self._data[name] = Table(
                self.path, name, self.schema[name], self.encoding)
        return self._data[name]

    def select_from(self,
                    name: str,
                    columns: Optional[Iterable[str]] = None,
                    cast: bool = True):
        """
        Select fields given by *names* from each row in table *name*.

        If no field names are given, all fields are returned.

        If *cast* is `False`, simple tuples of raw data are returned
        instead of :class:`Row` objects.

        Yields:
            Row
        Examples:
            >>> next(ts.select_from('item'))
            Row(10, 'unknown', 'formal', 'none', 1, 'S', 'It rained.', ...)
            >>> next(ts.select_from('item', ('i-id')))
            Row(10)
            >>> next(ts.select_from('item', ('i-id', 'i-input')))
            Row(10, 'It rained.')
            >>> next(ts.select_from('item', ('i-id', 'i-input'), cast=False))
            ('10', 'It rained.')
        """
        if not columns:
            columns = []
        return self[name].select(*columns, cast=cast)

    def reload(self) -> None:
        """Discard temporary changes and reload the database from disk."""
        for name in self.schema:
            if name in self._data:
                self._data[name]._sync_with_file()

    def commit(self) -> None:
        """
        Commit the current changes to disk.

        This method writes the current state of the test suite to
        disk. The effect is similar to using
        :func:`tsdb.write_database`, except that it also updates the
        test suite's internal bookkeeping so that it is aware that the
        current transaction is complete. It also may be more efficient
        if the only changes are adding new rows to existing tables.
        """
        for name in self.schema:
            if name not in self._data:
                continue
            table = self._data[name]
            fields = self.schema[name]
            if table._in_transaction:
                data: tsdb.Records = []
                if table._volatile_index >= table._persistent_count:
                    append = True
                    data = table[table._persistent_count:]
                else:
                    append = False
                    data = table
                tsdb.write(
                    self.path,
                    name,
                    data,
                    fields,
                    append=append,
                    encoding=self.encoding
                )
            table._sync_with_file()

    def processed_items(
            self,
            fieldmapper: Optional[FieldMapper] = None
    ) -> Iterator[interface.Response]:
        """
        Iterate over the data as :class:`Response` objects.
        """
        if fieldmapper is None:
            fieldmapper = FieldMapper()
        yield from fieldmapper.collect(self)

    def process(
            self,
            cpu: interface.Processor,
            selector: Optional[Tuple[str, str]] = None,
            source: Optional[tsdb.Database] = None,
            fieldmapper: Optional[FieldMapper] = None,
            gzip: bool = False,
            buffer_size: int = 1000,
            callback: Optional[Callable[[interface.Response], Any]] = None,
    ) -> None:
        """
        Process each item in a [incr tsdb()] test suite.

        The output rows will be flushed to disk when the number of new
        rows in a table is *buffer_size*.

        The *callback* parameter can be used, for example, to update a
        progress indicator.

        Args:
            cpu (:class:`~delphin.interface.Processor`): processor
                interface (e.g., :class:`~delphin.ace.ACEParser`)
            selector: a pair of (table_name, column_name) that specify
                the table and column used for processor input (e.g.,
                `('item', 'i-input')`)
            source (:class:`~delphin.tsdb.Database`): test suite from
                which inputs are taken; if `None`, use the current
                test suite
            fieldmapper (:class:`FieldMapper`): object for
                mapping response fields to [incr tsdb()] fields; if
                `None`, use a default mapper for the standard schema
            gzip: if `True`, compress non-empty tables with gzip
            buffer_size (int): number of output rows to hold in memory
                before flushing to disk; ignored if the test suite is all
                in-memory; if `None`, do not flush to disk
            callback: a function that is called with the response for
                each item processed; the return value is ignored
        Examples:
            >>> ts.process(ace_parser)
            >>> ts.process(ace_generator, 'result:mrs', source=ts2)
        """
        if selector is None:
            assert isinstance(cpu.task, str)
            input_table, input_column = _default_task_selectors[cpu.task]
        else:
            input_table, input_column = selector
        if (input_table not in self.schema
            or all(f.name != input_column
                   for f in self.schema[input_table])):
            raise ITSDBError('invalid table or column: {!s}, {!s}'
                             .format(input_table, input_column))
        if source is None:
            source = self
        if fieldmapper is None:
            fieldmapper = FieldMapper(source=source)
        index = tsdb.make_field_index(source.schema[input_table])

        affected = set(fieldmapper.affected_tables).intersection(self.schema)
        for name in affected:
            self[name].clear()

        key_names = [f.name for f in source.schema[input_table] if f.is_key]

        for row in source[input_table]:
            datum = row[index[input_column]]
            keys = [row[index[name]] for name in key_names]
            keys_dict = dict(zip(key_names, keys))
            response = cpu.process_item(datum, keys=keys_dict)

            logger.info(
                'Processed item {:>16}  {:>8} results'
                .format(tsdb.join(keys), len(response['results']))
            )
            if callback:
                callback(response)

            for tablename, data in fieldmapper.map(response):
                _add_row(self, tablename, data, buffer_size)

        for tablename, data in fieldmapper.cleanup():
            _add_row(self, tablename, data, buffer_size)

        tsdb.write_database(self, self.path, gzip=gzip)

def _add_row(ts: TestSuite,
             name: str,
             data: Dict,
             buffer_size: int) -> None:
    """
    Prepare and append a Row into its Table; flush to disk if necessary.
    """
    fields = ts.schema[name]
    # remove any keys that aren't relation fields
    for invalid_key in set(data).difference([f.name for f in fields]):
        del data[invalid_key]

    ts[name].append(tsdb.make_record(data, fields))

    num_changes = 0
    for _name in ts:
        table = ts[_name]
        num_changes += len(table) - table._persistent_count
    if num_changes > buffer_size:
        ts.commit()

##############################################################################
# Non-class (i.e. static) functions

Match = Tuple[tsdb.Value, Rows, Rows]
_Matched = Tuple[List[Row], List[Row]]

def match_rows(rows1: Rows,
               rows2: Rows,
               key: Union[str, int],
               sort_keys: bool = True) -> Iterator[Match]:
    """
    Yield triples of `(value, left_rows, right_rows)` where
    `left_rows` and `right_rows` are lists of rows that share the same
    column value for *key*. This means that both *rows1* and *rows2*
    must have a column with the same name *key*.

    .. warning::

       Both *rows1* and *rows2* will exist in memory for this
       operation, so it is not recommended for very large tables on
       low-memory systems.

    Args:
        rows1: a :class:`Table` or list of :class:`Row` objects
        rows2: a :class:`Table` or list of :class:`Row` objects
        key (str, int): the column name or index on which to match
        sort_keys (bool): if `True`, yield matching rows sorted by the
            matched key instead of the original order
    Yields:
        tuple: a triple containing the matched value for *key*, the
            list of any matching rows from *rows1*, and the list of
            any matching rows from *rows2*
    """
    matched: Dict[tsdb.Value, _Matched] = collections.OrderedDict()
    for i, rows in enumerate([rows1, rows2]):
        for row in rows:
            val: tsdb.Value = row[key]
            try:
                data = matched[val]
            except KeyError:
                matched[val] = ([], [])
                data = matched[val]
            data[i].append(row)
    vals = list(matched.keys())
    if sort_keys:
        vals = sorted(vals, key=util.safe_int)
    for val in vals:
        left, right = matched[val]
        yield (val, left, right)


import time
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.animation as animation
#define constants
m1 = 10
m2 = 1
L1 = 1
L2 = 2

g=9.81
dt = 0.01
tmax = 30
t=0
times=np.linspace(0, tmax, int(tmax/dt))

theta1 = np.pi/1
theta2 = 0.1
y1 = 0.0 #omega1
y2 = 0.0

def sin(x):
    return(np.sin(x))

def cos(x):
    return(np.cos(x))

def euler(theta1, theta2, y1, y2):
    theta1 = theta1 + dt*y1
    theta2 = theta2 + dt*y2

    c=cos(theta1-theta2)
    s=sin(theta1-theta2)

    y1 = y1 + dt * (( (m2*g*sin(theta2)*c) - (m2*s*(L1*y1**2*c + L2*y2**2)) - ((m1+m2)*g*sin(theta1))) / (L1* (m1 + m2*s**2)))
    y2 = y2 + dt * (( (m1+m2)*(L1*y1**2*s - g*sin(theta2) + g*sin(theta1)*c) + m2*L2*y2**2*s*c) / ( L2*(m1 + m2*s**2) ))
    return (theta1, theta2, y1, y2)

theta1_vals = np.zeros(int(tmax/dt))
omega1_vals = np.zeros(int(tmax/dt))
theta2_vals = np.zeros(int(tmax/dt))
omega2_vals = np.zeros(int(tmax/dt))

for i in range(int(tmax/dt)):
    (theta1, theta2, y1, y2) = euler(theta1, theta2, y1, y2)
    theta1_vals[i] = theta1
    theta2_vals[i] = theta2
    omega1_vals[i] = y1
    omega2_vals[i] = y2
    t += dt

#convert to cartesian coords
x1 = L1*sin(theta1_vals)
y1 = -L1*cos(theta1_vals)
x2 = x1+L2*sin(theta2_vals)
y2 = y1-L2*cos(theta2_vals)

fig = plt.figure()
ax = fig.add_subplot(111, autoscale_on=False, xlim=(-1.2*(L1+L2), 1.2*(L1+L2)), ylim=(-1.2*(L1+L2), 1.2*(L1+L2)))
ax.set_aspect('equal')
ax.grid()

line, = ax.plot([], [], 'o-', lw=2)
time_template = 'time = %.1fs'
time_text = ax.text(0.05, 0.9, '', transform=ax.transAxes)

def init():
    line.set_data([], [])
    time_text.set_text('')
    return line, time_text

def animate(i):
    thisx = [0, x1[i], x2[i]]
    thisy = [0, y1[i], y2[i]]

    line.set_data(thisx, thisy)
    time_text.set_text(time_template % (i*dt))
    return line, time_text

ani = animation.FuncAnimation(fig, animate, range(1, len(x1)),
                              interval=dt*1, blit=True, init_func=init)
plt.show()


# import sys
#
# sys.stdin = open('input.txt', 'r')
#
# data = list(map(int, input().split()))

data = [
    [9, 20, 2, 18, 11],
    [19, 1, 25, 3, 21],
    [8, 24, 10, 17, 7],
    [15, 4, 16, 5, 6],
    [12, 13, 22, 23, 14]
]

length = len(data)
selection_l = []
for _ in range(length ** 2):
    min = length ** 2
    for y in range(length):
        for x in range(len(data[y])):
            if data[y][x] <= min:
                min = data[y][x]
                Y, X = y, x
    selection_l.append(min)
    data[Y][X] = length ** 2 + 1

dx = [1, 0, -1, 0]
dy = [0, 1, 0, -1]

array_l = [[0 for _ in range(length)] for _ in range(length)]

i,j, delta = 0, 0, 0
for num in selection_l:
    array_l[i][j] = num
    if j + dx[delta % 4] >= length or i + dy[delta % 4] >= length or array_l[i + dy[delta % 4]][j + dx[delta % 4]] != 0:
        delta += 1
    j += dx[delta % 4]
    i += dy[delta % 4]
    print(i, j, delta)

print(array_l)


from os import makedirs, chdir, getcwd, system
from sys import stderr
from pathlib import Path
from shutil import rmtree
from pkg_resources import resource_filename
from zipfile import ZipFile
from .template import Adjustments

DEFAULT_GIT_CONFIG = """
[user]
        email = your@email.com
        name = Your Name
"""

class Project:
    def __init__(self, name: str, project_dir: str):
        self._project_dir = project_dir
        self._name = name or Path(project_dir).name

    def create(self, force=False):
        proj_dir = self._project_dir
        print(f"Creating project {self._name} at {proj_dir}")
        self._make_dir(proj_dir, force)

        template_zip = resource_filename(__name__, "template.zip")
        with ZipFile(template_zip) as zip_file:
            zip_file.extractall(proj_dir)
        adj = Adjustments(proj_dir, self._name)
        adj.run()
        #  self._git_init()

    @staticmethod
    def _run(command):
        rc = system(command)
        if rc != 0:
            exit(rc)

    def _git_init(self):
        abs_proj_dir = Path(self._project_dir).resolve()
        cwd = getcwd()
        chdir(self._project_dir)
        self._run("git init")
        git_config = Path("~", ".gitconfig").expanduser()
        if not git_config.exists():
            with open(Path(abs_proj_dir, ".git", "config"), 'a') as git_config:
                git_config.write(DEFAULT_GIT_CONFIG)
        self._run("git add .")
        self._run("git commit -a -m 'Initial commit'")
        chdir(cwd)

    @staticmethod
    def _make_dir(path, force):
        try:
            makedirs(path)
        except FileExistsError:
            if not force:
                print(f"Directory {path} already exists", file=stderr)
                exit(2)
            else:
                rmtree(path)