# coding: utf-8
"""
CALS Parser Implementation
==========================

This module can parse the tables (``table`` elements) of a CALS file.

Specifications and examples:

- The CALS DTD is available online in the OASIS web site:
  `CALS Table Model Document Type Definition <https://www.oasis-open.org/specs/a502.htm>`_.

- An example of CALS table is available in Wikipedia:
  `CALS Table Model <https://en.wikipedia.org/wiki/CALS_Table_Model>`_

The main elements of a CALS table are:

*   ``table``: a table can contains one or several ``tgroup``.

    *   ``titles``: table titles (*not supported by the CALS parser*)

    *   ``tgroup``: a portion of table

        *   ``colspec``: column specifications

        *   ``spanspec``: spanning specifications (*not supported by the CALS parser*)

        *   ``thead``: table header

            *   ``colspec``: header column specifications (*not supported by the CALS parser*)

            *   ``row``: table row (see ``tbody``)

        *   ``tfoot``: table footer

            *   ``colspec``: footer column specifications (*not supported by the CALS parser*)

            *   ``row``: table row (see ``tbody``)

        *   ``tbody``: table body

            *   ``row``: table row

                *   ``entry``: table entry which contains paragraphs

                *   ``entrytbl``: table entry which contains a table (*not supported by the CALS parser*)

An example of CALS table is available in Wikipedia:
`CALS Table Model <https://en.wikipedia.org/wiki/CALS_Table_Model>`_

.. versionadded:: 0.5.0
"""
import re

from lxml import etree

from benker.box import Box
from benker.common.lxml_iterwalk import iterwalk
from benker.common.lxml_qname import QName
from benker.parsers.base_parser import BaseParser
from benker.parsers.cals.frame_styles import BORDER_NONE
from benker.parsers.cals.frame_styles import BORDER_SOLID
from benker.parsers.cals.frame_styles import get_frame_styles
from benker.table import Table
from benker.units import convert_value
from benker.units import parse_width

# noinspection PyProtectedMember
#: Element Type
ElementType = etree._Element

class CalsParser(BaseParser):
    """
    CALS tables parser
    """

    def __init__(self, builder, cals_ns=None, width_unit="mm", **options):
        """
        Construct a parser

        :type  builder: benker.builders.base_builder.BaseBuilder
        :param builder:
            Builder used by this parser to instantiate :class:`~benker.table.Table` objects.

        :param str cals_ns:
            Namespace to use for CALS elements and attributes.
            Set ``None`` (or "") if you don't use namespace in your XML.

        :param str width_unit:
            Unit to use for table/column widths.
            Possible values are: 'cm', 'dm', 'ft', 'in', 'm', 'mm', 'pc', 'pt', 'px'.

        :keyword str options: Extra conversion options.
            See :meth:`~benker.converters.base_converter.BaseConverter.convert_file`
            to have a list of all possible options.

        .. versionchanged:: 0.5.1
           Add the options *width_unit*.
        """
        self.cals_ns = cals_ns
        self.width_unit = width_unit
        super(CalsParser, self).__init__(builder, **options)

    def get_cals_qname(self, name):
        return QName(self.cals_ns, name)

    def transform_tables(self, tree):
        if self.cals_ns:
            nodes = tree.xpath("//cals:table", namespaces={"cals": self.cals_ns})
        else:
            nodes = tree.xpath("//table")
        for node in nodes:
            table = self.parse_table(node)
            table_elem = self.builder.generate_table_tree(table)
            parent = node.getparent()
            index = parent.index(node)
            parent.insert(index, table_elem)
            table_elem.tail = node.tail
            parent.remove(node)

    # noinspection PyPep8Naming
    def parse_table(self, cals_table):
        """
        Convert a ``<table>`` CALS element into table object.

        :type  cals_table: ElementType
        :param cals_table: CALS element.

        :rtype: benker.table.Table
        :return: Table.
        """
        state = self._state
        state.reset()

        # -- CALS elements
        cals = self.get_cals_qname

        table = cals("table").text
        # titles = cals("titles").text  # not supported
        tgroup = cals("tgroup").text
        colspec = cals("colspec").text
        thead = cals("thead").text
        tfoot = cals("tfoot").text
        tbody = cals("tbody").text
        row = cals("row").text
        # entrytbl = cals("entrytbl").text  # not supported
        entry = cals("entry").text

        elements = {table, tgroup, colspec, thead, tfoot, tbody, row, entry}
        context = iterwalk(cals_table, events=("start", "end"), tag=elements)

        depth = 0
        for action, elem in context:
            elem_tag = elem.tag
            if elem_tag == table:
                if action == "start":
                    depth += 1
                else:
                    depth -= 1
            if depth > 1:
                # .. note:: context.skip_subtree() is not available for all version of lxml
                # This <TBL> element is inside the table.
                # It will be handled separately in another call to transform_tables()
                continue
            if action == "start":
                # tags sorted by frequency:
                if elem_tag == entry:
                    state.next_col()
                    self.parse_cals_entry(elem)

                elif elem_tag == row:
                    state.next_row()
                    self.parse_cals_row(elem)

                elif elem_tag in {tbody, tfoot, thead}:
                    # everything is done in parse_fmx_row()
                    pass

                elif elem_tag == colspec:
                    state.next_col()
                    self.parse_cals_colspec(elem)

                elif elem_tag == tgroup:
                    self.parse_cals_tgroup(elem)

                elif elem_tag == table:
                    self.parse_cals_table(elem)

                else:
                    raise NotImplementedError(elem_tag)
            else:
                if elem_tag in {row}:
                    bounding_box = Box(1, state.row_pos, len(state.table.cols), state.row_pos)
                    state.table.fill_missing(bounding_box, None, nature=state.row.nature)
                elif elem_tag == table:
                    state.table.fill_missing(state.table.bounding_box, None)

        return state.table

    def setup_table(self, styles=None, nature=None):
        table = Table(styles=styles, nature=nature)
        self._state.table = table
        return self._state

    def parse_cals_table(self, cals_table):
        """
        Parse a CALS ``table`` element.

        :type  cals_table: ElementType
        :param cals_table: CALS table Element.

        :return: State of the parser (for debug purpose).

        .. versionchanged:: 0.5.1
           Add support for the ``@cals:width`` attribute (table width).
        """
        cals = self.get_cals_qname
        styles = {}
        nature = None

        # -- attribute @cals:frame
        frame = cals_table.attrib.get(cals("frame"))
        styles.update(get_frame_styles(frame))

        # -- attribute @cals:colsep
        colsep = cals_table.attrib.get(cals("colsep"))
        colsep_map = {"0": BORDER_NONE, "1": BORDER_SOLID}
        if colsep in colsep_map:
            styles["x-cell-border-right"] = colsep_map[colsep]

        # -- attribute @cals:rowsep
        rowsep = cals_table.attrib.get(cals("rowsep"))
        rowsep_map = {"0": BORDER_NONE, "1": BORDER_SOLID}
        if rowsep in rowsep_map:
            styles["x-cell-border-bottom"] = rowsep_map[rowsep]

        # -- attribute @cals:orient
        orient = cals_table.attrib.get(cals("orient"))
        orient_map = {"land": "landscape", "port": "portrait"}
        if orient in orient_map:
            styles["x-sect-orient"] = orient_map[orient]

        # -- attribute @cals:pgwide
        pgwide = cals_table.attrib.get(cals("pgwide"))
        pgwide_map = {"0": "2", "1": "1"}
        if pgwide in pgwide_map:
            styles["x-sect-cols"] = pgwide_map[pgwide]

        # -- attribute @cals:bgcolor
        bgcolor = cals_table.attrib.get(cals("bgcolor"))
        if bgcolor:
            styles["background-color"] = bgcolor

        # -- attribute @cals:tabstyle
        tabstyle = cals_table.attrib.get(cals("tabstyle"))
        if tabstyle:
            nature = tabstyle

        # -- attribute @cals:tabstyle
        width = cals_table.attrib.get(cals("width"))
        if width:
            width, unit = parse_width(width)
            value = convert_value(width, unit, self.width_unit)
            styles["width"] = u"{value:0.2f}{unit}".format(value=value, unit=self.width_unit)

        return self.setup_table(styles, nature)

    def parse_cals_tgroup(self, cals_tgroup):
        cals = self.get_cals_qname
        styles = {}
        nature = self._state.table.nature

        # -- attribute @cals:cols => ignored (*table.cols*)

        # -- attribute @cals:colsep
        colsep = cals_tgroup.attrib.get(cals("colsep"))
        colsep_map = {"0": BORDER_NONE, "1": BORDER_SOLID}
        if colsep in colsep_map:
            styles["x-cell-border-right"] = colsep_map[colsep]

        # -- attribute @cals:rowsep
        rowsep = cals_tgroup.attrib.get(cals("rowsep"))
        rowsep_map = {"0": BORDER_NONE, "1": BORDER_SOLID}
        if rowsep in rowsep_map:
            styles["x-cell-border-bottom"] = rowsep_map[rowsep]

        # -- attribute @cals:tgroupstyle
        tgroupstyle = cals_tgroup.attrib.get(cals("tgroupstyle"))
        if tgroupstyle:
            nature = self._state.table.nature
            if nature:
                parts = nature.split(" ")
                nature = " ".join(parts[:-1] + [tgroupstyle])
            else:
                nature = tgroupstyle

        # -- Override the table defaults
        state = self._state
        table = state.table
        table.styles.update(styles)
        table.nature = nature

        return state  # mainly for unit test

    def parse_cals_row(self, cals_row):
        """
        Parse a ``row`` element which contains ``entry`` elements.

        This element may be in a ``BLK```

        :type  cals_row: ElementType
        :param cals_row: table row

        .. versionchanged:: 0.5.1
           The "vertical-align" style is built from the ``@cals:valign`` attribute.
        """
        cals = self.get_cals_qname
        styles = {}
        nature = None  # overridden below if parent's element exists

        cals_parent = cals_row.getparent()  # type:  ElementType
        if cals_parent is not None:
            # -- nature of the row
            tag_map = {"thead": "header", "tfoot": "footer", "tbody": "body"}
            localname = QName(cals_parent.tag).localname
            nature = tag_map[localname]

            # -- attribute @cals:valign
            valign = cals_parent.attrib.get(cals("valign"))
            valign_map = {'top': 'top', 'middle': 'middle', 'bottom': 'bottom'}
            if valign in valign_map:
                styles["vertical-align"] = valign_map[valign]

        # -- attribute @cals:valign
        valign = cals_row.attrib.get(cals("valign"))
        valign_map = {'top': 'top', 'middle': 'middle', 'bottom': 'bottom'}
        if valign in valign_map:
            # overrides parent's value
            styles["vertical-align"] = valign_map[valign]

        # -- attribute @cals:rowsep
        rowsep = cals_row.attrib.get(cals("rowsep"))
        rowsep_map = {"0": BORDER_NONE, "1": BORDER_SOLID}
        if rowsep in rowsep_map:
            styles["border-bottom"] = rowsep_map[rowsep]

        # -- attribute @cals:bgcolor
        bgcolor = cals_row.attrib.get(cals("bgcolor"))
        if bgcolor:
            styles["background-color"] = bgcolor

        # -- attribute @cals:rowstyle (extension)
        rowstyle = cals_row.attrib.get(cals("rowstyle"))
        if rowstyle:
            # overrides the previously calculated @cals:rowstyle attribute
            styles["rowstyle"] = rowstyle

        # -- Create a row
        state = self._state
        state.row = state.table.rows[state.row_pos]
        state.row.nature = nature
        state.row.styles = styles

        return state  # mainly for unit test

    def parse_cals_entry(self, cals_entry):
        """
        Parse a ``entry`` element.

        :type  cals_entry: ElementType
        :param cals_entry: table entry

        .. versionchanged:: 0.5.1
           The "vertical-align" style is built from the ``@cals:valign`` attribute.

        .. versionchanged:: 0.5.2
           Add support for the ``@cals:cellstyle`` attribute (extension).
           This attribute is required for two-way conversion of Formex tables to CALS and vice versa.
           If the ``CELL/@TYPE`` and the ``ROW/@TYPE`` are different, we add a specific "cellstyle" style.
           This style will keep the ``CELL/@TYPE`` value.

        .. versionchanged:: 0.5.3
           Improved empty cells detection for Formex4 conversion (``<IE/>`` tag management).
        """
        cals = self.get_cals_qname
        styles = {}

        # -- attribute @cals:colsep
        colsep = cals_entry.attrib.get(cals("colsep"))
        colsep_map = {"0": BORDER_NONE, "1": BORDER_SOLID}
        if colsep in colsep_map:
            styles["border-right"] = colsep_map[colsep]

        # -- attribute @cals:rowsep
        rowsep = cals_entry.attrib.get(cals("rowsep"))
        rowsep_map = {"0": BORDER_NONE, "1": BORDER_SOLID}
        if rowsep in rowsep_map:
            styles["border-bottom"] = rowsep_map[rowsep]

        # -- attribute @cals:bgcolor
        bgcolor = cals_entry.attrib.get(cals("bgcolor"))
        if bgcolor:
            styles["background-color"] = bgcolor

        # -- attributes @cals:namest and @cals:nameend
        name_start = cals_entry.attrib.get(cals("namest"), str(self._state.col_pos))
        name_end = cals_entry.attrib.get(cals("nameend"), str(self._state.col_pos))
        col_start = int(re.findall(r"\d+", name_start)[0])
        col_end = int(re.findall(r"\d+", name_end)[0])
        width = col_end - col_start + 1

        # -- attribute @cals:morerows
        morerows = cals_entry.attrib.get(cals("morerows"), "0")
        height = int(morerows) + 1

        # -- attribute @cals:valign
        valign = cals_entry.attrib.get(cals("valign"))
        valign_map = {'top': 'top', 'middle': 'middle', 'bottom': 'bottom'}
        if valign in valign_map:
            # overrides parent's value
            styles["vertical-align"] = valign_map[valign]

        # -- attribute @cals:rowsep
        align = cals_entry.attrib.get(cals("align"))
        align_map = {"left": "left", "right": "right", "center": "center", "justify": "justify", "char": "left"}
        if align in align_map:
            styles["align"] = align_map[align]

        # -- attribute @cals:bgcolor
        cellstyle = cals_entry.attrib.get(cals("cellstyle"))
        if cellstyle:
            styles["cellstyle"] = cellstyle

        # todo: calculate the ``@rotate`` attribute.

        # -- Create a entry
        text = [cals_entry.text] if cals_entry.text else []
        content = text + cals_entry.getchildren()

        # The detection of empty cells is used when converting
        # to the Formex4 format in order to insert an empty tag ``<IE/>``.
        # see: https://github.com/laurent-laporte-pro/benker/issues/13
        #
        # When parsing a CALS table, we will consider a cell to be empty when it contains no text
        # or child tag. As we do not know the elements contained in an ``<entry>`` element,
        # we cannot know precisely if it is an empty content or not (eg. empty paragraph or BR).
        if not content:
            styles["x-cell-empty"] = "true"

        nature = self._state.row.nature
        self._state.row.insert_cell(content, width=width, height=height, styles=styles, nature=nature)

        return self._state  # mainly for unit test

    def parse_cals_colspec(self, cals_colspec):
        """
        Parse a CALS-like ``colspec`` element.

        For instance:

        .. code-block:: xml

           <colspec
             colname="c1"
             colnum="1"
             colsep="1"
             rowsep="1"
             colwidth="30mm"
             align="center"/>

        :type  cals_colspec: ElementType
        :param cals_colspec: CALS-like ``colspec`` element.
        """
        cals_parent = cals_colspec.getparent()
        if cals_parent is not None:  # pragma: no cover
            localname = QName(cals_parent).localname
            if localname not in {"table", "tgroup"}:
                raise NotImplementedError("colspec in {} not supported".format(localname))

        cals = self.get_cals_qname
        styles = {}

        # -- attribute @cals:colname is ignored
        # -- attribute @cals:char is ignored
        # -- attribute @cals:charoff is ignored

        # -- attribute @cals:colnum
        colnum = cals_colspec.attrib.get(cals("colnum"))
        colnum = int(colnum) if colnum else self._state.col_pos

        # -- attribute @cals:colsep
        colsep = cals_colspec.attrib.get(cals("colsep"))
        colsep_map = {"0": BORDER_NONE, "1": BORDER_SOLID}
        if colsep in colsep_map:
            styles["border-right"] = colsep_map[colsep]

        # -- attribute @cals:rowsep
        rowsep = cals_colspec.attrib.get(cals("rowsep"))
        rowsep_map = {"0": BORDER_NONE, "1": BORDER_SOLID}
        if rowsep in rowsep_map:
            styles["border-bottom"] = rowsep_map[rowsep]

        # -- attribute @cals:rowsep
        colwidth = cals_colspec.attrib.get(cals("colwidth"))
        if colwidth:
            styles["width"] = colwidth

        # -- attribute @cals:rowsep
        align = cals_colspec.attrib.get(cals("align"))
        align_map = {"left": "left", "right": "right", "center": "center", "justify": "justify", "char": "left"}
        if align in align_map:
            styles["align"] = align_map[align]

        state = self._state
        state.col = state.table.cols[colnum]
        state.col.styles.update(styles)

        return state


"""
Main module what launch the game.
Contains argv flag to set Units health.
"""
import sys
from unit import Unit
from battle import Battle

health = 100

if len(sys.argv) > 1:
    try:
        health = int(sys.argv[1])
    except ValueError:
        print('Wrong flag value')
        sys.exit(1)

# Creating Units
computer = Unit(health=health, heal_boost=True)
human = Unit(health=health, name='Human')

# Creating battle with created Units
battle = Battle(computer, human)

print('Hello, we have two players.\n\n %s and\n %s' % (computer, human))

input('Press any key to start a fight!\n')

# Starting fight
battle.fight()

input('\nPress enter to exit the game...')


from djitellopy import tello

drone = tello.Tello()
drone.connect()

FPS = "Tello.FPS_30"
RESOLUTION = "Tello.RESOLUTION_720P"
BITRATE = "Tello.BITRATE_5MBPS"

print("SDK VERSION")
drone.query_sdk_version()

print("serial number")
drone.query_serial_number()

#print("Setting connection settings", FPS, RESOLUTION, BITRATE )
drone.set_video_fps(tello.Tello.FPS_30)
drone.set_video_resolution(tello.Tello.RESOLUTION_720P)
drone.set_video_bitrate(tello.Tello.BITRATE_5MBPS)

#drone.turn_motor_on()


# -*- coding: utf-8 -*-
"""
@author: Achilleas Karagiannidis
"""
# import needed libraries to create the map with the pointers
import matplotlib.pyplot as plt
import contextily as ctx
import pandas as pd

# Function to create the map, and appoint the pointers on it.
def poisOnMap(poiNum, x, y):
    # Create a Matplotlib figure
    fig, ax = plt.subplots(figsize=(10,10))

    # Set the limits to cover the POIs
    ax.set_xlim(min(x)-0.001, max(x)+0.001)
    ax.set_ylim(min(y)-0.001, max(y)+0.001)

    # Add the OpenStreetMap base map
    ctx.add_basemap(ax, crs='epsg:4326', source=ctx.providers.OpenStreetMap.Mapnik)

    # Add a marker for each POI
    for i in range(len(poiNum)):
        ax.plot(x[i], y[i], 'ro', markersize=10, label=f'POI {poiNum[i]}')

    ax.legend()
    plt.show()

# Read data from the XLSX file
data = pd.read_excel('TELIKO.xlsx')
#print(data)

# Extract columns from the data
poiNum = data['poinnum'].tolist()
x = data['x'].tolist()
y = data['y'].tolist()

# Call the function with the updated data
poisOnMap(poiNum, x, y)


class QuizBrain:
    # initializing to pass over list and create a value for question_number
    def __init__(self, q_list):
        self.question_number = 0
        self.score = 0
        self.question_list = q_list

    def still_has_questions(self):
        # getting the length of the question_data list/dictionary and returning 12 because there are 12 questions
        # returning True or False to determine if user will go to the next question
        # same thing as writing an if/else statement to return True or False
        return self.question_number < len(self.question_list)

    def next_question(self):
        # getting hold of current question from question_list and getting hold of the current question number
        current_question = self.question_list[self.question_number]
        # making question number increase by 1 for each question
        self.question_number += 1
        # inputting Question number with current question passing in text from Question class and asking True or False
        user_input = input(f"Q.{self.question_number}: {current_question.text} (True/False) ")
        # identifying check_answer and inputting user_input and current_question.answer
        # because check_answer was called here, don't have to put it in main.py
        self.check_answer(user_input, current_question.answer)

    # receiving user input and correct_answer(current_question.answer) once we identified it in the next_question method
    # and use it as a parameter
    def check_answer(self, user_input, correct_answer):
        if user_input.lower() == correct_answer.lower():
            # calculating the score to add 1 if user gets question right
            self.score += 1
            print("You got it right!")
        else:
            print("Sorry, that's wrong!")
        print(f"The correct answer was {correct_answer}.")
        print(f"Your current score is: {self.score}/{self.question_number}")
        print("\n")


"""Turn a string of 24h time into words.

You can trust that you'll be given a valid string (it will
always have a two-digit hour 00-23, and a two-digit minute
00-59). Hours 0-11 are am, and hours 12-23 are pm.

Handle noon and midnight specially:

    >>> time_word("00:00")
    'midnight'

    >>> time_word("12:00")
    'noon'

Otherwise, covert times to text:

    >>> time_word("01:00")
    "one o'clock am"

    >>> time_word("06:01")
    'six oh one am'

    >>> time_word("06:10")
    'six ten am'

    >>> time_word("06:18")
    'six eighteen am'

    >>> time_word("06:30")
    'six thirty am'

    >>> time_word("10:34")
    'ten thirty four am'

Don't forget to handle early morning properly:

    >>> time_word("00:12")
    'twelve twelve am'

For times after noon, add 'pm'

    >>> time_word("12:09")
    'twelve oh nine pm'

    >>> time_word("23:23")
    'eleven twenty three pm'

By Joel Burton <joel@joelburton.com>.
"""

def time_word(time):
    """Convert time to text."""

    if time == "00:00":
        return('midnight')
    if time == "12:00":
        return("noon")

    time = time.split(":")
    hours = time[0]
    minutes = time[1]
    written_time = ""

    if hours == "00" or hours == "12":
        written_time = "twelve"

    elif hours == "01" or hours == "13":
        written_time = "one"

    elif hours == "02" or hours == "14":
        written_time = "two"

    elif hours == "03" or hours == "15":
        written_time = "three"

    elif hours == "04" or hours == "16":
        written_time = "four"

    elif hours == "05" or hours == "17":
        written_time = "five"

    elif hours == "06" or hours == "18":
        written_time = "six"

    elif hours == "07" or hours == "19":
        written_time = "seven"

    elif hours == "08" or hours == "20":
        written_time = "eight"

    elif hours == "09" or hours == "21":
        written_time = "nine"

    elif hours == "10" or hours == "22":
        written_time = "ten"

    elif hours == "11" or hours == "23":
        written_time = "eleven"

    numbers = { "20":"twenty", "30":"thirty", "40":"fourty", "50":"fifty",
        "1":"one", "2":"two","3":"three","4":"four","5":"five","6":"six","7":"seven",
        "8":"eight","9":"nine", "10":"ten", "11":"eleven","12":"twelve","13":"thirteen",
        "14":"fourteen","15":"fifteen","16":"sixteen","17":"seventeen","18":"eighteen",
        "19":"nineteen"}
    if minutes == "00":
        written_time += " o'clock"
    elif minutes[0] == "0":
        written_time += " oh " + numbers[minutes[1]]
    elif minutes[0] == "1":
        written_time += " " + numbers[minutes]
    else:
        n = int(minutes[0]) * 10
        if minutes[1] == "0":
            written_time += " " + numbers[str(n)]
        else:
            written_time += " " + numbers[str(n)] + " " + numbers[minutes[1]]

    if int(hours) < 12:
        written_time += " am"
    else:
        written_time += " pm"

    return written_time

if __name__ == '__main__':
    import doctest
    if doctest.testmod().failed == 0:
        print("\n*** ALL TEST PASSED. YOU'RE A TIME WIZARD!\n")


from functools import reduce

names = ['Michael', 'Bob', 'Tracy']
for name in names:
	print(name)

sum = 0
for x in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]:
	sum = sum + x
print(sum)

sum = reduce((lambda x, y: x + y), [1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
print(sum)

sum = 0
for x in range(101):
	sum = sum + x
print(sum)

sum = 0
n = 99
while n > 0:
	sum = sum + n
	n = n - 2
print(sum)

L = ['Bart', 'Lisa', 'Adam']
for name in L:
	print('Hello, %s' % name)

n = 1
while n <= 100:
	if n > 10:
		break
	print(n)
	n = n + 1
print('End')

n = 0
while n < 10:
	n = n + 1
	if n % 2 == 0:
		continue
	print(n)

L = ['a', 'a', 'b', 'a', 'b', 'c']
L.sort()
print(L)

s = set()
for x in L:
	s.add(x)

print(s)


# -*- coding: utf-8 -*-
import torch.nn as nn
import torch.nn.functional as F

class TFSamepaddingLayer(nn.Module):
    """To align with tf `same` padding.

    Putting this before any conv layer that need padding
    Assuming kernel has Height == Width for simplicity
    """

    def __init__(self, ksize, stride):
        super(TFSamepaddingLayer, self).__init__()
        self.ksize = ksize
        self.stride = stride

    def forward(self, x):
        if x.shape[2] % self.stride == 0:
            pad = max(self.ksize - self.stride, 0)
        else:
            pad = max(self.ksize - (x.shape[2] % self.stride), 0)

        if pad % 2 == 0:
            pad_val = pad // 2
            padding = (pad_val, pad_val, pad_val, pad_val)
        else:
            pad_val_start = pad // 2
            pad_val_end = pad - pad_val_start
            padding = (pad_val_start, pad_val_end, pad_val_start, pad_val_end)
        # print(x.shape, padding)
        x = F.pad(x, padding, "constant", 0)
        # print(x.shape)
        return x


from config import client

data = {"user_id": 1, "travel_id": 2, "action": "add", "value": 12}
incorrect_data_var_names = {
    "user_id": 1,
    "travel_id": 2,
    "act2ion": "add",
    "val2ue": 12,
}

log_url = "/log"

def test_log_with_correct_data():
    response = client.post(log_url, json=data)
    assert response.status_code == 200
    assert response.json() == {"OK": "Mensagem enviada"}

def test_log_with_incorrect_data_var_names():
    response = client.post(log_url, json=incorrect_data_var_names)
    assert response.status_code == 422
    assert response.json() == {
        "detail": [
            {
                "loc": ["body", "action"],
                "msg": "field required",
                "type": "value_error.missing",
            },
            {
                "loc": ["body", "value"],
                "msg": "field required",
                "type": "value_error.missing",
            },
        ]
    }


import sys

input = sys.stdin.readline
n = int(input())
k = 1

while n > 0 :
    n -= k
    k += 1
if n <= 0:
    k -= 1
    n += k
# print(n)
# print(k)

if k%2:
    print("{}/{}".format(k-n+1, n))
else:
    print("{}/{}".format(n,k-n+1))


import re
from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.common.by import By
from selenium.webdriver import ActionChains
from bs4 import BeautifulSoup
import argparse
import json
import Car_and_Seller
from Cars_DBM import Cars_DBM
import logging
import config
from API import update_type_in_db

# Create logger
logger = logging.getLogger('cars')
logger.setLevel(logging.DEBUG)

# Create Formatter
formatter = logging.Formatter(
    '%(asctime)s-%(levelname)s-FILE:%(filename)s-FUNC:%(funcName)s-LINE:%(lineno)d-%(message)s')

# create a file handler and add it to logger
file_handler = logging.FileHandler('carsdotcom.log')
file_handler.setLevel(logging.DEBUG)
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)

# create a file handler for stdoutput
stream_handler = logging.FileHandler('stdout.log')
stream_handler.setLevel(logging.INFO)
stream_handler.setFormatter(formatter)
logger.addHandler(stream_handler)

def start_parser():
    """
    Starts parser to get inputs from user
    :return: arguments provided by user
    """
    body_choices = ['any', 'cargo_van', 'coupe', 'convertible', 'hatchback', 'minivan', 'passenger_van', 'pickup_truck',
                    'suv', 'sedan', 'wagon']
    radius_choices = ['10', '20', '30', '40', '50', '75', '100', '150', '200', '250', '500', 'any']
    condition_choices = ['all', 'new', 'new_cpo', 'used', 'cpo']

    parser = argparse.ArgumentParser(description='Retrieves data from cars.com')

    parser.add_argument('--body', help='Add body type to search', choices=body_choices, default='any')
    parser.add_argument('--cond', help='Add condition to search', choices=condition_choices, default='all')
    parser.add_argument('--radius', help='Add radius to search', choices=radius_choices, default=None)
    parser.add_argument('--price_min', type=int, help='Add min price (in thousands of USD) to search (1 to 2500)',
                        choices=range(0, 2501), default=None, metavar='[1-2500]')
    parser.add_argument('--price_max', type=int, help='Add max price (in thousands of USD) to search (1 to 2500)',
                        choices=range(0, 2501), default=None, metavar='[1-2500]')
    parser.add_argument('--year_min', type=int, help='Add min year of car to search (1900 to 2022)',
                        choices=range(1900, 2023), default=None, metavar='[1900-2022]')
    parser.add_argument('--year_max', type=int, help='Add max year of car to search (1900 to 2022)',
                        choices=range(1900, 2023), default=None, metavar='[1900-2022]')
    parser.add_argument('--ads_max', type=int, help='Add max number of ads to scrape')

    args = parser.parse_args()

    if args.price_min is not None and args.price_max is not None and args.price_min > args.price_max:
        parser.error('max price must be higher or equal to min price.')
        logger.info(f'max price must be higher or equal to min price.')
        logger.error(
            f'User inputted minimum price higher than maximum price: Min={args.price_min} Max={args.price_max}')

    if args.year_min is not None and args.year_max is not None and args.year_min > args.year_max:
        parser.error('max year must be higher or equal to min year.')
        logger.info('max year must be higher or equal to min year.')
        logger.error(
            f'User inputted minimum year higher than maximum year: Min={args.year_min} Max={args.year_max}')

    if args.ads_max is not None and args.ads_max < 1:
        parser.error('ads_max must be a positive integer greater than zero.')
        logger.info('ads_max must be a positive integer greater than zero.')
        logger.error(
            f'User inputted an invalid integer as max ads: {args.ads_max}')

    return args

def get_url():
    """
    Constructs the cars.com URL based on user input and the max number of ads to look through
    :return: string with url and in with number of ads
    """
    args = start_parser()
    logger.debug(f'CLI arguments selected: {args}')

    body = '' if args.body == 'any' else f'body_style_slugs[]={args.body}'
    condition = '' if args.cond == 'all' else args.cond
    radius = 'all' if args.radius == 'any' else args.radius
    min_price = '' if args.price_min is None else str(args.price_min * 1000)
    max_price = '' if args.price_max is None else str(args.price_max * 1000)
    min_year = args.year_min
    max_year = args.year_max

    url = f'https://www.cars.com/shopping/results/?{body}&dealer_id=&list_price_max={max_price}&' \
          f'list_price_min={min_price}&makes[]=&maximum_distance={radius}&mileage_max=&' \
          f'page_size={config.NUMBER_OF_ADS_PER_PAGE}&' \
          f'sort=best_match_desc&stock_type={condition}&year_max={max_year}&year_min={min_year}&zip=10001'

    logger.debug(f'URL for search: {url}')
    return url, args.ads_max

def go_to_ads(car_driver):
    """
    goes to the first ad in the cars.com search page
    :param car_driver: selenium driver
    :return: None
    """
    try:
        first_ad = car_driver.find_element_by_class_name("vehicle-card-link")
    except NoSuchElementException:
        print('Could not find first ad of the page')
        logger.info('Could not find first ad of the page')
        logger.error('element with class name "vehicle-card-link" was not found')
    else:
        ActionChains(car_driver).click(first_ad).perform()
        logger.debug('clicked on first ad of the page')

def back_to_search_next_ad(car_driver, cars_looped_on_page):
    """
    goes back to search page and clicks on next car ad
    :param car_driver: selenium driver
    :param cars_looped_on_page: how many cars were clicked on the current search page
    :return: True if there are more cars to loop through, False otherwise
    """
    car_driver.back()
    car_driver.implicitly_wait(config.IMPLICIT_WAIT_TIME)
    logger.debug('driver went back to search page')

    ads_in_page = car_driver.find_elements_by_class_name("vehicle-card-link")
    logger.debug(f'Number of ad links found: {ads_in_page}')
    if ads_in_page != config.NUMBER_OF_ADS_PER_PAGE:
        logger.warning(f'Number of ad links found in page diverges from expected:'
                       f' Ads found:{ads_in_page}, Expected: {NUMBER_OF_ADS_PER_PAGE}')

    if cars_looped_on_page >= len(ads_in_page):
        logger.debug('looped through all ads in page, returning False and going to next page ')
        return False
    else:
        ActionChains(car_driver).click(ads_in_page[cars_looped_on_page]).perform()
        car_driver.implicitly_wait(config.IMPLICIT_WAIT_TIME)
        logger.debug('clicked on next add and returning True')
        return True

def next_ad(car_driver, cars_looped_on_page):
    """
    Goes to the cars.com search's next ad
    :param car_driver: selenium driver
    :param cars_looped_on_page: number of cars looped on current page
    :return: Return True if there are more ads or False otherwise
    """
    try:
        next_page_link = car_driver.find_element_by_class_name("srp-carousel-next-link")
    except NoSuchElementException:
        logger.debug('Car Carousel not found, calling back_to_search_next_ad() ')
        last_ad_check = back_to_search_next_ad(car_driver, cars_looped_on_page)
        logger.debug(f'returning {last_ad_check}')
        return last_ad_check
    else:
        logger.debug('Car Carousel found')
        last_page_check = next_page_link.get_attribute('href')[-1]
        if last_page_check == '#':
            logger.debug('Current ad is last ad on carousel/search page')
            return False

        ActionChains(car_driver).click(next_page_link).perform()
        car_driver.implicitly_wait(config.IMPLICIT_WAIT_TIME)
        logger.debug('Click on next ad and returning True')
        return True

def next_page(car_driver, previous_url):
    """
    Goes to the cars.com search's next search page and checks if ad limit was reached
    :param car_driver: selenium driver
    :param previous_url: selenium driver
    :return: Return True if program should keep going trough ads or False otherwise
    """
    try:
        current_page = int(re.search(r'page=(\d*)&', previous_url).group(1))
    except AttributeError:
        logger.debug('Current page not found on url, first page detected')
        page_index = previous_url.index('/results/?') + len('/results/?')
        new_url = previous_url[:page_index] + 'page=2&' + previous_url[page_index:]
    else:
        new_page = current_page + 1
        logger.debug(f'Current page number: {current_page}. New page: {new_page}')
        new_url = re.sub(r'page=(\d*)&', f'page={new_page}&', previous_url)

    logger.debug(f'New url constructed: {new_url}')
    car_driver.get(new_url)
    car_driver.implicitly_wait(config.IMPLICIT_WAIT_TIME)
    logger.debug('Loaded new url and returning it')
    return new_url

def new_page_check(car_driver):
    """
    Checks if there are car ads in new page
    :return: True if there are car ads or False otherwise
    """
    try:
        no_more_data = car_driver.find_element_by_class_name("sds-heading--4")
    except NoSuchElementException:
        logger.debug('Page has car ads, returning True')
        return True
    else:
        logger.debug('Page does not have car ads, returning False')
        return False

def get_soup(car_driver):
    """
    Downloads html data from website
    :param car_driver: selenium driver
    :return: beautiful soup with html
    """
    logger.debug(f'Getting soup for car: {car_driver.current_url}')
    if 'vehicledetail' not in car_driver.current_url:
        logger.warning(f'Current page is not a car ad')
    html = car_driver.page_source
    soup = BeautifulSoup(html, 'lxml')
    logger.debug('Beautiful soup for new ad generated')

    return soup

def get_general_info(soup):
    """
    Gets general information about the car: make, model, trim, year, price, exterior color, interior color, drivetrain,
    fuel type and mileage
    :param soup: html data from ad
    :return: dictionary with features
    """
    filter_keys = ['make', 'model', 'trim', 'year', 'price', 'exterior_color', 'interior_color', 'drivetrain',
                   'fuel_type', 'mileage']
    try:
        basics = soup.findAll('script')
        basics_regex = re.search('"initialActivity"] = (.*})', str(basics)).group(1)
        basics_dict = json.loads(basics_regex)
    except AttributeError:
        print('Could not fetch general information')
        logger.info('Could not fetch general information')
        filtered_dict = dict((key, 'NA') for key in filter_keys)
        filtered_dict['price'] = 0
        filtered_dict['year'] = 0
        filtered_dict['mileage'] = 0
    else:
        for key in filter_keys:
            if key not in basics_dict.keys():
                logger.debug(f'Could not find {key} for car ad, adding NA for it')
                basics_dict[key] = 'NA'
        filtered_dict = {key: basics_dict[key] for key in filter_keys}

    if filtered_dict['price'] is None:
        filtered_dict['price'] = 0

    logger.debug(f'general info: {filtered_dict}')

    return filtered_dict

def get_number_of_features(soup):
    """
    Gets the total number of feature the car has
    :param soup: html data from ad
    :return: int
    """
    feature_list = []
    all_features = soup.findAll('div', class_="all-features-item")
    for feature in all_features:
        feature_list.append(feature.text)

    n_features = len(all_features)
    logger.debug(f'number of features found: {n_features}')
    return n_features

def get_other_info(soup):
    """
    Gets other information about the car: transmission, engine and miles per gallon range
    :param soup: html data from ad
    :return: dictionary with features
    """
    other_info_dict = {}

    other_info = soup.find('dl', class_="fancy-description-list")

    try:
        other_info_dict['transmission'] = re.search(r'Transmission.*\s*
*(.*)<\/', str(other_info)).group(1)
    except AttributeError:
        print('Could not fetch transmission information')
        logger.info('Could not fetch transmission information')
        other_info_dict['transmission'] = 'NA'

    try:
        other_info_dict['engine'] = re.search(r'Engine.*\s*
*(.*)<\/', str(other_info)).group(1)
    except AttributeError:
        print('Could not fetch engine information')
        logger.info('Could not fetch engine information')
        other_info_dict['engine'] = 'NA'

    try:
        mpg = other_info.find('span', class_="sds-tooltip").find('span').text.split('–')
    except AttributeError:
        print('Could not fetch mpg information')
        logger.info('Could not fetch mpg information')
        # change later the 0
        other_info_dict['mpg_min'] = 0
        other_info_dict['mpg_max'] = 0
    else:
        if len(mpg) == 1:
            other_info_dict['mpg_min'] = mpg[0]
            other_info_dict['mpg_max'] = mpg[0]
        else:
            other_info_dict['mpg_min'] = mpg[0]
            other_info_dict['mpg_max'] = mpg[1]

    logger.debug(f'other info: {other_info_dict}')

    return other_info_dict

def get_car_reviews(soup):
    """
    Gets user car review data
    :param soup: html data from ad
    :return: dictionary with review data
    """
    car_ratings = dict()
    rating_soup = soup.find('section', class_="sds-page-section vehicle-reviews")
    try:
        car_ratings['rating'] = re.search(r'(.*)<', str(rating_soup)).group(1)
    except AttributeError:
        print('Car has no ratings')
        logger.info('Car has no ratings')
        car_ratings['rating'] = 'NA'
        car_ratings['recommended'] = 'NA'
        car_ratings['n_reviews'] = 'NA'
        car_ratings['Comfort'] = 'NA'
        car_ratings['Interior design'] = 'NA'
        car_ratings['Performance'] = 'NA'
        car_ratings['Value for the money'] = 'NA'
        car_ratings['Exterior styling'] = 'NA'
        car_ratings['Reliability'] = 'NA'
    else:
        car_ratings['recommended'] = re.search(r'^(\d*%)', soup.find('div', class_="reviews-recommended").text).group(1)

        rating_breakdown = soup.find('ul', class_="sds-definition-list review-breakdown--list").findAll('li')
        for rating in rating_breakdown:
            car_ratings[rating.find('span', class_="sds-definition-list__display-name").text] = rating.find('span',
                                                                                class_="sds-definition-list__value").text
        review_soup = soup.findAll('a', class_="sds-rating__link sds-button-link")
        for review in review_soup:
            if re.search("page-over-page", str(review)):
                car_ratings['n_reviews'] = re.search(r'\((\d*).*r', review.text).group(1)

    logger.debug(f'car ratings: {car_ratings}')
    return car_ratings

def get_seller_info(soup):
    """
    Gets seller information from ad
    :param soup: html data from ad
    :return: dictionary with seller info
    """
    seller = dict()
    try:
        seller['name'] = soup.find('h3', class_="sds-heading--5 heading seller-name").text
        seller['address'] = soup.find('div', class_="dealer-address").text
    except AttributeError:
        print('Could not fetch seller information')
        logger.info('Could not fetch seller information')
        seller['name'] = 'NA'
        seller['address'] = 'NA'
        seller['rating'] = 'NA'
        seller['n_reviews'] = 'NA'
        return seller

    rating_soup = soup.find('section', class_="sds-page-section seller-info")
    try:
        seller['rating'] = re.search(r'(.*)<', str(rating_soup)).group(1)
    except AttributeError:
        print('Seller has no reviews')
        logger.info('Seller has no reviews')
        seller['rating'] = 'NA'
        seller['n_reviews'] = 'NA'
    else:
        review_soup = soup.findAll('a', class_="sds-rating__link sds-button-link")
        for review in review_soup:
            if re.search("click-vdp-to-dpp-reviews", str(review)):
                seller['n_reviews'] = re.search(r'\((\d*).*r', review.text).group(1)

    logger.debug(f'Seller info: {seller}')
    return seller

def write_to_db(my_dbm, my_car, my_seller):
    """function that receives a Cars_DBM object and writes with it to the database the information of an ad"""
    my_dbm.insert_seller_row(my_seller)
    logger.debug('Inserted new row in seller db')
    my_dbm.insert_car_type_row(my_car)
    logger.debug('Inserted new row in car_type db')
    my_dbm.insert_car_row(my_car, my_seller)
    logger.debug('Inserted new row in car db')

def check_and_close_pop_up(car_driver):
    """
    Checks if pop up ad from cars.com has showed up and closes it if it does
    :param car_driver: selenium driver
    :return: None
    """
    ad_close_button_style = "position: absolute; top: 0px; left: 0px; width: 27px; height: 26px; overflow: hidden; display: block;"
    try:
        ad_close_button = car_driver.find_element(By.XPATH, f"//div[@style={ad_close_button_style}]")
    except NoSuchElementException:
        logger.debug('No pop up found on current page')
        pass
    else:
        ActionChains(car_driver).click(ad_close_button).perform()
        car_driver.implicitly_wait(config.IMPLICIT_WAIT_TIME)
        logger.debug('Closed pop up')

def main():
    """ function that runs the program"""
    car_dbm = Cars_DBM()

    start_parser()
    url, max_ads = get_url()
    if config.BROWSER == 'Chrome':
        driver = webdriver.Chrome()
    elif config.BROWSER == 'Firefox':
        driver = webdriver.Firefox()

    if config.BROWSER == 'Chrome':
        driver = webdriver.Chrome()
    elif config.BROWSER == 'Firefox':
        driver = webdriver.Firefox()
    driver.get(url)
    driver.implicitly_wait(config.IMPLICIT_WAIT_TIME)

    go_to_ads(driver)
    driver.implicitly_wait(config.IMPLICIT_WAIT_TIME)

    keep_looping = True
    cars_looped_current_page = 0
    total_cars_looped = 0
    last_link = None
    while keep_looping:
        driver.implicitly_wait(config.IMPLICIT_WAIT_TIME)

        if last_link != driver.current_url:
            last_link = driver.current_url
            car_soup = get_soup(driver)
            general_info = get_general_info(car_soup)
            other_info = get_other_info(car_soup)
            seller_info_dict = get_seller_info(car_soup)

            number_of_features = get_number_of_features(car_soup)
            car_reviews = get_car_reviews(car_soup)

            car_info = Car_and_Seller.Car(general_info, other_info)
            seller_info = Car_and_Seller.Seller(seller_info_dict)

            cars_looped_current_page += 1
            total_cars_looped += 1
            logger.debug(f'Cars looped on current search page:{cars_looped_current_page}')
            logger.debug(f'Total cars looped:{total_cars_looped}')

            if general_info['make'] != 'NA' and general_info['model'] != 'NA' and seller_info_dict['name'] != 'NA' \
                    and seller_info_dict['rating'] != 'NA':
                write_to_db(car_dbm, car_info, seller_info)

            if max_ads is not None:
                max_ads -= 1
                logger.debug(f'Current max ads:{max_ads}')
                if max_ads == 0:
                    keep_looping = False
        else:
            logger.warning(f'Previous car ad is the same as current, error in loop '
                           f'Link: {last_link}')

        ads_remaining = next_ad(driver, cars_looped_current_page)
        if not ads_remaining:
            logger.debug('No ads remaining, going to next page')
            url = next_page(driver, url)
            check_and_close_pop_up(driver)
            ads_in_page = new_page_check(driver)
            cars_looped_current_page = 0
            if not ads_in_page:
                logger.debug(f'Stopping while loop')
                keep_looping = False
            else:
                go_to_ads(driver)
                driver.implicitly_wait(config.IMPLICIT_WAIT_TIME)
    update_type_in_db(car_dbm)
    print('Car ads scrapped successfully')
    logger.info('Car ads scrapped successfully')
    driver.close()

if __name__ == '__main__':
    main()


# Definition for a binary tree node.
# class TreeNode:
#     def __init__(self, val=0, left=None, right=None):
#         self.val = val
#         self.left = left
#         self.right = right
class Solution:
    def sumEvenGrandparent_my(self, root: TreeNode) -> int:  #  66.60%
        stack = [(root, 0, [])]
        out = 0
        while stack:
            root = stack.pop()
            if root[0].val % 2 == 0:
                root[2].append(root[1]+2)
            if root[1] in root[2]:
                out += root[0].val
            if root[0].left:
                stack.append((root[0].left, root[1]+1, root[2][:]))
            if root[0].right:
                stack.append((root[0].right, root[1]+1, root[2][:]))
        return out

    def sumEvenGrandparent_best(self, root: TreeNode) -> int:
        res_sum = 0

        def findSum(curr, parent, gParent):
            nonlocal res_sum
            if gParent and not gParent.val % 2:
                res_sum += curr.val
            if curr.left:
                findSum(curr.left, curr, parent)
            if curr.right:
                findSum(curr.right, curr, parent)
        findSum(root, None, None)
        return res_sum

1300 - 1399/1315 - Sum of Nodes with Even-Valued Grandparent/1315 - Sum of Nodes with Even-Valued Grandparent.py


from algoritmia.datastructures.digraphs import UndirectedGraph
from algoritmia.datastructures.queues import Fifo

# busqueda tesoro en anchura

def bfst(lab, v_ini, v_tes):
    q = Fifo()
    seen = set()

    q.push(v_ini)
    seen.add(v_ini)

    while len(q) > 0:
        at = q.pop()
        if v_tes == at:
            return at

        for suc in lab.succs(at):
            if suc not in seen:
                seen.add(suc)
                q.push(suc)

    return None

if __name__ == '__main__':
    pasillos = [((0, 0), (0, 1)), ((0, 2), (0, 3)), ((1, 0), (1, 1)), ((1, 1), (1, 2)),
                ((2, 0), (2, 1)), ((2, 1), (2, 2)), ((2, 2), (2, 3)), ((0, 1), (1, 1)),
                ((0, 2), (1, 2)), ((0, 3), (1, 3)), ((1, 1), (2, 1)), ((1, 2), (2, 2))]

    lab = UndirectedGraph(E=pasillos)
    v_ini = (0, 0)
    v_tes = (1, 3)

    pos = bfst(lab, v_ini, v_tes)

    if pos is None:
        print("Tesoro no encontrado")
    else:
        print("Tesoso encontrado en el vertice {0}".format(pos))


# -*- coding: utf-8 -*-

import socket
nb_requetes = 5 #nombre de requetes de connexions
host = 'localhost'
port = 9000

ourSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
	ourSocket.bind((host, port))
except socket.error :
	("server error")
	exit()
ourSocket.listen(nb_requetes)
print("serveur en ligne")
unending = True
while unending:
	connexion , adresse = ourSocket.accept()

	print("un connecté , ip : {0} , port : {1}".format(adresse[0],adresse[1]))


from copy import deepcopy
from datetime import datetime
import numpy as np
import numpy.testing as nptest
import pandas as pd
import pytesmo.temporal_matching as tmatching
import pytest

def test_df_match_borders():
    """
    Border values can be problematic for temporal matching.

    See issue #51
    """

    ref_df = pd.DataFrame(
        {"data": np.arange(5)},
        index=pd.date_range(datetime(2007, 1, 1, 0), "2007-01-05", freq="D"),
    )
    match_df = pd.DataFrame(
        {"matched_data": np.arange(5)},
        index=[
            datetime(2007, 1, 1, 9),
            datetime(2007, 1, 2, 9),
            datetime(2007, 1, 3, 9),
            datetime(2007, 1, 4, 9),
            datetime(2007, 1, 5, 9),
        ],
    )
    with pytest.deprecated_call():
        matched = tmatching.df_match(ref_df, match_df)

    nptest.assert_allclose(
        np.array([0.375, 0.375, 0.375, 0.375, 0.375]), matched.distance.values
    )
    nptest.assert_allclose(np.arange(5), matched.matched_data)

def test_df_match_match_on_window_border():
    """
    test matching if a value lies exactly on the window border.
    """

    ref_df = pd.DataFrame(
        {"data": np.arange(5)},
        index=pd.date_range(datetime(2007, 1, 1, 0), "2007-01-05", freq="D"),
    )
    match_df = pd.DataFrame(
        {"matched_data": np.arange(4)},
        index=[
            datetime(2007, 1, 1, 9),
            datetime(2007, 1, 2, 9),
            datetime(2007, 1, 3, 12),
            datetime(2007, 1, 5, 9),
        ],
    )
    with pytest.deprecated_call():
        matched = tmatching.df_match(ref_df, match_df, window=0.5)

    nptest.assert_allclose(
        np.array([0.375, 0.375, 0.5, -0.5, 0.375]), matched.distance.values
    )
    nptest.assert_allclose([0, 1, 2, 2, 3], matched.matched_data)

    # test asym_window keyword
    with pytest.deprecated_call():
        matched = tmatching.df_match(
            ref_df, match_df, window=0.5, asym_window="<="
        )

    nptest.assert_allclose(
        np.array([0.375, 0.375, 0.5, np.nan, 0.375]), matched.distance.values
    )
    nptest.assert_allclose([0, 1, 2, np.nan, 3], matched.matched_data)

    with pytest.deprecated_call():
        matched = tmatching.df_match(
            ref_df, match_df, window=0.5, asym_window=">="
        )

    nptest.assert_allclose(
        np.array([0.375, 0.375, np.nan, -0.5, 0.375]), matched.distance.values
    )
    nptest.assert_allclose([0, 1, np.nan, 2, 3], matched.matched_data)

def test_df_match_borders_unequal_query_points():
    """
    Border values can be problematic for temporal matching.

    See issue #51
    """

    ref_df = pd.DataFrame(
        {"data": np.arange(5)},
        index=pd.date_range(datetime(2007, 1, 1, 0), "2007-01-05", freq="D"),
    )
    match_df = pd.DataFrame(
        {"matched_data": np.arange(4)},
        index=[
            datetime(2007, 1, 1, 9),
            datetime(2007, 1, 2, 9),
            datetime(2007, 1, 4, 9),
            datetime(2007, 1, 5, 9),
        ],
    )
    with pytest.deprecated_call():
        matched = tmatching.df_match(ref_df, match_df)

    nptest.assert_allclose(
        np.array([0.375, 0.375, -0.625, 0.375, 0.375]), matched.distance.values
    )
    nptest.assert_allclose(np.array([0, 1, 1, 2, 3]), matched.matched_data)

def test_matching():
    """
    test matching function
    """
    data = np.arange(5.0)
    data[3] = np.nan

    ref_df = pd.DataFrame(
        {"data": data},
        index=pd.date_range(datetime(2007, 1, 1, 0), "2007-01-05", freq="D"),
    )
    match_df = pd.DataFrame(
        {"matched_data": np.arange(5)},
        index=[
            datetime(2007, 1, 1, 9),
            datetime(2007, 1, 2, 9),
            datetime(2007, 1, 3, 9),
            datetime(2007, 1, 4, 9),
            datetime(2007, 1, 5, 9),
        ],
    )
    with pytest.deprecated_call():
        matched = tmatching.matching(ref_df, match_df)

    nptest.assert_allclose(np.array([0, 1, 2, 4]), matched.matched_data)
    assert len(matched) == 4

def test_matching_series():
    """
    test matching function with pd.Series as input
    """
    data = np.arange(5.0)
    data[3] = np.nan

    ref_ser = pd.Series(
        data,
        index=pd.date_range(datetime(2007, 1, 1, 0), "2007-01-05", freq="D"),
    )
    match_ser = pd.Series(
        np.arange(5),
        index=[
            datetime(2007, 1, 1, 9),
            datetime(2007, 1, 2, 9),
            datetime(2007, 1, 3, 9),
            datetime(2007, 1, 4, 9),
            datetime(2007, 1, 5, 9),
        ],
        name="matched_data",
    )

    with pytest.deprecated_call():
        matched = tmatching.matching(ref_ser, match_ser)

    nptest.assert_allclose(np.array([0, 1, 2, 4]), matched.matched_data)
    assert len(matched) == 4

#############################################################################
# Tests for new implementation
#############################################################################

@pytest.fixture
def test_data():
    """
    Test data for temporal matching.

    The test frames have modified time indices:
    - shifted by 3 hours
    - shifted by 7 hours
    - shifted by 3 hours, and in a timezone such that displayed numbers are >6
      hours apart
    - shifted by 7 hours, and in a timezone such that displayed numbers are <6
      hours apart
    - randomly shifted by a value between -12h and 12h
    - same as above, but with some dropped values
    - shifted by 3 hours, with duplicates

    Returns
    -------
    ref_frame : pd.DataFrame
        Reference data frame
    test_frames : dict of pd.DataFrame
        Dictionary of data frames, keys are: "shifted_3", "shifted_7",
        "shifted_3_asia", "shifted_7_us", "random_shift", "duplicates".
    expected_nan : dict of np.ndarray
        Dictionary with same keywords as `test_frames`, each entry is a mask
        indicating where NaNs are expected (i.e. no matching was taking place)
    """
    # the reference date range
    ref_dr = pd.date_range("1970", "2020", freq="D", tz="UTC")

    test_dr = {}
    test_dr["shifted_3"] = ref_dr + pd.Timedelta(3, "H")
    test_dr["shifted_7"] = ref_dr + pd.Timedelta(7, "H")
    test_dr["shifted_3_asia"] = test_dr["shifted_3"].tz_convert(
        "Asia/Yekaterinburg"
    )
    test_dr["shifted_7_us"] = test_dr["shifted_7"].tz_convert("US/Eastern")

    # random shifts
    random_hours = np.random.uniform(-12.0, 12.0, len(ref_dr))
    random_mask = np.abs(random_hours) > 6
    dr_random_shift = ref_dr + pd.to_timedelta(random_hours, "H")
    test_dr["random_shift"] = dr_random_shift

    # missing data
    drop_mask = np.zeros(len(ref_dr), dtype=bool)
    drop_mask[100:200] = True
    dr_random_shift = dr_random_shift[~drop_mask]
    test_dr["missing"] = dr_random_shift
    missing_mask = random_mask | drop_mask

    # with duplicates
    test_dr["duplicates"] = deepcopy(test_dr["shifted_3"])
    duplicates_mask = np.zeros(len(ref_dr), dtype=bool)
    for idx in np.random.randint(0, len(test_dr["duplicates"]) - 1, 5):
        test_dr["duplicates"].values[idx] = test_dr["duplicates"].values[
            idx + 1
        ]
        duplicates_mask[idx] = True

    # setting up dataframes
    test_frames = {
        key: pd.DataFrame(
            np.random.randn(len(test_dr[key]), 3), index=test_dr[key]
        )
        for key in test_dr
    }
    ref_frame = pd.DataFrame(np.random.randn(len(ref_dr), 3), index=ref_dr)

    # mask for where we expect nans in the output
    all_nan = np.ones(len(ref_dr), dtype=bool)
    expected_nan = {
        "shifted_3": ~all_nan,
        "shifted_7": all_nan,
        "shifted_3_asia": ~all_nan,
        "shifted_7_us": all_nan,
        "random_shift": random_mask,
        "missing": missing_mask,
        "duplicates": duplicates_mask,
    }
    return ref_frame, test_frames, expected_nan

def setup_data(data, key):
    """Returns only relevant data of test_data for given key"""
    return data[0], data[1][key], data[2][key]

def compare_with_nan(a, b):
    return (a == b) | (np.isnan(a) & np.isnan(b))

def assert_equal_except_nan(res, ref, nan_mask, index_shifted=False):
    expected_nan_idx = nan_mask.nonzero()[0]
    expected_nonan_idx = (~nan_mask).nonzero()[0]
    # using column zero here, all should be the same
    nan_idx = np.isnan(res.iloc[:, 0].values).nonzero()[0]
    nonan_idx = (~np.isnan(res.iloc[:, 0].values)).nonzero()[0]
    assert len(expected_nan_idx) == len(nan_idx)
    if len(nan_idx) > 0:
        assert np.all(nan_idx == expected_nan_idx)
    if len(nonan_idx) > 0 and not index_shifted:
        assert np.all(nonan_idx == expected_nonan_idx)
        assert np.all(
            res.iloc[nonan_idx, 0].values == ref.iloc[nonan_idx, 0].values
        )

@pytest.mark.parametrize(
    "key",
    [
        "shifted_3",
        "shifted_7",
        "shifted_7_us",
        "shifted_3_asia",
        "random_shift",
    ],
)
def test_collocation_nearest_neighbour(test_data, key):
    ref_frame, test_frame, expected_nan = setup_data(test_data, key)
    res = tmatching.temporal_collocation(
        ref_frame, test_frame, pd.Timedelta(6, "H")
    )
    assert_equal_except_nan(res, test_frame, expected_nan)

@pytest.mark.parametrize("key", ["missing", "duplicates"])
def test_collocation_missing_duplicates(test_data, key):
    ref_frame, test_frame, expected_nan = setup_data(test_data, key)
    res = tmatching.temporal_collocation(
        ref_frame,
        test_frame,
        pd.Timedelta(6, "H"),
    )
    # indices of test_frame are shifted w.r.t expected_nan, therefore we can't
    # compare values
    assert_equal_except_nan(res, test_frame, expected_nan, index_shifted=True)

@pytest.mark.parametrize("key", ["shifted_3"])
def test_collocation_window(test_data, key):
    ref_frame, test_frame, expected_nan = setup_data(test_data, key)
    res = tmatching.temporal_collocation(
        ref_frame, test_frame, 6 / 24, dropduplicates=True
    )
    assert_equal_except_nan(res, test_frame, expected_nan, index_shifted=True)

@pytest.mark.parametrize("key", ["shifted_3"])
def test_collocation_input(test_data, key):
    ref_frame, test_frame, expected_nan = setup_data(test_data, key)

    no_timezone = pd.date_range("1970", "2020", freq="D")
    # test with series and index:
    for ref in [ref_frame[0], ref_frame.index, no_timezone]:
        res = tmatching.temporal_collocation(
            ref, test_frame, pd.Timedelta(6, "H")
        )
        assert_equal_except_nan(res, test_frame, expected_nan)

@pytest.mark.parametrize(
    "key",
    [
        "shifted_3",
        "shifted_7",
        "shifted_7_us",
        "shifted_3_asia",
        "random_shift",
    ],
)
def test_collocation_dropna(test_data, key):
    ref_frame, test_frame, expected_nan = setup_data(test_data, key)
    res = tmatching.temporal_collocation(
        ref_frame, test_frame, pd.Timedelta(6, "H"), dropna=True
    )
    expected_nonan_idx = (~expected_nan).nonzero()[0]
    assert np.all(test_frame.iloc[expected_nonan_idx, :].values == res.values)

@pytest.mark.parametrize(
    "key",
    [
        "shifted_3",
        "shifted_7",
        "shifted_7_us",
        "shifted_3_asia",
        "random_shift",
    ],
)
def test_collocation_flag(test_data, key):
    ref_frame, test_frame, expected_nan = setup_data(test_data, key)
    flag = np.random.choice([True, False], len(ref_frame))

    # with array
    res = tmatching.temporal_collocation(
        ref_frame,
        test_frame,
        pd.Timedelta(6, "H"),
        flag=flag,
    )

    compare_with_nan(
        res.iloc[:, 0].values[~flag], test_frame.iloc[:, 0].values[~flag]
    )
    assert np.all(np.isnan(res.values[:, 0][flag]))

    # with array, using invalid as replacement
    res = tmatching.temporal_collocation(
        ref_frame,
        test_frame,
        pd.Timedelta(6, "H"),
        flag=flag,
        use_invalid=True,
    )
    compare_with_nan(res.iloc[:, 0].values, test_frame.iloc[:, 0].values)

    # with dataframe
    test_frame["flag"] = flag
    res = tmatching.temporal_collocation(
        ref_frame,
        test_frame,
        pd.Timedelta(6, "H"),
        flag="flag",
    )
    compare_with_nan(
        res.iloc[:, 0].values[~flag], test_frame.iloc[:, 0].values[~flag]
    )
    assert np.all(np.isnan(res.iloc[:, 0].values[flag]))

# using only shifted_3, because comparison won't work when there are nans
@pytest.mark.parametrize("key", ["shifted_3"])
def test_return_index(test_data, key):
    ref_frame, test_frame, expected_nan = setup_data(test_data, key)
    res = tmatching.temporal_collocation(
        ref_frame, test_frame, pd.Timedelta(6, "H"), return_index=True
    )
    assert_equal_except_nan(res, test_frame, expected_nan)
    assert np.all(test_frame.index.values == res["index_other"].values)

# using only shifted_3, because comparison won't work when there are nans
@pytest.mark.parametrize("key", ["shifted_3", "shifted_7"])
def test_return_distance(test_data, key):
    ref_frame, test_frame, expected_nan = setup_data(test_data, key)
    res = tmatching.temporal_collocation(
        ref_frame, test_frame, pd.Timedelta(6, "H"), return_distance=True
    )
    assert_equal_except_nan(res, test_frame, expected_nan)
    if key == "shifted_3":
        assert np.all(res["distance_other"] == pd.Timedelta(3, "H"))
    if key == "shifted_7":
        assert np.all(np.isnan(res["distance_other"]))

def test_timezone_handling():
    # Issue #150
    data = np.arange(5.0)
    data[3] = np.nan

    match_df = pd.DataFrame(
        {"matched_data": data},
        index=pd.date_range(
            datetime(2007, 1, 1, 0), "2007-01-05", freq="D", tz="UTC"
        ),
    )
    index = pd.DatetimeIndex(
        [
            datetime(2007, 1, 1, 9),
            datetime(2007, 1, 2, 9),
            datetime(2007, 1, 3, 9),
            datetime(2007, 1, 4, 9),
            datetime(2007, 1, 5, 9),
        ]
    ).tz_localize("utc")
    ref_df = pd.DataFrame({"data": np.arange(5)}, index=index)
    matched = tmatching.temporal_collocation(
        ref_df,
        match_df,
        pd.Timedelta(12, "H"),
        dropna=True,
    )

    nptest.assert_allclose(np.array([0, 1, 2, 4]), matched.matched_data)
    assert len(matched) == 4

def test_warning_on_no_match(test_data):
    # Issue #152
    ref_frame, test_frame, expected_nan = setup_data(test_data, "shifted_7")
    with pytest.warns(UserWarning):
        tmatching.temporal_collocation(
            ref_frame, test_frame, pd.Timedelta(6, "H"), checkna=True
        )

def test_combined_matching():
    index = pd.DatetimeIndex([datetime(2007, 1, i + 1, 0) for i in range(10)])
    ref = pd.DatetimeIndex([datetime(2007, 1, i + 1, 5) for i in range(10)])

    data = {
        "data1": np.random.randn(10),
        "data2": np.random.randn(10),
        "missing": np.random.randn(10),
    }
    data["missing"][2] = np.nan
    frames = {
        name: pd.DataFrame({name: data[name]}, index=index) for name in data
    }

    # everything together
    merged = tmatching.combined_temporal_collocation(
        ref,
        (frames[name] for name in frames),
        pd.Timedelta(6, "H"),
        combined_dropna=False,
    )

    assert len(merged) == 10
    for name in frames:
        assert name in merged.columns
        nptest.assert_equal(
            merged[name].values.ravel(), frames[name].values.ravel()
        )

    # test with dropna but not combined_dropna
    merged = tmatching.combined_temporal_collocation(
        ref,
        (frames[name] for name in frames),
        pd.Timedelta(6, "H"),
        combined_dropna=False,
        dropna=True,
    )

    assert len(merged) == 10
    for name in frames:
        assert name in merged.columns
        nptest.assert_equal(
            merged[name].values.ravel(), frames[name].values.ravel()
        )

    # test with combined_dropna
    merged = tmatching.combined_temporal_collocation(
        ref,
        (frames[name] for name in frames),
        pd.Timedelta(6, "H"),
        combined_dropna="any",
        dropna=True,
    )

    assert len(merged) == 9
    for name in frames:
        assert name in merged.columns
        nptest.assert_equal(
            merged[name].values.ravel()[2:], frames[name].values.ravel()[3:]
        )

    # test with 2d-dataframe
    df2d = pd.DataFrame(
        {"2d1": np.random.randn(10), "2d2": np.random.randn(10)}, index=index
    )
    merged = tmatching.combined_temporal_collocation(
        ref,
        (frames["missing"], df2d),
        pd.Timedelta(6, "H"),
        combined_dropna=False,
    )
    assert len(merged) == 10

    # test without match
    for comb_drop in [True, False]:
        merged = tmatching.combined_temporal_collocation(
            ref,
            (frames["missing"], df2d),
            pd.Timedelta(1, "H"),
            combined_dropna=comb_drop,
            dropna=True,
        )
        assert len(merged) == 0

def test_timezone_warning():
    dr = pd.date_range("2000-01-01", "2000-01-31", freq="D")
    dr_berlin = pd.date_range(
        "2000-01-01", "2000-01-31", freq="D", tz="Europe/Berlin"
    )
    n = len(dr)
    with pytest.warns(UserWarning, match="No timezone given"):
        matched = tmatching.temporal_collocation(
            pd.DataFrame(np.random.randn(n), index=dr),
            pd.DataFrame(np.random.randn(n), index=dr_berlin),
            pd.Timedelta(6, "H"),
        )
        assert str(matched.index.tz) == "Europe/Berlin"

def test_combined_timezones():
    dr = pd.date_range("2000-01-01", "2000-01-31", freq="D")
    dr_utc = pd.date_range("2000-01-01", "2000-01-31", freq="D", tz="UTC")
    dr_berlin = pd.date_range(
        "2000-01-01", "2000-01-31", freq="D", tz="Europe/Berlin"
    )
    n = len(dr)

    # test timezone naive
    merged = tmatching.combined_temporal_collocation(
        pd.DataFrame(np.random.randn(n), index=dr),
        (
            pd.DataFrame(np.random.randn(n), index=dr),
            pd.DataFrame(np.random.randn(n), index=dr),
        ),
        pd.Timedelta(6, "H"),
        add_ref_data=True,
    )
    assert merged.index.tz is None

    # test with same timezone
    merged = tmatching.combined_temporal_collocation(
        pd.DataFrame(np.random.randn(n), index=dr_berlin),
        (
            pd.DataFrame(np.random.randn(n), index=dr_berlin),
            pd.DataFrame(np.random.randn(n), index=dr_berlin),
        ),
        pd.Timedelta(6, "H"),
        add_ref_data=True,
    )
    assert str(merged.index.tz) == "Europe/Berlin"

    # test with missing timezone
    with pytest.warns(UserWarning, match="No timezone given"):
        merged = tmatching.combined_temporal_collocation(
            pd.DataFrame(np.random.randn(n), index=dr),
            (
                pd.DataFrame(np.random.randn(n), index=dr_berlin),
                pd.DataFrame(np.random.randn(n), index=dr),
            ),
            pd.Timedelta(6, "H"),
            add_ref_data=True,
        )
        assert str(merged.index.tz) == "Europe/Berlin"

    # test with different timezones and no ref timezone
    with pytest.warns(UserWarning) as warn_record:
        merged = tmatching.combined_temporal_collocation(
            pd.DataFrame(np.random.randn(n), index=dr),
            (
                pd.DataFrame(np.random.randn(n), index=dr_berlin),
                pd.DataFrame(np.random.randn(n), index=dr_utc),
            ),
            pd.Timedelta(6, "H"),
            add_ref_data=True,
        )
        assert str(merged.index.tz) == "UTC"
    assert len(warn_record) == 3
    assert "No timezone given" in warn_record[0].message.args[0]
    assert "Europe/Berlin" in warn_record[0].message.args[0]
    assert "No timezone given" in warn_record[1].message.args[0]
    assert "UTC" in warn_record[1].message.args[0]
    assert "mixed timezones" in warn_record[2].message.args[0]

    # test with different timezones and ref timezone
    merged = tmatching.combined_temporal_collocation(
        pd.DataFrame(np.random.randn(n), index=dr_berlin),
        (
            pd.DataFrame(np.random.randn(n), index=dr_berlin),
            pd.DataFrame(np.random.randn(n), index=dr_utc),
        ),
        pd.Timedelta(6, "H"),
        add_ref_data=True,
    )
    assert str(merged.index.tz) == "Europe/Berlin"

def test_resample_mean():

    times = np.arange(10, dtype=float)
    vals = np.arange(10, dtype=float)

    target_times = np.asarray([3, 7, 10, 12], dtype=float)
    window = 3.0

    expected = np.asarray([3, 7, 9, np.nan])

    resampled = tmatching.resample_mean(times, vals, target_times, window)

    def repeat(x, n):
        return np.repeat(x[np.newaxis, :], n, axis=0)

    newvals = repeat(vals, 12)
    new_expected = repeat(expected, 12)
    resampled = tmatching.resample_mean(times, newvals, target_times, window)
    np.testing.assert_equal(new_expected, resampled)

def test_mean_collocation():
    dr = pd.date_range("1970", "2020", freq="D", tz="UTC")
    other = pd.DataFrame(
        np.vstack(
            (np.arange(len(dr), dtype=float), np.arange(len(dr), dtype=float))
        ).T,
        index=dr,
    )
    # window is actually half window in temporal_collocation
    window = pd.Timedelta(1.5, "D")

    dateidx = list(range(0, len(dr) - 3, 3))
    expected = list(range(0, len(dr) - 3, 3))
    expected[0] = 0.5
    ref_dr = pd.DatetimeIndex([dr[i] for i in dateidx])

    resampled = tmatching.temporal_collocation(
        ref_dr, other, window, method="mean"
    )
    assert (resampled[0] == resampled[1]).all()
    assert (resampled[0] == expected).all()

    # try with Series
    s = tmatching.temporal_collocation(ref_dr, other[0], window, method="mean")
    assert (s == resampled[0]).all()
    assert isinstance(s, pd.Series)

def test_mean_collocation_missing_start_end():
    # In this test the first and last ten timestamps have no values within the
    # window. This is to make sure that indexing at the start and end works as
    # expected.

    dr = pd.date_range("2019", "2020", freq="D", tz="UTC")
    other_dr = pd.date_range("2019", "2020", freq="D", tz="UTC").values
    other_dr[0:10] += pd.Timedelta(12, "H")
    other_dr[-10:] -= pd.Timedelta(12, "H")
    other = pd.DataFrame(
        np.vstack(
            (np.arange(len(dr), dtype=float), np.arange(len(dr), dtype=float))
        ).T,
        index=other_dr,
    )

    resampled = tmatching.temporal_collocation(
        dr, other, pd.Timedelta(6, "H"), method="mean"
    )

    assert np.all(np.isnan(resampled[0:10]))
    assert np.all(np.isnan(resampled[-10:]))
    np.testing.assert_equal(other[10:-10].values, resampled[10:-10].values)

    # test with "random" data that uses a seed with similar properties
    np.random.seed(8986)
    ref = pd.date_range("2020-01-01", "2020-12-31", freq="D")
    values = np.random.randn(len(ref), 3)
    random_shift = np.random.uniform(-12, 12, len(ref))
    random = pd.DataFrame(
        values, index=ref + pd.to_timedelta(random_shift, "H"),
        columns=list(map(lambda x: f"random_{x}", range(3)))
    )
    window = pd.Timedelta(hours=6)
    matched = tmatching.temporal_collocation(
        ref, random, window, method="mean"
    )

    should_be_nan = np.abs(random_shift) > 6
    expected = np.array(values)
    expected[should_be_nan, :] = np.nan
    np.testing.assert_equal(expected, matched)


#!/usr/bin/env python3
__author__ = 'thomason-jesse'
# Takes in the split and labeled data folds, runs baselines and reports performance.

import argparse
import json
import os
import pandas as pd
from PIL import Image
from models import *
from sklearn.metrics import cohen_kappa_score
from torchvision.models import resnet
from torchvision.transforms import ToTensor
from torchvision.transforms import Normalize
from torchvision.transforms.functional import resize
from utils import *

def main(args):
    assert not args.rgbd_only or args.exec_robo_indir is not None
    num_expected_trials = 5  # discards pairs with fewer than 5 trials

    # Read in labeled folds.
    print("Reading in labeled folds from '" + args.infile + "'...")
    with open(args.infile, 'r') as f:
        all_d = json.load(f)
        names = all_d["names"]
        lf = all_d["folds"]
        train = lf['train']
        preps = train.keys()
        if args.test:
            test = lf['test']
        else:
            test = lf['dev']  # only ever peak at the dev set.
    print("... done")

    # Read in metadata.
    print("Reading in metadata from '" + args.metadata_infile + "'...")
    with open(args.metadata_infile, 'r') as f:
        d = json.load(f)
        res = d["res"]
        imgs = d["imgs"]
    print("... done")

    # Read in RBDG features.
    print("Reading in RGBD features from '" + args.robot_infile + "'...")
    with open(args.robot_infile, 'r') as f:
        rgbd = json.load(f)
        rgbd_tr = rgbd['train']
        if args.test:
            rgbd_te = rgbd['test']
        else:
            rgbd_te = rgbd['dev']
    print("... done")

    # Select subset of data on which to evaluate.
    print("Selecting evaluation data...")
    available_train = {p: None for p in preps}
    available_test = {p: None for p in preps}
    # Class cup mergers equate f_cups and e_cups (two medium-most cups), so blacklist f_cups from the training set since
    # e_cups appears in the test set.
    blacklist = ['065-f_cups']
    for p in preps:
        if args.rgbd_only:
            available_train[p] = [[ix, train[p]["ob1"][ix], train[p]["ob2"][ix]]
                                  for ix in range(len(train[p]["ob1"]))
                                  if names[train[p]["ob1"][ix]] not in blacklist and
                                  names[train[p]["ob2"][ix]] not in blacklist and
                                  str(train[p]["ob1"][ix]) in rgbd_tr and
                                  str(train[p]["ob2"][ix]) in rgbd_tr[str(train[p]["ob1"][ix])] and
                                  len(rgbd_tr[str(train[p]["ob1"][ix])][str(train[p]["ob2"][ix])][0]) == num_expected_trials]
            available_test[p] = [[ix, test[p]["ob1"][ix], test[p]["ob2"][ix]]
                                 for ix in range(len(test[p]["ob1"]))
                                 if names[test[p]["ob1"][ix]] not in blacklist and
                                 names[test[p]["ob2"][ix]] not in blacklist and
                                 str(test[p]["ob1"][ix]) in rgbd_te and
                                 str(test[p]["ob2"][ix]) in rgbd_te[str(test[p]["ob1"][ix])] and
                                 len(rgbd_te[str(test[p]["ob1"][ix])][str(test[p]["ob2"][ix])][0]) == num_expected_trials]
            print("... done; %d / %d available training and %d / %d available testing examples with RGBD data for %s" %
                  (len(available_train[p]), len(train[p]["ob1"]), len(available_test[p]), len(test[p]["ob1"]), p))
        else:
            available_train[p] = [[ix, train[p]["ob1"][ix], train[p]["ob2"][ix]]
                                  for ix in range(len(train[p]["ob1"]))
                                  if names[train[p]["ob1"][ix]] not in blacklist and
                                  names[train[p]["ob2"][ix]] not in blacklist]
            available_test[p] = [[ix, test[p]["ob1"][ix], test[p]["ob2"][ix]]
                                 for ix in range(len(test[p]["ob1"]))
                                 if names[test[p]["ob1"][ix]] not in blacklist and
                                 names[test[p]["ob2"][ix]] not in blacklist]
            print("... done; %d / %d available training and %d / %d available testing examples for %s" %
                  (len(available_train[p]), len(train[p]["ob1"]), len(available_test[p]), len(test[p]["ob1"]), p))
    tr_o = {p: [train[p]["label"][ix] for ix, _, _ in available_train[p]] for p in preps}
    te_o = {p: [test[p]["label"][ix] for ix, _, _ in available_test[p]] for p in preps}
    tr_names = {p: [(names[ob1], names[ob2]) for _, ob1, ob2 in available_train[p]] for p in preps}
    te_names = {p: [(names[ob1], names[ob2]) for _, ob1, ob2 in available_test[p]] for p in preps}

    # Read in robot execution ground truth labels from annotation CSVs.
    tr_robo_o = {p: [-1 for ix, _, _ in available_train[p]] for p in preps}
    te_robo_o = {p: [-1 for ix, _, _ in available_test[p]] for p in preps}
    if args.exec_robo_indir is not None:
        print("Reading in robot execution ground truth labels from '" + args.exec_robo_indir + "'...")
        gt_labels = {p: {} for p in preps}
        l2c = {"y": 2, "m": 1, "n": 0}  # Matching 3 class split.
        for _, _, fns in os.walk(args.exec_robo_indir):
            for fn in fns:
                ps = fn.split('.')
                if len(ps) >= 2 and ps[-1] == "csv" and ps[-2] == "labels":  # annotation file
                    print("... reading annotations from '%s'..." % fn)
                    num_annot = 0
                    df = pd.read_csv(os.path.join(args.exec_robo_indir, fn))
                    for idx in df.index:
                        if type(df["Pair"][idx]) is not str or " + " not in df["Pair"][idx]:
                            continue
                        lp = df["Pair"][idx].strip().split(" + ")
                        aidx = names.index(lp[0])
                        bidx = names.index(lp[1])
                        k = (aidx, bidx)
                        try:
                            nt = int(df["Num Trials"][idx])
                        except ValueError:
                            continue
                        if nt > 0:
                            for p in preps:
                                al = l2c[df["GT %s" % p][idx].lower()]
                                gt_labels[p][k] = al
                            num_annot += 1
                    print("...... done; read %d useful pair annotations" % num_annot)
        # Tie annotations to label structure.
        for p in preps:
            for fold, fold_struct, available in [["train", tr_robo_o, available_train],
                                                 ["test", te_robo_o, available_test]]:
                gt_found = 0
                for idx in range(len(fold_struct[p])):
                    key = (available[p][idx][1], available[p][idx][2])
                    if key in gt_labels[p]:
                        fold_struct[p][idx] = gt_labels[p][key]
                        gt_found += 1
                print("... done; %d / %d robot execution ground truth %s labels found for %s" %
                      (gt_found, len(fold_struct[p]), fold, p))

    # Read in human execution ground truth labels from annotation CSVs from multiple annotators and calculate agreement.
    tr_human_o = {p: [-1 for ix, _, _ in available_train[p]] for p in preps}
    te_human_o = {p: [-1 for ix, _, _ in available_test[p]] for p in preps}
    if args.exec_human_indir is not None:
        print("Reading in human execution ground truth labels from '" + args.exec_human_indir + "'...")
        annotators = []
        # indexed by p, then (aidx, bidx), then [annotator_id0_label, anntotor_id1_label, ...]
        annotations = {p: {} for p in preps}
        l2c = {"on": {"on": 1, "in": 0},
               "in": {"on": 1, "in": 1},
               "no": {"on": 0, "in": 0},
               "same": None, "-": None, "": None}
        for _, _, fns in os.walk(args.exec_human_indir):
            for fn in fns:
                ps = fn.split('.')
                if ps[-1] == 'csv':  # annotation file
                    annotator = ps[0]
                    if annotator not in annotators:
                        annotators.append(annotator)
                    fn = os.path.join(args.exec_human_indir, fn)
                    print("... reading annotator %s annotations from '%s'..." % (annotator, fn))
                    num_annot = 0
                    with open(fn, 'r') as f:
                        for lidx, line in enumerate(f.readlines()):
                            if lidx == 0:
                                continue
                            lp = line.strip().split(',')
                            aidx = names.index(lp[0])
                            bidx = names.index(lp[1])
                            k = (aidx, bidx)
                            al = l2c[lp[4]]
                            if al is not None:  # label is not "same" or blank, so we have an annotation to add
                                for p in preps:
                                    if k not in annotations[p]:
                                        annotations[p][k] = []
                                    annotations[p][k].append(al[p])
                                num_annot += 1
                    print("...... done; read %d useful pair annotations" % num_annot)
        # Reduce annotations to single values using majority vote and calculate kappa agreement.
        print("... taking majority vote among annotators and calculating agreement...")
        mv_annotations = {p: {} for p in preps}  # indexed p, (aidx, bidx), single value entry.
        for p in preps:
            lfk = [[] for _ in range(len(annotators))]  # labels for kappa calculation
            for k in annotations[p]:
                if len(annotations[p][k]) != len(annotators):
                    print("...... ERROR: %s pair (%s, %s) has %d / %d annotations" %
                          (p, names[k[0]], names[k[1]], len(annotations[p][k]), len(annotators)))
                c0 = annotations[p][k].count(0)
                c1 = annotations[p][k].count(1)
                if c1 > c0:  # favor 'no' in the event of a tie
                    mv_annotations[p][k] = 1
                else:
                    mv_annotations[p][k] = 0
                for an in range(len(annotators)):
                    lfk[an].append(annotations[p][k][an])
            kappas = [cohen_kappa_score(lfk[ani], lfk[anj]) for ani in range(len(annotators) - 1)
                      for anj in range(ani + 1, len(annotators))]
            print("...... inter-annotator Cohen's kappa avg for %s: %.3f +/- %.3f" % (p, np.average(kappas),
                                                                                np.std(kappas)))
            mvfk = [mv_annotations[p][k] for k in annotations[p]]
            kappas = [cohen_kappa_score(lfk[ani], mvfk) for ani in range(len(annotators))]
            print("...... annotator vs MV Cohen's kappa avg for %s: %.3f +/- %.3f" % (p, np.average(kappas),
                                                                                np.std(kappas)))
        # Tie annotations to label structure.
        for p in preps:
            for fold, fold_struct, available in [["train", tr_human_o, available_train],
                                                 ["test", te_human_o, available_test]]:
                gt_found = 0
                for idx in range(len(fold_struct[p])):
                    key = (available[p][idx][1], available[p][idx][2])
                    if key in mv_annotations[p]:
                        fold_struct[p][idx] = mv_annotations[p][key]
                        gt_found += 1
                print("... done; %d / %d human execution ground truth %s labels found for %s" %
                      (gt_found, len(fold_struct[p]), fold, p))

    # Prep language dictionary.
    print("Preparing infrastructure to include GloVe info...")
    word_to_i = {}
    word_to_i_all = {}
    i_to_word = {}
    maxlen = {}
    tr_enc_exps = {}
    te_enc_exps = {}
    for p in preps:
        tr_f = [[res[oidx], res[ojdx]] for _, oidx, ojdx in available_train[p]]
        te_f = [[res[oidx], res[ojdx]] for _, oidx, ojdx in available_test[p]]
        word_to_i[p], i_to_word[p], maxlen[p], tr_enc_exps[p], te_enc_exps[p] = make_lang_structures(tr_f, te_f)
        word_to_i_all[p], _, _, _, _ = make_lang_structures(tr_f, te_f, inc_test=True)
    print("... done")

    # https://github.com/pytorch/examples/blob/42e5b996718797e45c46a25c55b031e6768f8440/imagenet/main.py#L89-L101
    normalize = Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])  # bc of imagenet pretraining
    print("Preparing infrastructure to run ResNet-based feed forward model...")
    ob_to_vs = {}  # object names to list of numpy vectors resulting from representative images being run through ResNet
    resnet_m = resnet.resnet152(pretrained=True)
    plm = nn.Sequential(*list(resnet_m.children())[:-1])
    tt = ToTensor()
    v_width = None
    for idx in range(len(names)):
        img_fns = []
        img_dir = imgs[idx].replace(".jpeg", "")  # previously this stored a single image; now it points to a directory
        for _, _, fns in os.walk(img_dir):  # extract all five veiwpoints from image directory
            for fn in fns:
                if fn.split('.')[-1] == 'jpg':
                    img_fns.append(os.path.join(img_dir, fn))
        for img_fn in img_fns:
            ffn = img_fn + '.resnet'
            if os.path.isfile(ffn):
                with open(ffn, 'r') as f:
                    d = f.read().strip().split(' ')
                    v = np.array([float(n) for n in d])
                    v_width = len(v)
                    if names[idx] not in ob_to_vs:
                        ob_to_vs[names[idx]] = []
                    ob_to_vs[names[idx]].append(v)
            else:
                pil = Image.open(img_fn)
                pil = resize(pil, (224, 244))
                im = tt(pil)
                im = normalize(im)
                im = torch.unsqueeze(im, 0)
                v = plm(im).detach().data.numpy().flatten()
                v_width = len(v)
                if names[idx] not in ob_to_vs:
                    ob_to_vs[names[idx]] = []
                ob_to_vs[names[idx]].append(v)
                with open(ffn, 'w') as f:
                    f.write(' '.join([str(i) for i in v]))
    print("... done")

    # Average GLoVe embeddings concatenated and used to predict class.
    print("Preparing infrastructure to run GLoVe-based feed forward model...")
    ws = set()
    for p in preps:
        ws.update(word_to_i_all[p].keys())
    g, missing = get_glove_vectors(args.glove_infile, ws)
    l_width = len(g['unk'])
    print("... done; missing " + str(missing) + " vectors out of " + str(len(ws)))

    for p in preps:
        print("Preparing tensors for " + p)

        print("... preparing output tensors...")
        tr_outputs = torch.tensor(tr_o[p], dtype=torch.long).view(len(tr_o[p]), 1).numpy()
        te_outputs = torch.tensor(te_o[p], dtype=torch.long).view(len(te_o[p]), 1).numpy()
        tr_robo_outputs = torch.tensor(tr_robo_o[p], dtype=torch.long).view(len(tr_robo_o[p]), 1).numpy()
        te_robo_outputs = torch.tensor(te_robo_o[p], dtype=torch.long).view(len(te_robo_o[p]), 1).numpy()
        tr_human_outputs = torch.tensor(tr_human_o[p], dtype=torch.long).view(len(tr_human_o[p]), 1).numpy()
        te_human_outputs = torch.tensor(te_human_o[p], dtype=torch.long).view(len(te_human_o[p]), 1).numpy()
        print("...... done")

        if args.rgbd_only:
            print("... preparing RGBD inputs...")
            # Prepare input to model.
            tr_f_rgbd = [rgbd_tr[str(oidx)][str(ojdx)] for _, oidx, ojdx in available_train[p]]
            te_f_rgbd = [rgbd_te[str(oidx)][str(ojdx)] for _, oidx, ojdx in available_test[p]]

            # Convert structured data into tensor inputs.
            # Dimensions:
            # 5 trials (maximum) per object pair
            # 3 input channels for RGB, 1 for depth
            # 48 x 64 is the region size downsampled from the camera.
            tr_inputs_rgb = np.zeros((len(available_train[p]), 5, 4, 48, 64))
            tr_inputs_d = np.zeros((len(available_train[p]), 5, 1, 48, 64))
            te_inputs_rgb = np.zeros((len(available_test[p]), 5, 4, 48, 64))
            te_inputs_d = np.zeros((len(available_test[p]), 5, 1, 48, 64))
            for rgb_in, d_in, orig_in in [[tr_inputs_rgb, tr_inputs_d, tr_f_rgbd],
                                          [te_inputs_rgb, te_inputs_d, te_f_rgbd]]:
                for idx in range(len(orig_in)):
                    # Convert RGB and D numpy arrays to tensors.
                    rgb_in[idx] = torch.tensor(orig_in[idx][0], dtype=torch.float).numpy()
                    d_in[idx] = torch.tensor(orig_in[idx][1], dtype=torch.float).numpy()
            print("...... done")
        else:
            tr_inputs_rgb = tr_inputs_d = te_inputs_rgb = te_inputs_d = None

        print("... preparing GloVe inputs...")
        # tr_inputs_l = np.zeros((len(available_train[p]), l_width * 2))  # concatenate
        # te_inputs_l = np.zeros((len(available_test[p]), l_width * 2))  # concatenate
        tr_inputs_l = np.zeros((len(available_train[p]), l_width))  # diff
        te_inputs_l = np.zeros((len(available_test[p]), l_width))  # diff
        tr_f_l = [[res[oidx], res[ojdx]] for _, oidx, ojdx in available_train[p]]
        te_f_l = [[res[oidx], res[ojdx]] for _, oidx, ojdx in available_test[p]]
        for model_in, orig_in in [[tr_inputs_l, tr_f_l], [te_inputs_l, te_f_l]]:
            for idx in range(len(orig_in)):
                # For each object, take the average GloVe embedding of all tokens used in all referring expressions
                # for that object. The representation for this pair is the concatenation of those two average embeddings
                ob1_tks = [tk for re in orig_in[idx][0] for tk in re]
                avg_ob1_v = np.sum([g[w] for w in ob1_tks], axis=0) / len(ob1_tks)
                ob2_tks = [tk for re in orig_in[idx][1] for tk in re]
                avg_ob2_v = np.sum([g[w] for w in ob2_tks], axis=0) / len(ob2_tks)
                # inl = np.concatenate((avg_ob1_v, avg_ob2_v))  # concatenate
                inl = avg_ob1_v - avg_ob2_v  # diff
                model_in[idx] = torch.tensor(inl, dtype=torch.float).numpy()
        print("...... done")

        print("... preparing ResNet inputs...")
        # tr_inputs_v = np.zeros((len(available_train[p]), v_width * 2))  # concatenate
        # te_inputs_v = np.zeros((len(available_test[p]), v_width * 2))  # concatenate
        tr_inputs_v = np.zeros((len(available_train[p]), v_width))  # diff
        te_inputs_v = np.zeros((len(available_test[p]), v_width))  # diff
        tr_f_v = [[oidx, ojdx] for _, oidx, ojdx in available_train[p]]
        te_f_v = [[oidx, ojdx] for _, oidx, ojdx in available_test[p]]
        for model_in, orig_in in [[tr_inputs_v, tr_f_v], [te_inputs_v, te_f_v]]:
            for idx in range(len(orig_in)):
                # For each object, take the average ResNet embedding of all the image views available for that object
                # as its representation, and represent the pair as the concatenation of those two averages.
                avg_ob1_v = np.sum(ob_to_vs[names[orig_in[idx][0]]], axis=0) / len(ob_to_vs[names[orig_in[idx][0]]])
                avg_ob2_v = np.sum(ob_to_vs[names[orig_in[idx][1]]], axis=0) / len(ob_to_vs[names[orig_in[idx][1]]])
                # inv = np.concatenate((avg_ob1_v, avg_ob2_v))  # concatenate
                inv = avg_ob1_v - avg_ob2_v  # diff
                model_in[idx] = torch.tensor(inv, dtype=torch.float).numpy()
        print("...... done")

        # Write resulting vectors to json file as numpy.
        out_fn = args.out_fn_prefix + "." + p
        print("... writing tensors as numpy jsons to '" + out_fn + "' for " + p)
        with open(out_fn, 'w') as f:
            d = {"train":
                 {"mturk_label": tr_outputs.tolist(),
                  "robo_label": tr_robo_outputs.tolist(),
                  "human_label": tr_human_outputs.tolist(),
                  "names": tr_names[p],
                  "lang": tr_inputs_l.tolist(),
                  "vis": tr_inputs_v.tolist(),
                  "rgb": tr_inputs_rgb.tolist() if tr_inputs_rgb is not None else None,
                  "d": tr_inputs_d.tolist() if tr_inputs_d is not None else None},
                 "test":
                 {"mturk_label": te_outputs.tolist(),
                  "robo_label": te_robo_outputs.tolist(),
                  "human_label": te_human_outputs.tolist(),
                  "names": te_names[p],
                  "lang": te_inputs_l.tolist(),
                  "vis": te_inputs_v.tolist(),
                  "rgb": te_inputs_rgb.tolist() if te_inputs_rgb is not None else None,
                  "d": te_inputs_d.tolist() if te_inputs_d is not None else None}}
            json.dump(d, f)
        print("... done")

if __name__ == "__main__":

    parser = argparse.ArgumentParser()
    parser.add_argument('--infile', type=str, required=True,
                        help="input json file with train/dev/test split")
    parser.add_argument('--metadata_infile', type=str, required=True,
                        help="input json file with object metadata")
    parser.add_argument('--glove_infile', type=str, required=True,
                        help="input glove vector text file")
    parser.add_argument('--robot_infile', type=str, required=True,
                        help="input robot feature file")
    parser.add_argument('--rgbd_only', type=int, required=True,
                        help="whether to restrict to extracting rgbd data")
    parser.add_argument('--exec_robo_indir', type=str, required=False,
                        help="input dir of robot execution ground truth affordance labels csvs")
    parser.add_argument('--exec_human_indir', type=str, required=False,
                        help="input dir of human annotator execution ground truth affordance labels csvs")
    parser.add_argument('--out_fn_prefix', type=str, required=True,
                        help="where to write the json output torch data")
    parser.add_argument('--test', type=int, required=False,
                        help="if set to 1, evaluates on the test set; NOT FOR TUNING")
    main(parser.parse_args())


# for x in range(10):
#     for y in range(10):
#         print(x*y)
#         if x*y > 50:
#             break
#     else:
#         continue    # only executed if the inner loop DID NOT break
#     break    # only executed if the inner loop DID break

# # a = [1, 1, 3, 1, 5, 1, 1, 1, 1]
# a = [1, 3, 1, 1]
# b = [1, 1, 3, 1, 1, 1, 1]
# index = 0
# for aa in a:
#     for bb in b:
#         print(index)
#         index += 1
#         if aa == 3:
#             print("found")
#             break
#     else:
#         continue

# test of enumerate
# my_list = [1, 3, 4, 5, 7, 8, 2, 11, 19]
# for c, value in enumerate(my_list, 0):
#     # print(c, value)
#     print(value)

# test of groupby
from itertools import groupby

things = [("animal", "bear"), ("animal", "duck"), ("plant", "cactus"), ("plant", "flower"), ("vehicle", "speed boat"), ("vehicle", "school bus")]

for key, group in groupby(things, lambda x: x[0]):
    # print(group)
    # print(key)
    for thing in group:
        print(thing)
        # print("A %s is a %s." % (thing[1], key))
    print("\n")