from __future__ import annotations

import copy
import hashlib
import itertools
import json
import pathlib
import sys

from typing import Any, TYPE_CHECKING

if TYPE_CHECKING:
    from hashlib import _Hash
    from modules.config import Config
    from modules.dats import Dat, DatNode
    from modules.input import UserInput

from modules.titletools import Removes, TitleTools, TraceTools
from modules.utils import download, eprint, ExitRetool, Font, printwrap, regex_test

class CloneList:
    def __init__(self,
                 min_retool_version: str='2.00',
                 categories: list[dict[str, Any]] = [],
                 mias: list[str] = [],
                 overrides: list[dict[str, str|dict[str, str|list[str]]]] = [],
                 removes: list[dict[str, str]] = [],
                 variants: list[dict[str, Any]] = []):
        """ Creates an object that contains data originally stored in a clone list.

        Args:
            - `min_retool_version (str, optional)` The minimum Retool version required to
              process the imported clone list. Defaults to `2.00`.

            - `categories (list[dict[str, str|list[str]]], optional)` A dictionary to store the
              `categories` object found in a related clone list, if it exists. Defaults to
              `{}`.

            - `mias (list[str], optional)` A list to store the `mias` arrary found in a
              related clone list, if it exists. Defaults to `[]`.

            - `overrides (list[dict[str, str|dict[str, str|list[str]]]], optional)` A dictionary
              to store the `overrides` object found in a related clone list, if it exists.
              Defaults to `{}`.

            - `removes (dict[str, dict[str, Any]], optional)` A dictionary to store the
              `removes` object found in a related clone list, if it exists. Defaults to
              `{}`.

            - `variants (dict[str, list[dict[str, Any]]], optional)` A dictionary to store
              the `variants` object found in a related clone list, if it exists. Defaults to
              `{}`.
        """

        self.min_retool_version: str = min_retool_version
        self.categories: list[dict[str, Any]] = categories
        self.mias: list[str] = mias
        self.overrides: list[dict[str, Any]] = overrides
        self.removes: list[dict[str, str]] = removes
        self.variants: list[dict[str, Any]] = variants

class CloneListTools(object):
    """ Methods for applying clone list entries to a dictionary of DatNode titles. """

    @staticmethod
    def categories(processed_titles: dict[str, set[DatNode]], config: Config, input_dat: Dat) -> dict[str, set[DatNode]]:
        """ Applies new categories to a dictionary of DatNode titles, defined by the
        related clone list.

        Args:
            - `processed_titles (dict[str, set[DatNode]])` A work in progress dictionary
              of DatNodes, originally populated from the input DAT and actively being worked
              on by Retool.

            - `config (Config)` The Retool config object.

            - `input_dat (Dat)` The Retool input_dat object.

        Returns:
            `dict[str, set[DatNode]]` A dictionary of DatNodes with new categories
            assigned based on the related clone list.
        """

        if input_dat.clone_list.categories:
            eprint('* Applying clone list categories... ', sep=' ', end='', flush=True)

            report_on_match: bool = False
            warning_given: bool = False
            name_type: str = 'tagFree'
            categories: list[str] = []
            missing_titles: set[str] = set()

            # Check the name type we're matching for, and the categories to change the
            # title to
            for value in input_dat.clone_list.categories:
                if 'nameType' in value:
                    name_type = value['nameType']

                    if not (name_type == 'full'
                        or name_type == 'short'
                        or name_type == 'regionFree'
                        or name_type == 'tagFree'
                        or name_type == 'regex'):
                            name_type = 'tagFree'

                if 'categories' in value:
                    if type(value['categories']) is list:
                        categories = value['categories']
                    else:
                        if config.user_input.verbose:
                            warning_given = True
                            printwrap(
                                f'{Font.warning}* The following category title\'s '
                                f'"categories" key isn\'t an array and will be skipped:',
                                'error')
                            eprint(f'\n  {value["searchTerm"]}{Font.end}')

                            if config.user_input.warningpause:
                                eprint(f'\n{Font.disabled}Press enter to continue{Font.end}')
                                input()
                        continue
                else:
                    if config.user_input.verbose:
                        warning_given = True
                        printwrap(
                            f'{Font.warning}* The following category title has no '
                            f'"categories" key and will be skipped:', 'error')
                        eprint(f'\n  {value["searchTerm"]}{Font.end}')

                        if config.user_input.warningpause:
                            eprint(f'\n{Font.disabled}Press enter to continue{Font.end}')
                            input()
                    continue

                # Look up the title in the dictionary, then process the required changes
                if name_type == 'regex':
                    valid_regex: list[str] = regex_test([value['searchTerm']], 'categories', 'clone list')

                    if not valid_regex:
                        continue

                    found_titles = TitleTools.find_title(value['searchTerm'], name_type, processed_titles, missing_titles, config, deep_search=True)
                else:
                    found_titles = TitleTools.find_title(value['searchTerm'], name_type, processed_titles, missing_titles, config)

                if config.user_input.trace:
                    report_on_match = TraceTools.trace_enable(set(found_titles), config.user_input.trace)

                for title in found_titles:
                    if title in processed_titles[title.group_name]:
                        if report_on_match:
                            eprint('')
                            TraceTools.trace_title('REF0053')
                            eprint(f'* {title.full_name}')
                            eprint(f'  New categories: {categories}\n{Font.disabled}  Old categories: {title.categories}{Font.end}')
                            eprint(f'\n{Font.disabled}Press enter to continue{Font.end}')
                            input()

                        title.categories = categories

            if missing_titles and config.user_input.verbose:
                eprint('')
                printwrap(
                    f'{Font.warning}* The following category titles in the '
                    'clone list can\'t be found in the input DAT and will '
                    'be skipped:', 'error')

                eprint('')

                for missing_title in sorted(missing_titles):
                    eprint(f'  *  {missing_title}')

                eprint(f'{Font.end}')

                if config.user_input.warningpause:
                    eprint(f'\n{Font.disabled}Press enter to continue{Font.end}')
                    input()

            if (
                (warning_given
                 or missing_titles)
                and config.user_input.verbose):
                eprint('* Applying clone list categories... done')
            else:
                eprint('done.')

        return processed_titles

    @staticmethod
    def compare_priorities(title_set: set[DatNode], report_on_match: bool) -> set[DatNode]:
        """ Compare any two titles from a set of DatNodes, and select the one
        with the lowest priority number set in a clone list.

        Args:
            - `title_set (set[DatNode])` A set of titles as DatNode instances.

            - `report_on_match (bool)` Whether Retool needs to report any titles being
            traced.

        Returns:
            `set[DatNode]` A set of DatNodes filtered by clone list priority.
        """

        title_set_temp: set[DatNode] = title_set.copy()

        for title_1, title_2 in itertools.combinations(title_set_temp, 2):
            if (
                title_1.short_name == title_2.short_name
                and title_1 in title_set
                and title_2 in title_set):
                    if not (
                        # Compare non-superset priority titles
                        title_1.is_superset
                        and title_2.is_superset):
                            if title_1.clonelist_priority < title_2.clonelist_priority:
                                if report_on_match:
                                    TraceTools.trace_title('REF0020')
                                    eprint(f'+ Keeping:  ({title_1.clonelist_priority}) {title_1.full_name}')
                                    eprint(f'{Font.disabled}- Removing: ({title_2.clonelist_priority}) {title_2.full_name}{Font.end}')
                                    eprint(f'\n{Font.disabled}Press enter to continue{Font.end}')
                                    input()

                                title_set.remove(title_2)
                            elif title_2.clonelist_priority < title_1.clonelist_priority:
                                if report_on_match:
                                    TraceTools.trace_title('REF0021')
                                    eprint(f'+ Keeping:  ({title_2.clonelist_priority}) {title_2.full_name}')
                                    eprint(f'{Font.disabled}- Removing: ({title_1.clonelist_priority}) {title_1.full_name}{Font.end}')
                                    eprint(f'\n{Font.disabled}Press enter to continue{Font.end}')
                                    input()

                                title_set.remove(title_1)
                    else:
                        # Compare superset titles
                        if title_1.clonelist_priority < title_2.clonelist_priority:
                            if report_on_match:
                                TraceTools.trace_title('REF0022')
                                eprint(f'+ Keeping:  ({title_1.clonelist_priority}) {title_1.full_name}')
                                eprint(f'{Font.disabled}- Removing: ({title_2.clonelist_priority}) {title_2.full_name}{Font.end}')
                                eprint(f'\n{Font.disabled}Press enter to continue{Font.end}')
                                input()

                            title_set.remove(title_2)
                        elif title_2.clonelist_priority < title_1.clonelist_priority:
                            if report_on_match:
                                TraceTools.trace_title('REF0023')
                                eprint(f'+ Keeping:  ({title_2.clonelist_priority}) {title_2.full_name}')
                                eprint(f'{Font.disabled}- Removing: ({title_1.clonelist_priority}) {title_1.full_name}{Font.end}')
                                eprint(f'\n{Font.disabled}Press enter to continue{Font.end}')
                                input()

                            title_set.remove(title_1)

        return title_set

    @staticmethod
    def mias(processed_titles: dict[str, set[DatNode]], config: Config, input_dat: Dat) -> dict[str, set[DatNode]]:
        """ Applies MIA tags to a dictionary of DatNode titles, defined by the related
        clone list.

        Args:
            - `processed_titles (dict[str, set[DatNode]])` A work in progress dictionary
              of DatNodes, originally populated from the input DAT and actively being worked
              on by Retool.

            - `config (Config)` The Retool config object.

            - `input_dat (Dat)` The Retool input_dat object.

        Returns:
            `dict[str, set[DatNode]]` A dictionary of DatNodes with MIA tags applied
            based on the related clone list.
        """

        if input_dat.clone_list.mias:
            eprint('* Applying clone list MIA tags... ', sep=' ', end='', flush=True)

            report_on_match: bool = False
            warning_given: bool = False
            missing_titles: set[str] = set()

            name_type = 'full'

            # Find the MIA titles
            for mia in input_dat.clone_list.mias:
                if type(mia) is not str:
                    if config.user_input.verbose:
                        warning_given = True
                        printwrap(
                            f'{Font.warning}* The following MIA title entry isn\'t a '
                            f'string and will be skipped:',
                            'error')
                        eprint(f'  {mia}{Font.end}')

                        if config.user_input.warningpause:
                            eprint(f'\n{Font.disabled}Press enter to continue{Font.end}')
                            input()
                    continue

                # Look up the title in the dictionary, then process the required changes
                found_titles = TitleTools.find_title(mia, name_type, processed_titles, missing_titles, config)

                if config.user_input.trace:
                    report_on_match = TraceTools.trace_enable(set(found_titles), config.user_input.trace)

                for title in found_titles:
                    if title in processed_titles[title.group_name]:
                        title.is_mia = True

                        for rom in title.roms:
                            rom['mia'] = 'yes'

                            if report_on_match:
                                eprint('')
                                TraceTools.trace_title('REF0085', [title.full_name], set(), keep_remove=False)
                                eprint(f'* {rom["name"]}')

                        if report_on_match:
                            eprint('')
                            TraceTools.trace_title('REF0063', [], set([title]), keep_remove=False)

            if missing_titles and config.user_input.verbose:
                eprint('')
                printwrap(
                    f'{Font.warning}* The following MIA titles in the '
                    'clone list can\'t be found in the input DAT and will '
                    'be skipped:', 'error')

                eprint('')

                for missing_title in sorted(missing_titles):
                    eprint(f'  *  {missing_title}')

                eprint(f'{Font.end}')

                if config.user_input.warningpause:
                    eprint(f'\n{Font.disabled}Press enter to continue{Font.end}')
                    input()

            if (
                (warning_given
                or missing_titles)
                and config.user_input.verbose):
                eprint('* Applying clone list MIA tags... done')
            else:
                eprint('done.')

        return processed_titles

    @staticmethod
    def overrides(processed_titles: dict[str, set[DatNode]], config: Config, input_dat: Dat) -> dict[str, set[DatNode]]:
        """ Overrides the default groups and short names assigned to titles by Retool as
        defined by the related clone list. Overridden groups can either be applied
        directly, or conditionally based on the user's region order.

        Args:
            - `processed_titles (dict[str, set[DatNode]])` A work in progress dictionary
              of DatNodes, originally populated from the input DAT and actively being worked
              on by Retool.

            - `config (Config)` The Retool config object.

            - `input_dat (Dat)` The Retool input_dat object.

        Returns:
            `dict[str, set[DatNode]]` A dictionary of DatNodes with groups overriden as
            defined by the related clone list.
        """

        if input_dat.clone_list.overrides:
            eprint('* Applying clone list overrides... ', sep=' ', end='', flush=True)

            report_on_match: bool = False
            warning_given: bool = False

            # Check the name type we're matching for, and the group to move the title to
            missing_titles: set[str] = set()

            for value in input_dat.clone_list.overrides:
                condition_processed: bool = True
                name_type: str = 'tagFree'
                else_group: str = ''
                priority: int = 0

                if 'nameType' in value:
                    name_type = value['nameType']

                    if not (
                        name_type == 'full'
                        or name_type == 'short'
                        or name_type == 'regionFree'
                        or name_type == 'tagFree'
                        or name_type == 'regex'):
                            name_type = 'tagFree'

                if 'newGroup' in value:
                    if type(value['newGroup']) is not str:
                        if config.user_input.verbose:
                            warning_given = True
                            printwrap(
                                f'{Font.warning}* The following override\'s "newGroup" '
                                f'key isn\'t a string and will be skipped:', 'error')
                            eprint(f'\n  {value["searchTerm"]}{Font.end}')

                            if config.user_input.warningpause:
                                eprint(f'\n{Font.disabled}Press enter to continue{Font.end}')
                                input()
                        continue
                else:
                    if config.user_input.verbose:
                        warning_given = True
                        printwrap(
                            f'{Font.warning}* The following override has no "newGroup" '
                            f'key and will be skipped:', 'error')
                        eprint(f'\n  {value["searchTerm"]}{Font.end}')

                        if config.user_input.warningpause:
                            eprint(f'\n{Font.disabled}Press enter to continue{Font.end}')
                            input()
                    continue

                # Look up the title in the dictionary, then process the required changes
                if name_type == 'regex':
                    valid_regex:list[str] = regex_test([value['searchTerm']], 'overrides', 'clone list')

                    if not valid_regex:
                        continue

                    found_titles = TitleTools.find_title(value['searchTerm'], name_type, processed_titles, missing_titles, config, deep_search=True)
                else:
                    found_titles = TitleTools.find_title(value['searchTerm'], name_type, processed_titles, missing_titles, config)

                if config.user_input.trace:
                    report_on_match = TraceTools.trace_enable(set(found_titles), config.user_input.trace)

                for title in found_titles:
                    if report_on_match:
                        eprint('')
                        TraceTools.trace_title('REF0056')
                        eprint(f'\n* {title.full_name}')

                    # Get the new group the title will be moved to
                    new_group: str = TitleTools.get_group_name(value['newGroup'], config)

                    # Check if the override is conditional
                    if 'condition' in value:
                        regions: list[str] = []
                        higher_than_regions: list[str] = []
                        condition_processed = False

                        # Look for the regions that need to be higher in priority than
                        # those defined in "higher than"
                        if 'regionOrder' in value['condition']:
                            if type(value['condition']['regionOrder']) is dict:
                                if 'higherRegions' in value['condition']['regionOrder']:
                                    if type(value['condition']['regionOrder']['higherRegions']) is list:
                                        regions = value['condition']['regionOrder']['higherRegions']

                                        if report_on_match:
                                            eprint('\n  IF any of these regions:\n')

                                            for condition_region in regions:
                                                eprint(f'    * {condition_region}')
                                    else:
                                        printwrap(
                                            f'{Font.warning}* The following override\'s '
                                            f'"higherRegions" key isn\'t an array and will be '
                                            'skipped:', 'error')
                                        eprint(f'\n  {value["searchTerm"]}{Font.end}')

                                        if config.user_input.warningpause:
                                            eprint(f'\n{Font.disabled}Press enter to continue{Font.end}')
                                            input()
                                        continue
                                else:
                                    warning_given = True
                                    printwrap(
                                        f'{Font.warning}* The following override has no '
                                        '"higherRegions" key inside the "regionOrder" key, and '
                                        'will be skipped:', 'error')
                                    eprint(f'\n  {value["searchTerm"]}{Font.end}')

                                    if config.user_input.warningpause:
                                        eprint(f'\n{Font.disabled}Press enter to continue{Font.end}')
                                        input()
                                    continue

                                # Look for the regions that need to be lower in priority than the
                                # those defined in "region"
                                if 'lowerRegions' in value['condition']['regionOrder']:
                                    if type(value['condition']['regionOrder']['lowerRegions']) is list:
                                        higher_than_regions = value['condition']['regionOrder']['lowerRegions']

                                        if report_on_match:
                                            eprint('\n  Are higher than all of these regions:\n')

                                            for condition_region in higher_than_regions:
                                                eprint(f'    * {condition_region}')
                                    else:
                                        warning_given = True
                                        printwrap(
                                            f'{Font.warning}* The following override\'s '
                                            '"lowerRegions" key isn\'t an array and will be '
                                            'skipped:', 'error')
                                        eprint(f'\n  {value["searchTerm"]}{Font.end}')

                                        if config.user_input.warningpause:
                                            eprint(f'\n{Font.disabled}Press enter to continue{Font.end}')
                                            input()
                                        continue
                                else:
                                    warning_given = True
                                    printwrap(
                                        f'{Font.warning}* The following override has no '
                                        '"lowerRegions" key inside the "regionOrder" key, '
                                        'and will be skipped:', 'error')
                                    eprint(f'\n  {value["searchTerm"]}{Font.end}')

                                    if config.user_input.warningpause:
                                        eprint(f'\n{Font.disabled}Press enter to continue{Font.end}')
                                        input()
                                    continue

                            # Look for a "priority" entry, which reassigns a title's priority if the condition is true
                            if 'priority' in value['condition']['regionOrder']:
                                if type(value['condition']['regionOrder']['priority']) is not int:
                                    if config.user_input.verbose:
                                        warning_given = True
                                        printwrap(
                                            f'{Font.warning}* The following override\'s "priority" '
                                            f'key isn\'t an integer and will be skipped:', 'error')
                                        eprint(f'\n  {value["searchTerm"]}{Font.end}')

                                        if config.user_input.warningpause:
                                            eprint(f'\n{Font.disabled}Press enter to continue{Font.end}')
                                            input()
                                    continue
                                else:
                                    priority = value['condition']['regionOrder']['priority']

                            # Look for an "elseGroup" entry, which reassigns a title's group
                            # if the condition isn't true
                            if report_on_match:
                                eprint(f'\n  THEN move to this group: {Font.bold}{new_group}{Font.end}')

                                if priority:
                                    eprint(f'  AND set priority to: {Font.bold}{value["condition"]["regionOrder"]["priority"]}{Font.end}\n')

                            if 'elseGroup' in value['condition']['regionOrder']:
                                if type(value['condition']['regionOrder']['elseGroup']) is str:
                                    else_group = TitleTools.get_group_name(value['condition']['regionOrder']['elseGroup'], config)

                                    if report_on_match:
                                        eprint(f'  ELSE move to this group: {Font.bold}{else_group}{Font.end}')
                                        if priority:
                                            eprint(f'  AND set priority to: {Font.bold}2{Font.end}\n')
                                else:
                                    warning_given = True
                                    printwrap(
                                        f'{Font.warning}* The following override\'s '
                                        '"elseGroup" key isn\'t a string and will be '
                                        'skipped:', 'error')
                                    eprint(f'\n  {value["searchTerm"]}{Font.end}')

                                    if config.user_input.warningpause:
                                        eprint(f'\n{Font.disabled}Press enter to continue{Font.end}')
                                        input()
                            else:
                                if report_on_match:
                                    eprint(f'  ELSE use the default group: {Font.bold}{title.group_name}{Font.end}')
                                    if priority:
                                        eprint(f'  AND set priority to: {Font.bold}2{Font.end}\n')
                        else:
                            warning_given = True
                            printwrap(
                                f'{Font.warning}* The following override has no '
                                f'"regionOrder" key inside its condition, and will be '
                                f'skipped:', 'error')
                            eprint(f'\n  {value["searchTerm"]}{Font.end}')

                            if config.user_input.warningpause:
                                eprint(f'\n{Font.disabled}Press enter to continue{Font.end}')
                                input()
                            continue

                        # Check if a system config is in play
                        region_order: list[str] = config.region_order_user

                        if config.system_region_order_user:
                            if {'override': 'true'} in config.system_region_order_user:
                                region_order = [str(x) for x in config.system_region_order_user if 'override' not in x]

                        # Check that the regions are available in the user's current region
                        # order, and store their priority if so
                        higher_regions: list[int] = [i for i, region in enumerate(region_order) if region in regions]
                        lower_regions: list[int] = [i for i, region in enumerate(region_order) if region in higher_than_regions]

                        # If any of the higher regions is higher than ALL of the lower regions
                        # move the title to the new group. If this isn't the case, and there's
                        # an "elseGroup" entry, move the title to the "elseGroup". Otherwise,
                        # do nothing.
                        if higher_regions and lower_regions:
                            lower_regions_lowest: int = int(sorted(lower_regions)[0])

                            for higher_region in higher_regions:
                                if higher_region < lower_regions_lowest:
                                    if not condition_processed:
                                        if report_on_match:
                                            eprint(f'  CONDITION is true, moving to group: {Font.bold}{new_group}{Font.end}')
                                            if priority:
                                                eprint(f'  Changing to priority: {Font.bold}{priority}{Font.end}')

                                    condition_processed = True

                            if not condition_processed:
                                if else_group:
                                    new_group = else_group
                                    condition_processed = True

                                    if report_on_match:
                                        eprint(f'  CONDITION is false, moving to group: {Font.bold}{new_group}{Font.end}')
                                        if priority:
                                            eprint(f'  Changing to priority: {Font.bold}2{Font.end}')
                                if priority:
                                    priority = 2
                    else:
                        if report_on_match:
                            eprint(f'  New group: {Font.bold}{new_group}{Font.end}\n{Font.disabled}  Old group: {Font.bold}{title.group_name}{Font.end}')

                    # Assign the new group and if it's available, priority
                    if condition_processed:
                        if report_on_match:
                            if config.user_input.warningpause:
                                eprint(f'\n{Font.disabled}Press enter to continue{Font.end}')
                                input()

                        if title in processed_titles[title.group_name]:
                            if new_group not in processed_titles:
                                processed_titles[new_group] = set()

                            processed_titles[title.group_name].remove(title)

                            title.group_name = new_group

                            if title.short_name.endswith('(demo)'):
                                title.short_name = f'{new_group} (demo)'
                            else:
                                title.short_name = new_group

                            if priority:
                                title.clonelist_priority = priority

                            processed_titles[new_group].add(title)
                    else:
                        if report_on_match:
                            eprint(f'  CONDITION is false, group remains as: {Font.bold}{title.group_name}{Font.end}')
                            if priority:
                                eprint(f'  Changing to priority: 2')

                            if config.user_input.warningpause:
                                eprint(f'\n{Font.disabled}Press enter to continue{Font.end}')
                                input()

                        if title in processed_titles[title.group_name]:
                            if priority:
                                title.clonelist_priority = 2

            if missing_titles and config.user_input.verbose:
                eprint('')
                printwrap(
                    f'{Font.warning}* The following override titles in the '
                    'clone list can\'t be found in the input DAT and will '
                    'be skipped:', 'error')

                eprint('')

                for missing_title in sorted(missing_titles):
                    eprint(f'  *  {missing_title}')

                eprint(f'{Font.end}')

                if config.user_input.warningpause:
                    eprint(f'\n{Font.disabled}Press enter to continue{Font.end}')
                    input()

            if (
                (warning_given
                or missing_titles)
                and config.user_input.verbose):
                    eprint('* Applying clone list overrides... done')
            else:
                eprint('done.')

        return processed_titles

    @staticmethod
    def removes(processed_titles: dict[str, set[DatNode]], config: Config, input_dat: Dat, removes: Removes) -> dict[str, set[DatNode]]:
        """ Removes titles from a dictionary of DatNodes, as defined by the related clone
        list. As a general rule, removes are a nuclear option, as they completely take the
        title out of contention, destroying any relationships set up by Retool. Another
        method should be attempted first.

        Args:
            - `processed_titles (dict[str, set[DatNode]])` A work in progress dictionary
              of DatNodes, originally populated from the input DAT and actively being worked
              on by Retool.

            - `config (Config)` The Retool config object.

            - `input_dat (Dat)` The Retool input_dat object.

            - `removes (Removes)` A Removes object that contains and categorizes all the
              titles that have been removed from consideration. Is used for stats and other
              output files generated by Retool.

        Returns:
            `dict[str, set[DatNode]]` A dictionary of DatNodes with titles removed as
            defined by the related clone list.
        """

        report_on_match: bool = False

        if input_dat.clone_list.removes:
            eprint('* Applying clone list removes... ', sep=' ', end='', flush=True)

            report_on_match = False
            name_type: str = 'tagFree'
            missing_titles: set[str] = set()
            removes_count: int = 0

            # Check the name type we're matching for, and the titles to remove
            for value in input_dat.clone_list.removes:
                if 'nameType' in value:
                    name_type = value['nameType']

                    if not (name_type == 'full'
                        or name_type == 'short'
                        or name_type == 'regionFree'
                        or name_type == 'tagFree'):
                            name_type = 'tagFree'

                # Look up the title in the dictionary, then process the required changes
                found_titles = TitleTools.find_title(value['searchTerm'], name_type, processed_titles, missing_titles, config)

                if config.user_input.trace:
                    report_on_match = TraceTools.trace_enable(set(found_titles), config.user_input.trace)

                for title in found_titles:
                    if title in processed_titles[TitleTools.get_group_name(value['searchTerm'], config)]:

                        if report_on_match:
                            eprint('')
                            TraceTools.trace_title('REF0054')
                            eprint(f'{Font.disabled}- Remove: {title.full_name}{Font.end}')
                            eprint(f'\n{Font.disabled}Press enter to continue{Font.end}')
                            input()

                        processed_titles[TitleTools.get_group_name(value['searchTerm'], config)].remove(title)
                        title.exclude_reason = 'Clone list remove'
                        removes.clonelist_removes.add(title)
                        removes_count += 1

            if missing_titles and config.user_input.verbose:
                eprint('')
                printwrap(
                    f'{Font.warning}* The following remove titles in the '
                    'clone list can\'t be found in the input DAT and will '
                    'be skipped:', 'error')

                eprint('')

                for missing_title in sorted(missing_titles):
                    eprint(f'  *  {missing_title}')

                eprint(f'{Font.end}')

                if config.user_input.warningpause:
                    eprint(f'\n{Font.disabled}Press enter to continue{Font.end}')
                    input()

                eprint('* Applying clone list removes... done')
            else:
                eprint('done.')

            config.stats.removes_count = removes_count

        return processed_titles

    @staticmethod
    def update_clonelists_metadata(config: Config, gui_input: UserInput|None, no_exit: bool = False) -> None:
        """ Downloads the latest clone lists and support files

        Args:
            - `config (Config)` The Retool config object.

            - `gui_input (UserInput)` Used to determine whether or not the update is being
              run from the GUI. If so, check if a custom download location has been set in
              user-config.yaml.

        Raises:
            `ExitRetool` Silently exit if run from the GUI, so UI elements can
            re-enable.
        """

        # Download the latest internal-config.json and datafile.dtd
        download_location: str = config.clone_list_metadata_download_location

        if gui_input:
            download_location = config.user_input.user_clone_list_metadata_download_location

        failed: bool = False

        eprint(f'* Downloading {Font.bold}{config.config_file}{Font.end}... ')

        failed = download(f'{download_location}/{config.config_file}', str(pathlib.Path(f'{config.config_file}')))

        if not failed:
            eprint(f'\033[F\033[K* Downloading {Font.bold}{config.config_file}{Font.end}... done.')

        eprint(f'* Downloading {Font.bold}datafile.dtd{Font.end}... ')

        failed = download(f'{download_location}/datafile.dtd', str(pathlib.Path('datafile.dtd')))

        if not failed:
            eprint(f'\033[F\033[K* Downloading {Font.bold}datafile.dtd{Font.end}... done.')

        def get_updates(local_path: str, update_name: str, download_url: str, file_count: int) -> int:
            """ An online updater for clone lists and metadata. Downloads a specific
            hash.json file, which contains names and hashes for a set of files.
            Then compares these details with the hashes of local files found in
            `local_path`, and download new files if there's a hash mismatch.

            Args:
                - `local_path (str)` The directory that contains the files that need
                  hashing.

                - `update_name (str)` The name of the update being performed, to report to
                  the user.

                - `download_url (str)` The download directory where the online version of
                  the file is found.

                - `file_count (int)` A counter for how many files have been downloaded, so
                  this can be reported to the user.

            Returns:
                `int` The final count of how many files have been downloaded.
            """

            # Create folders if they're missing
            pathlib.Path(local_path).mkdir(parents=True, exist_ok=True)

            # Get the hash.json
            eprint(f'* Checking for updated {update_name}...')
            download(f'{download_url}/hash.json', str(pathlib.Path(f'{local_path}/hash.json')))

            if pathlib.Path(f'{local_path}/hash.json').exists():
                try:
                    with open(pathlib.Path(f'{local_path}/hash.json'), 'r', encoding='utf-8') as file_hash:
                        file_hash_str: str = file_hash.read()
                except OSError as e:
                    eprint(f'\n{Font.error_bold}* Error: {Font.end}{str(e)}\n')
                    raise

                # Use the filenames and hash values in hash.json to determine if updates
                # need to be downloaded
                pathlib.Path(local_path).mkdir(parents=True, exist_ok=True)

                for file_name, hash_value in json.loads(file_hash_str).items():
                    if pathlib.Path(f'{local_path}/{file_name}').resolve().exists():
                        hash_sha256: _Hash = hashlib.sha256()

                        # Convert files from CRLF to LF to verify if they need updating
                        try:
                            with open(pathlib.Path(f'{local_path}/{file_name}'), 'r+', newline='\n') as file:
                                contents: str = file.read()

                                file.seek(0)
                                file.write(contents)
                                file.truncate()
                        except OSError as e:
                            eprint(f'\n{Font.error_bold}* Error: {Font.end}{str(e)}\n')
                            raise

                        # Get the hash of the new file
                        try:
                            with open (pathlib.Path(f'{local_path}/{file_name}').resolve(), 'rb') as file:
                                for chunk in iter(lambda: file.read(4096), b''):
                                    hash_sha256.update(chunk)
                        except OSError as e:
                                eprint(f'\n{Font.error_bold}* Error: {Font.end}{str(e)}\n')
                                raise

                        if hash_sha256.hexdigest() != hash_value:
                            eprint(f'  * Found an updated {update_name[:-1]}, {Font.bold}{file_name}{Font.end}. Downloading... ')
                            file_count += 1
                            failed = download(f'{download_url}/{file_name}', str(pathlib.Path(f'{local_path}/{file_name}')))
                            if not failed:
                                eprint(f'\033[F\033[K  * Found an updated {update_name[:-1]}, {Font.bold}{file_name}{Font.end}. Downloading... done.')
                    else:
                        eprint(f'  * Found a new {update_name[:-1]}, {Font.bold}{file_name}{Font.end}. Downloading... ')
                        file_count += 1
                        failed = download(f'{download_url}/{file_name}', str(pathlib.Path(f'{local_path}/{file_name}')))
                        if not failed:
                            eprint(f'\033[F\033[K  * Found a new {update_name[:-1]}, {Font.bold}{file_name}{Font.end}. Downloading... done.')
            else:
                eprint(f'{Font.warning}{str(pathlib.Path(f"{local_path}/hash.json"))} doesn\'t exist, can\'t update {update_name}.{Font.end}')

            return file_count

        file_count: int = 0
        file_count = get_updates(config.path_clone_list, 'clone lists', f'{download_location}/clonelists', file_count)
        file_count = get_updates(config.path_metadata, 'metadata files', f'{download_location}/metadata', file_count)

        if file_count == 0:
            eprint(f'{Font.success}\n* Done. No new updates are available.{Font.end}')
        elif file_count == 1:
            eprint(f'{Font.success}\n* Done. Downloaded {file_count} file.{Font.end}')
        else:
            eprint(f'{Font.success}\n* Done. Downloaded {file_count} files.{Font.end}')

        if not no_exit:
            if gui_input:
                raise ExitRetool
            else:
                sys.exit()

    @staticmethod
    def variants(processed_titles: dict[str, set[DatNode]], config: Config, input_dat: Dat, is_includes: bool = False) -> dict[str, set[DatNode]]:
        """ Processes a dictionary of DatNodes and groups titles together that are the
        same, but have different names as defined by the related clone list.

        This is the primary function that deals with the fact that, for example,
        Title, The (USA) is equivalent to Title, Le (France).

        Args:
            - `processed_titles (dict[str, set[DatNode]])` A work in progress dictionary
              of DatNodes, originally populated from the input DAT and actively being worked
              on by Retool.

            - `config (Config)` The Retool config object.

            - `input_dat (Dat)` The Retool input_dat object.

            - `is_includes (bool, optional)` Set to `True` when processing includes. Is
              only used to produce reliable reporting when performing a trace. Defaults to
              `False`.

        Returns:
            `dict[str, set[DatNode]]` A dictionary of DatNodes that has had all like
            titles grouped together as defined by the related clone list.
        """

        if input_dat.clone_list.variants:
            eprint('* Analyzing clone list variants... ', sep=' ', end='', flush=True)

            report_on_match: bool = False
            warning_given: bool = False
            missing_titles: set[str] = set()
            delete_titles: set[tuple[DatNode, str]] = set()

            for value in input_dat.clone_list.variants:
                if type(value) is dict:
                    if not (
                        'group' and
                        ('titles' in value
                        or 'compilations' in value
                        or 'supersets' in value)):
                            if config.user_input.verbose:
                                warning_given = True
                                printwrap(
                                    f'{Font.warning}* The following variants entry is invalid, '
                                    f'as it\'s missing a {Font.bold}group{Font.warning} key and '
                                    f'one of the following keys: {Font.bold}titles{Font.warning}, '
                                    f'{Font.bold}compilations{Font.warning}, or '
                                    f'{Font.bold}supersets{Font.warning}',
                                    'error')
                                eprint(f'\n  {value}{Font.end}')

                            if config.user_input.warningpause:
                                eprint(f'\n{Font.disabled}Press enter to continue{Font.end}')
                                input()
                            continue
                else:
                    if config.user_input.verbose:
                        warning_given = True
                        printwrap(
                            f'{Font.warning}* The following variants entry isn\'t an object '
                            f'and will be skipped:',
                            'error')
                        eprint(f'\n  {value}{Font.end}')

                        if config.user_input.warningpause:
                            eprint(f'\n{Font.disabled}Press enter to continue{Font.end}')
                            input()
                    continue

                def process_variants(variant_titles: list[dict[str, Any]], variant_type: str, report_on_match: bool) -> None:
                    """ Looks up a variant from a clone list and modifies its DatNode entry
                    accordingly.

                    Args:
                        - `variant_titles (list[dict[str, str])` The titles, supersets, or
                          compilations array from a variants object in the clone list.

                        - `variant_type (str)` The type of variant being processed, either
                          'title', 'compilation', or 'superset'.

                        - `report_on_match (bool)` Whether Retool needs to report any
                          titles being traced.
                    """

                    variants: list[DatNode] = []

                    for variant_title in variant_titles:
                        if 'searchTerm' not in variant_title:
                            printwrap(
                                f'{Font.warning}* The following variants entry is missing a '
                                f'{Font.bold}searchTerm{Font.warning} key and will be skipped:',
                                'error')
                            eprint(f'\n  {variant_title}{Font.end}')

                            if config.user_input.warningpause:
                                eprint(f'\n{Font.disabled}Press enter to continue{Font.end}')
                                input()
                            continue
                        else:
                            variant_name = variant_title['searchTerm']

                        name_type: str = ''

                        if 'nameType' in variant_title:
                            name_type = variant_title['nameType']
                            if not (name_type == 'full'
                                or name_type == 'short'
                                or name_type == 'regionFree'
                                or name_type == 'tagFree'
                                or name_type == 'regex'):
                                    name_type = 'short'
                        else:
                            name_type = 'short'

                        # Look up the title in the dictionary, then process the required changes
                        found_titles: set[DatNode] = set()

                        if name_type == 'regex':
                            valid_regex: list[str] = regex_test([variant_name], 'variants', 'clone list')

                            if not valid_regex:
                                continue

                            found_titles = TitleTools.find_title(variant_name, name_type, processed_titles, missing_titles, config, deep_search=True)
                        else:
                            found_titles = TitleTools.find_title(variant_name, name_type, processed_titles, missing_titles, config)

                        old_group_names: set[str] = set()

                        for title in found_titles:
                            old_group_names.add(title.group_name)

                        new_group_name: str = TitleTools.get_group_name(value['group'], config)

                        new_group_name = new_group_name.lower()

                        if new_group_name not in processed_titles:
                            processed_titles[new_group_name] = set()

                        # If the title's not found in the DAT, add it to missing_titles,
                        # otherwise add it to the delete list, then move it to the new
                        # group
                        if not found_titles:
                            missing_titles.add(variant_name)
                        else:
                            if config.user_input.trace:
                                report_on_match = TraceTools.trace_enable(set(found_titles), config.user_input.trace)

                            for title in found_titles:
                                for old_group_name in old_group_names:
                                    if title in processed_titles[old_group_name]:
                                        if (
                                            report_on_match
                                            and new_group_name != title.group_name
                                            and not is_includes):
                                                eprint('')
                                                TraceTools.trace_title('REF0055')
                                                eprint(f'* {title.full_name}')
                                                eprint(f'  New group: {new_group_name}\n{Font.disabled}  Old group: {title.group_name}{Font.end}')
                                                eprint(f'\n{Font.disabled}Press enter to continue{Font.end}')
                                                input()

                                        if (
                                            variant_type == 'title'
                                            or variant_type == 'superset'):
                                                new_title: DatNode = copy.deepcopy(title)

                                                if 'priority' in variant_title:
                                                    if new_title.clonelist_priority == 1:
                                                        new_title.clonelist_priority = variant_title['priority']

                                                if variant_type == 'superset':
                                                    new_title.is_superset = True

                                                variants.append(new_title)

                                                for old_group_name in old_group_names:
                                                    if title in processed_titles[old_group_name]:
                                                        delete_titles.add((title, old_group_name))

                                        elif variant_type =='compilation':
                                            for compilation_title in processed_titles[old_group_name]:
                                                if compilation_title.full_name == title.full_name:
                                                    title_position: int = 1
                                                    if 'titlePosition' in variant_title:
                                                        title_position = variant_title['titlePosition']

                                                    if 'priority' in variant_title:
                                                        clonelist_priority: int = variant_title['priority']
                                                    else:
                                                        clonelist_priority = 1

                                                    compilation_title.contains_titles[value['group']] = {"position": title_position, "priority": clonelist_priority}

                                                    if 'compilationPriority' in variant_title:
                                                        compilation_title.clonelist_priority = variant_title['compilationPriority']

                    # Set these variant properties after the processing, to make sure they
                    # don't mess with title look ups
                    for variant in variants:
                        variant.group_name = new_group_name
                        variant.short_name = value['group'].lower()
                        processed_titles[new_group_name].add(variant)

                if 'titles' in value:
                    process_variants(value['titles'], 'title', report_on_match)
                if 'compilations' in value:
                    process_variants(value['compilations'], 'compilation', report_on_match)
                if 'supersets' in value:
                    process_variants(value['supersets'], 'superset', report_on_match)

            # Delete titles from old groups, and clean up empty groups
            for title, old_group_name in delete_titles:
                if title in processed_titles[old_group_name]:
                    processed_titles[old_group_name].remove(title)

                if not processed_titles[old_group_name]:
                    del processed_titles[old_group_name]

            if missing_titles and config.user_input.verbose:
                eprint('')
                printwrap(
                    f'{Font.warning}* The following variants titles in the '
                    'clone list can\'t be found in the input DAT and will '
                    'be skipped:', 'error')

                eprint('')

                for missing_title in sorted(missing_titles):
                    eprint(f'  *  {missing_title}')

                eprint(f'{Font.end}')

                if config.user_input.warningpause:
                    eprint(f'\n{Font.disabled}Press enter to continue{Font.end}')
                    input()

            if (
                (warning_given
                 or missing_titles)
                and config.user_input.verbose):
                    eprint('* Analyzing clone list variants... done')
            else:
                eprint('done.')

        return processed_titles


import datetime
import logging
import urllib3
import json
import boto3
import os
import cStringIO

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
s3 = boto3.resource("s3")
timestamp = (datetime.datetime.utcnow() - datetime.datetime(1970,1,1,0,0,0)).total_seconds()

def run(event, context):
    current_time = datetime.datetime.now().time()
    name = context.function_name
    logger.info("Your cron function " + name + " ran at " + str(current_time))

    # Global data collection for s3 bucket files
    commProjectJsonData = cStringIO.StringIO()

    # Community User Accounts
    parseAPICall(os.environ.get('DOMAIN_PATH') + 'userDataPackages/' + os.environ.get('COMMUNITY_USER_REPORT'), 'userAccountCommunity')

    currProjPage = 1
    TotalProjPages = 1
    while currProjPage <= TotalProjPages:
        commProjectDataPackage = DataPackage(os.environ.get('DOMAIN_PATH') + 'projectDataPackages/' + os.environ.get('COMMUNITY_PROJECT_REPORT'), os.environ.get('CENTERCODE_API_KEY'), currProjPage)
        currProjPage += 1
        if commProjectDataPackage.Status == 200:
            DPPaging = commProjectDataPackage.getPaging()
            TotalProjPages = DPPaging.TotalPages
            DPDataRows = commProjectDataPackage.getDataRows()

            commProjectJsonData.write(commProjectDataPackage.getJsonString())

            logger.info('commProjectDataPackage TotalPages:' + str(DPPaging.TotalPages) + ', CurrentPage:' + str(DPPaging.CurrentPage) + ', ResultsPerPage:' + str(DPPaging.ResultsPerPage) + ', TotalResults:' + str(DPPaging.TotalResults))

            for rows in DPDataRows:
                ProjectName = ''
                projRootDataPackageURL = ''
                for data in rows:
                    # logger.info('data - FieldOrdinal:' + str(data.FieldOrdinal) + ', DataType:' + data.DataType + ', ComputedValue:' + data.ComputedValue)

                    if data.FieldOrdinal == int(os.environ.get('COMMUNITY_PROJECT_REPORT_NAME_ORDINAL')):
                        ProjectName = data.ComputedValue

                    if data.FieldOrdinal == int(os.environ.get('COMMUNITY_PROJECT_REPORT_KEY_ORDINAL')):
                        projRootDataPackageURL = os.environ.get('DOMAIN_PATH') + 'projects/' + data.ComputedValue + '/'

                logger.info('Project [' + ProjectName + '] projIssueReportDataPackageURL:' + projRootDataPackageURL)

                if projRootDataPackageURL != '':
                    # Project User Accounts
                    parseAPICall(projRootDataPackageURL + os.environ.get('PROJECT_USER_REPORT'), 'userAccountProjects')

                    # Project Issue Reports
                    parseAPICall(projRootDataPackageURL + os.environ.get('PROJECT_ISSUE_REPORT'), 'issueReports')

                    # Project Feature Requests
                    parseAPICall(projRootDataPackageURL + os.environ.get('PROJECT_FEATURE_REQUEST'), 'featureRequests')

                    # Project General Discussions
                    parseAPICall(projRootDataPackageURL + os.environ.get('PROJECT_GENERAL_DISCUSSION'), 'generalDiscussions')

        else:
            logger.error('commProjectDataPackage [NOT FOUND]:' + os.environ.get('DOMAIN_PATH') + 'projectDataPackages/' + os.environ.get('COMMUNITY_PROJECT_REPORT') + ', CurrentPage:' + str(currProjPage))
            break

    # Finalize the Community Level project API calls and write to s3/close
    finalizeAPICall('projectCommunity', commProjectJsonData)

    # End of run function

# Write to S3 and close the string wirter
def parseAPICall(dataPackageURL, fileName):
    stringWriter = cStringIO.StringIO()
    currReportPage = 1
    TotalReportPages = 1
    while currReportPage <= TotalReportPages:
        currDataPackage = DataPackage(dataPackageURL, os.environ.get('CENTERCODE_API_KEY'), currReportPage)
        if currDataPackage.Status == 200:
            URPaging = currDataPackage.getPaging()
            TotalReportPages = URPaging.TotalPages
            stringWriter.write(currDataPackage.getJsonString())
            logger.info(fileName+' TotalPages:' + str(URPaging.TotalPages) + ', CurrentPage:' + str(URPaging.CurrentPage) + ', ResultsPerPage:' + str(URPaging.ResultsPerPage) + ', TotalResults:' + str(URPaging.TotalResults))
            currReportPage += 1
        else:
            logger.error(fileName+' [NOT FOUND]:' + dataPackageURL + ', CurrentPage:' + str(currReportPage))
            break

    finalizeAPICall(fileName, stringWriter)

def finalizeAPICall(fileName, stringWriter):
    s3.Bucket(os.environ.get('S3_BUCKET')).put_object(Key=fileName+'-'+str(timestamp)+'.json', Body=stringWriter.getvalue())
    stringWriter.close()

# Defines the API Datapackage container
# Populates the Paging, Fields, and Data
class DataPackage:
    _json_data = None
    _json_object = None
    Status = -1

    def __init__(self, url, apiKey, currPage):
        logger.info('DataPackage url:' + url)
        http = urllib3.PoolManager()
        r = http.request('GET', url + '?page='+str(currPage)+'&apiKey='+apiKey)  # result is now a dict
        self.Status = r.status
        if self.Status == 200:
            self._json_data = r.data.decode('utf-8')
            self._json_object = json.loads(self._json_data)
        else:
            # This means something went wrong.
            logger.error('invokeAPI (' + url + ') status:' + str(self.Status))

    def getPaging(self):
        return Paging(self._json_object['paging'])

    def getFields(self):
        return Fields(self._json_object['fields'])

    def getDataRows(self):
        return DataRows(self._json_object['data'])

    def getJsonString(self):
        return json.dumps(self._json_object)

    def getJsonData(self):
        return json.dumps(self._json_data)

# Define the paging for this particular API endpoint
class Paging:
    TotalPages = 0
    CurrentPage = 0
    ResultsPerPage = 0
    TotalResults = 0

    def __init__(self, paging):
        self.TotalPages = int(paging['totalPages'])
        self.CurrentPage = int(paging['currentPage'])
        self.ResultsPerPage = int(paging['resultsPerPage'])
        self.TotalResults = int(paging['totalResults'])

# Define the View Fields defined in the Data Package
class Fields:
    Ordinal = -1

    def __init__(self, fields):
        self._fields = []
        for field in fields:
            self.Ordinal += 1
            self._fields.append(Field(self.Ordinal, field))

    def __len__(self):
        return len(self._fields)

    def __iter__(self):
        return iter(self._fields)

#Define the individual Field of the Fields return
class Field:
    Ordinal = -1
    DataType = ""
    Name = ""

    def __init__(self, ordinal, field):
        self.Ordinal = ordinal
        self.DataType = str(field['dataType'])
        self.Name = str(field['name'])

# Gather all of the returned data rows from this endpoint
class DataRows:
    Ordinal = -1

    def __init__(self, datarows):
        self._datarows = []
        for datarow in datarows:
            self.Ordinal += 1
            self._datarows.append(DataRow(datarow))

    def __len__(self):
        return len(self._datarows)

    def __iter__(self):
        return iter(self._datarows)

# Populate individual data row
class DataRow:
    Data = []

    def __init__(self, datarow):
        self._datarow = []
        for data in datarow:
            self._datarow.append(Data(data))

    def __len__(self):
        return len(self._datarow)

    def __iter__(self):
        return iter(self._datarow)

# Define the data for the row
class Data:
    DataType = ""
    ComputedValue = ""
    FieldOrdinal = 0
    Values = []

    def __init__(self, data):
        self.DataType = str(data['dataType'])
        self.ComputedValue = str(data['computedValue'])
        self.FieldOrdinal = int(data['fieldOrdinal'])
        self.Values = DataValues(data['values'])

# Define the Data Value Array for the row
class DataValues:
    Ordinal = -1

    def __init__(self, dataValues):
        self._datavalues = []
        for dataValue in dataValues:
            self.Ordinal += 1
            self._datavalues.append(dataValue)

    def __len__(self):
        return len(self._datavalues)

    def __iter__(self):
        return iter(self._datavalues)


from app.api.service import *
import random

# Fonction renvoyant l'ensemble des id des films vus par un utilisateur donné en paramètre
def ids_movies_watched_by_user(id_user: int) :

  watched_movies_user = get_movies_watched_by_user(id_user)
  ids_movies_watched = []
  for usermovie in watched_movies_user :
    ids_movies_watched.append(usermovie.id_movie)

  return ids_movies_watched

# Fonction renvoyant l'ensemble des ids des films vus par un groupe d'utilisateurs donné en paramètre
def get_movies_ids(id_group: int):

  users_group = get_usersgroup_by_group(id_group)
  watchmovie_ids = []

  for user in users_group :
    movies_for_user = ids_movies_watched_by_user(user.id)
    for movie_id in movies_for_user :
        watchmovie_ids.append(movie_id)

  return watchmovie_ids

# Fonction calculant les deux plus grandes occurences dans un tableau et renvoyant ces deux occurences
def occurences_max(tab):
    freq = {}
    for num in tab:
        if num in freq:
            freq[num] += 1
        else:
            freq[num] = 1

    max1 = None
    max2 = None
    for num, count in freq.items():
        if max1 is None or count > freq[max1]:
            max2 = max1
            max1 = num
        elif max2 is None or count > freq[max2]:
            max2 = num

    return max1, max2

# Fonction renvoyant 10 éléments aléatoires dans une liste de films donnée en paramètre
def extract_random_elements(movies):
    if len(movies) < 10:
        raise ValueError("List of movies must contain at least 10 elements")
    return random.sample(movies, 10)

# Fonction retournant des recommandations de films à partir de films ayant été vus par un groupe d'utilisateurs passé en paramètre
def recommendations_movies(id_group: int) :

  users_group = get_usersgroup_by_group(id_group)

  two_most_view_movie = occurences_max(get_movies_ids(users_group))

  get_all_movie_genre_1 = get_genres_by_movie(two_most_view_movie[0])
  get_all_movie_genre_2 = get_genres_by_movie(two_most_view_movie[1])

  all_genres = get_all_movie_genre_1 + get_all_movie_genre_2

  genre_ids =  []

  for genre in all_genres :
    genre_ids.append(genre.id)

  unique_genres = []
  for genre_id in genre_ids:
      if genre_id not in unique_genres:
        unique_genres.append(genre_id)

  movies_in_genre = []

  movies_view_by_users = get_movies_ids(id_group)

  for genre_id in unique_genres :
    movies_by_idgenre = get_movies_by_genre(genre_id)
    for movie_id in movies_by_idgenre :
      if movie_id.id_movie not in movies_view_by_users :
        movies_in_genre.append(movie_id.id_movie)

  movies_recommanded = extract_random_elements(movies_in_genre)

  return movies_recommanded


from create_bot import bot, db, ADMIN_ID
from aiogram import types, Dispatcher
from aiogram.types import InlineKeyboardMarkup, InlineKeyboardButton
from aiogram.dispatcher import FSMContext
from aiogram.dispatcher.filters.state import State, StatesGroup
from keyboards.kb_admin import keyboard_admin1
from keyboards.kb_other import gen_markup_category

async def start_admin(message: types.Message):
    await bot.send_message(chat_id=message.from_user.id, text='Привет, админ', reply_markup=keyboard_admin1)

class FSMRAddCountry(StatesGroup):
    country = State()

async def add_country_btn(message: types.Message):
    """Обработка кнопки добавить страну"""
    countries = db.bd_checks_for_country()
    country_dict = {country: 'red_count' for country in countries}
    country_dict['Добавить страну'] = 'add_count'
    country_dict['Отмена'] = "cancel"
    await bot.send_message(message.from_user.id, text='Выберите страну для редактирования или добавьте новую',
                           reply_markup=await gen_markup_category(country_dict))

async def start_fsm_add_country(callback_query: types.CallbackQuery):
    """Обработка инлайн кнопки добавить страну Старт FSMAddCountry"""
    await callback_query.message.reply('Введите страну',
                                       reply_markup=InlineKeyboardMarkup().add(
                                           InlineKeyboardButton("Отмена", callback_data=f"cancel")))
    await FSMRAddCountry.country.set()

async def add_country(message: types.Message, state: FSMContext):
    """Завершение FSMAddCountry. Запись в БД новой страны"""
    db.bd_add_country(message.text.strip().capitalize())
    await state.finish()
    countries = db.bd_checks_for_country()
    country_dict = {country: 'red_count' for country in countries}
    country_dict['Добавить страну'] = 'add_count'
    country_dict['Отмена'] = "cancel"
    await bot.send_message(message.from_user.id, text='Выберите страну для редактирования или добавьте новую',
                           reply_markup=await gen_markup_category(country_dict))

class FSMRAddCategories(StatesGroup):
    categories = State()

async def add_categories_btn(message: types.Message):
    """Обработка кнопки добавить категории"""
    categories = db.bd_checks_for_categories()
    categories_dict = {category: 'red_categor' for category in categories}
    categories_dict['Добавить категории'] = 'add_categor'
    categories_dict['Отмена'] = "cancel"
    await bot.send_message(message.from_user.id, text='Выберите категорию для редактирования или добавьте новые',
                           reply_markup=await gen_markup_category(categories_dict))

async def start_fsm_add_categories(callback_query: types.CallbackQuery):
    """Обработка инлайн кнопки добавить категорию Старт FSMAddCategories"""
    await callback_query.message.reply('Введите категорию или несколько через запятую',
                                       reply_markup=InlineKeyboardMarkup().add(
                                           InlineKeyboardButton("Отмена", callback_data=f"cancel")))
    await FSMRAddCategories.categories.set()

async def add_categories(message: types.Message, state: FSMContext):
    """Завершение FSMAddCategories. Запись в БД новой категории"""
    db.bd_add_category([c.strip() for c in message.text.split(',') if c])
    await state.finish()
    categories = db.bd_checks_for_categories()
    categories_dict = {category: 'red_categor' for category in categories}
    categories_dict['Добавить категории'] = 'add_categor'
    categories_dict['Отмена'] = "cancel"
    await bot.send_message(message.from_user.id, text='Выберите категорию для редактирования или добавьте новые',
                           reply_markup=await gen_markup_category(categories_dict))

class FSMRAddProduct(StatesGroup):
    country = State()
    category = State()
    photo = State()
    product_name = State()
    specifications = State()
    prise = State()
    count = State()

async def add_product_start(message: types.Message):
    """Обработка  кнопки добавить товар/ Старт FSMAddProduct"""
    countries = db.bd_checks_for_country()
    country_dict = {country: 'add_prod_countr' for country in countries}
    country_dict['Отмена'] = "cancel"
    if countries:
        await bot.send_message(message.from_user.id, text='Выберите страну',
                               reply_markup=await gen_markup_category(country_dict))
        await FSMRAddProduct.country.set()
    else:
        await bot.send_message(message.from_user.id, text='Страны отсутствуют в БД добавьте с помощью кнопки'
                                                          '"Добавить страну')

async def add_product_counter(callback_query: types.CallbackQuery, state: FSMContext):
    inline_command = callback_query.data.split(':')
    async with state.proxy() as data:
        data['country'] = inline_command[1]
        await FSMRAddProduct.next()
        categories = db.bd_checks_for_categories()
        if categories:
            categories_dict = {category: 'add_prod_categor' for category in categories}
            categories_dict['Отмена'] = "cancel"
            await callback_query.message.reply(text='Выберите категорию',
                                               reply_markup=await gen_markup_category(categories_dict))
        else:
            await callback_query.message.answer(text='Нет категорий в базе, добавьте категорию')
            await state.reset_state()
            await callback_query.message.delete()

async def add_product_category(callback_query: types.CallbackQuery, state: FSMContext):
    inline_command = callback_query.data.split(':')
    async with state.proxy() as data:
        data['category'] = inline_command[1]
        await FSMRAddProduct.next()
        await callback_query.message.reply('Отправьте одно фото',
                                           reply_markup=InlineKeyboardMarkup().add(
                                               InlineKeyboardButton("Отмена", callback_data=f"cancel")))

async def add_product_photo(message: types.Message, state: FSMContext):
    async with state.proxy() as data:
        data['photo'] = message.photo[0].file_id
        await FSMRAddProduct.next()
        await message.reply('Отправьте название', reply_markup=InlineKeyboardMarkup().add(
            InlineKeyboardButton("Отмена", callback_data=f"cancel")))

async def add_product_product_name(message: types.Message, state: FSMContext):
    async with state.proxy() as data:
        data['product_name'] = message.text
        await FSMRAddProduct.next()
        await message.reply('Отправьте описание', reply_markup=InlineKeyboardMarkup().add(
            InlineKeyboardButton("Отмена", callback_data=f"cancel")))

async def add_product_specifications(message: types.Message, state: FSMContext):
    async with state.proxy() as data:
        data['specifications'] = message.text
        await FSMRAddProduct.next()
        await message.reply('Отправьте цену', reply_markup=InlineKeyboardMarkup().add(
            InlineKeyboardButton("Отмена", callback_data=f"cancel")))

async def add_product_prise(message: types.Message, state: FSMContext):
    async with state.proxy() as data:
        data['prise'] = message.text
        await FSMRAddProduct.next()
        await message.reply('Отправьте количество в наличии', reply_markup=InlineKeyboardMarkup().add(
            InlineKeyboardButton("Отмена", callback_data=f"cancel")))

async def add_product_count(message: types.Message, state: FSMContext):
    async with state.proxy() as data:
        data['count'] = message.text
        db.bd_add_product_in_stock(list(data.values()))
        await message.answer(text='опубликовано')
        await state.finish()

def register_handler_admin(dp: Dispatcher):
    dp.register_message_handler(start_admin, lambda message: str(message.from_user.id) in ADMIN_ID,
                                text=['Назад', '/startadministrator'])

    dp.register_message_handler(add_product_start, lambda message: str(message.from_user.id) in ADMIN_ID,
                                text='Добавить товар', state=None)
    dp.register_callback_query_handler(add_product_counter, lambda x: x.data.startswith('add_prod_countr'),
                                       state=FSMRAddProduct.country)
    dp.register_callback_query_handler(add_product_category, lambda x: x.data.startswith('add_prod_categor'),
                                       state=FSMRAddProduct.category)
    dp.register_message_handler(add_product_photo, content_types=['photo'], state=FSMRAddProduct.photo)
    dp.register_message_handler(add_product_product_name, state=FSMRAddProduct.product_name)
    dp.register_message_handler(add_product_specifications, state=FSMRAddProduct.specifications)
    dp.register_message_handler(add_product_prise, state=FSMRAddProduct.prise)
    dp.register_message_handler(add_product_count, state=FSMRAddProduct.count)

    dp.register_message_handler(add_country_btn, lambda message: str(message.from_user.id) in ADMIN_ID,
                                text='Добавить страну')
    dp.register_callback_query_handler(start_fsm_add_country, lambda x: x.data.startswith('add_count'), state=None)
    dp.register_message_handler(add_country, state=FSMRAddCountry.country)

    dp.register_message_handler(add_categories_btn, lambda message: str(message.from_user.id) in ADMIN_ID,
                                text='Добавить категорию')
    dp.register_callback_query_handler(start_fsm_add_categories, lambda x: x.data.startswith('add_categor'), state=None)
    dp.register_message_handler(add_categories, state=FSMRAddCategories.categories)


import sys
input = sys.stdin.readline

############ ---- Input Functions ---- ############
def inp():
    return(int(input()))
def inlt():
    return(list(map(int,input().split())))
def insr():
    s = input()
    return(list(s[:len(s) - 1]))
def invr():
    return(map(int,input().split()))

def merger(a, b):
    if (a[1] +1) < b[0]:
        return False
    ans = (a[0], max(b[1], a[1]))
    return ans

n, tests = inlt()
days = []
for _ in range(tests):
    s1, s2 = inlt()
    days.append((s1,s2))

days.sort()
def solve():
    ans = days[0]
    if ans[0] != 0: return "YES"
    for i in range(1, len(days)):
        ast = days[i]

        m = merger(ans,ast)
        # print(m)
        if not m: return "YES"
        ans = m
    # print(ans)
    if ans[1] +1 != n: return "YES"
    return "NO"

print(solve())


# -*- mode: python -*-

block_cipher = None

a = Analysis(['elodie.py'],
             pathex=['/Users/jaisenmathai/dev/tools/elodie'],
             binaries=None,
             datas=[('configs/ExifTool_config', 'configs')],
             hiddenimports=[],
             hookspath=None,
             runtime_hooks=None,
             excludes=None,
             win_no_prefer_redirects=None,
             win_private_assemblies=None,
             cipher=block_cipher)
pyz = PYZ(a.pure, a.zipped_data,
             cipher=block_cipher)
exe = EXE(pyz,
          a.scripts,
          exclude_binaries=True,
          name='elodie',
          debug=False,
          strip=None,
          upx=True,
          console=True )
coll = COLLECT(exe,
               a.binaries,
               a.zipfiles,
               a.datas,
               strip=None,
               upx=True,
               name='elodie')


#kiểm tra nếu có 2 biến có cùng bộ nhớ

var1 = 5
var1 = 5
var2 = 'Hello'
var2 = 'Hello'
var3 = [1,2,3]
var3 = [1,2,3]
print(var1 is not var1)
print(var2 is var2)
print(var3 is var3)

# -------- output --------
# False
# True
# False

Basic of operators/10 Identity Operators.py


from __future__ import annotations

from typing import Dict, List, Any

from Structures.Asset import Asset
from Structures.AssetPairData import AssetPairData
import Structures.AssetHandler as AH

from Utils.StringManipulation import tabulate

from datetime import datetime

class AssetPair:

    """
    This class represents a pair of tradable assets.
    """

    def __init__(self : AssetPair) -> AssetPair:
        """
        Creates a new AssetPair object.
        """
        self.name : str = None
        self.altname : str = None
        self.wsname : str = None
        self.base : Asset = None
        self.quote : Asset = None
        self.fee : float = None
        self.order_min : float = None
        self.history : Dict[int, float] = {}

        self.data : AssetPairData = None
        self.is_init : bool = False

    @staticmethod
    def __compute_fee(df : List[float]):
        maximum = 0
        for element in df:
            if float(element[1]) >= maximum:
                maximum = float(element[1])
        return maximum

    @staticmethod
    def build_asset_pair(name : str, ah : AH.AssetHandler, data : Dict[str, Any]) -> AssetPair:
        """
        This methods builds a new pair of assets and returns it.

        :param name: The name of the asset being created.
        :param data: Data returned from Kraken corresponding to
        this asset.

        :returns: The constructed AssetPair.
        """
        ap = AssetPair()

        ap.name = name
        ap.altname = data["altname"]
        ap.wsname = data["wsname"]
        ap.base = ah.assets[data["base"]]
        ap.quote = ah.assets[data["quote"]]
        ap.fee = AssetPair.__compute_fee(data["fees"])
        ap.order_min = data["ordermin"]

        return ap

    def update_prices(self : AssetPair, data : List[List[float]]) -> None:
        """
        Updates the price of the current AssetPair.

        :param data: The data holding the prices.
        """
        for i in range(0, len(data)):
            self.history[int(data[i][0])] = float(data[i][4])

    def update_data(self : AssetPair, invert : bool = False) -> None:
        """
        Updates all the data related to this pair of asset. The data is
        calculated from the history of recorded prices in the last
        48 hours.

        :param invert: If True, the quote and base are inverted.
        """
        history = self.history
        keys = sorted(list(history.keys()))
        data = AssetPairData()

        # renaming
        ascension_fall_threshold = 3
        aft = ascension_fall_threshold

        # used to detect ascension, falls, local min and max
        ascension = -1
        fall = 0
        values  = []

        for i, date in enumerate(keys):
            if history[date] == 0: continue
            value = history[date] if not invert else 1 / history[date]

            # maximum and minimum overall values
            data.maximum = value if data.maximum == None else max(data.maximum, value)
            data.minimum = value if data.minimum == None else min(data.minimum, value)
            data.mean += value

            # get the mean of the aft previous values to determine current trend
            tmp_slice = values[max(i - aft, 0):i]
            mean_last_values = sum(tmp_slice) / len(tmp_slice) if len(tmp_slice) != 0 else 0

            # computing ascension
            if value >= mean_last_values:
                ascension += 1

                # was it a local maximum ?
                if fall == aft:
                    data.local_maximums[keys[i - fall]] = history[keys[i - fall]]
                    data.longest_fall = fall if data.longest_fall == None else max(data.longest_fall, fall)
                    data.shortest_fall = fall if data.shortest_fall == None else min(data.shortest_fall, fall)
                    data.average_fall += fall
                    data.nb_local_max += 1

                if value > mean_last_values:fall = 0

            # computing fall
            if value <= mean_last_values:
                fall += 1

                # was it a local minimum ?
                if ascension == aft:
                    data.local_minimums[keys[i - ascension]] = history[keys[i - ascension]]
                    data.longest_ascension = ascension if data.longest_ascension == None else max(data.longest_ascension, ascension)
                    data.shortest_ascension = ascension if data.shortest_ascension == None else min(data.shortest_ascension, ascension)
                    data.average_ascension += ascension
                    data.nb_local_min += 1

                if value < mean_last_values: ascension = 0

            values.append(value)

        # variance of the currency
        data.variance = 1 - (data.minimum / data.maximum) if data.maximum != 0 else 0

        # mean of the currency
        data.mean = data.mean / len(keys) if len(keys) != 0 else 0

        # average local min and max
        data.average_local_max = sum(data.local_maximums.values()) / len(data.local_maximums) if len(data.local_maximums) != 0 else 0
        data.average_local_min = sum(data.local_minimums.values()) / len(data.local_minimums) if len(data.local_minimums) != 0 else 0

        # average ascension and fall
        data.average_ascension = data.average_ascension / data.nb_local_max if data.nb_local_max != 0 else 0
        data.average_fall = data.average_fall / data.nb_local_min if data.nb_local_min != 0 else 0

        # current state of the curve
        data.is_ascending = ascension >= aft
        data.current_ascension_duration = ascension
        data.is_falling = fall >= aft
        data.current_fall_duration = fall

        # true if upward (resp. true if downward)
        data.trending_upwards = history[keys[-1]] / history[keys[0]] >= 1 if history[keys[0]] != 0 else False
        data.trending_downwards = history[keys[-1]] / history[keys[0]] <= 1 if history[keys[0]] != 0 else False

        data.current = history[keys[-1]]

        self.data = data
        self.is_init = True

    def copy(self : AssetPair) -> AssetPair:
        """
        Creates and returns a copy of this AssetPair object.
        """
        ap = AssetPair()
        ap.name = self.name
        ap.altname = self.altname
        ap.wsname = self.wsname
        ap.base = self.base.copy()
        ap.quote = self.quote.copy()
        ap.fee = self.fee
        ap.order_min = self.order_min
        ap.history = self.history.copy()

        ap.data = self.data.copy()
        ap.is_init = self.is_init
        return ap

    def __str__(self : AssetPair) -> str:
        s = ""
        s += f"Name: {self.name}\n"
        s += f"Altname: {self.altname}\n"
        s += f"Wsname: {self.wsname}\n"
        s += f"Base:\n {tabulate(str(self.base))}\n"
        s += f"Quote:\n {tabulate(str(self.quote))}\n"
        s += f"Fee: {self.fee}\n"
        s += f"Min order: {self.order_min}\n"

        s += f"Data about this pair of assets ({self.name}):\n"
        s += tabulate(str(self.data)) + "\n"
        s += f"Is init: {self.is_init}"
        return s.rstrip("\n")

    def str(self : AssetPair) -> str:
        s = ""
        s += f"Name: {self.name}\n"
        s += f"Altname: {self.altname}\n"
        s += f"Wsname: {self.wsname}\n"
        s += f"Base:\n {tabulate(str(self.base))}\n"
        s += f"Quote:\n {tabulate(str(self.quote))}\n"
        s += f"Fee: {self.fee}\n"
        s += f"Min order: {self.order_min}\n"

        s += f"History of prices for this pair of assets {self.name}:\n"
        for t in self.history:
            date = datetime.fromtimestamp(t).strftime("%A, %B %d, %Y %I:%M:%S")
            s += tabulate(f"{date}: {self.history[t]}") + "\n"

        s += f"Data about this pair of assets ({self.name}):\n"
        s += tabulate(str(self.data)) + "\n"
        s += f"Is init: {self.is_init}"
        return s.rstrip("\n")


import csv

import cv2
import numpy as np
import tensorflow as tf
from DatasetGenerator.drawUtils import draw_keypoints, draw_connections
from constants import KEYPOINT_DICT

# This section loads the movenet_singlepose_thunder.tflite model to the interpreter.
interpreter = tf.lite.Interpreter(model_path="../App/movenet.tflite")
interpreter.allocate_tensors()
interpreter.invoke()

input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
INPUT_SIZE = interpreter.get_input_details()[0]['shape'][1]

def inference(image):
    image = cv2.resize(image, (INPUT_SIZE, INPUT_SIZE)).astype(np.float32)

    interpreter.set_tensor(input_details[0]['index'], np.expand_dims(image, 0))
    interpreter.invoke()
    model_output = interpreter.get_tensor(output_details[0]['index'])[0][0]

    return model_output

imagePath = "poza_pokaz.png"

image = cv2.imread(imagePath)
modelOutput = inference(image)

file = open('lista.csv','w')
fileWriter = csv.writer(file)

for i in range(17):
    x,y,conf = modelOutput[i]
    name = list(KEYPOINT_DICT.keys())[i]
    fileWriter.writerow((name,x,y,conf))

image_height, image_width, _ = image.shape
modelOutput[:, 1] *= image_width
modelOutput[:, 0] *= image_height

# image = np.empty((600, 600, 3), dtype=np.int32)
# image.fill(255)

draw_connections(image, modelOutput)
draw_keypoints(image, modelOutput)

cv2.imwrite('szkielet_pokaz.png', image)

file.close()


from bs4 import BeautifulSoup as bs
from requests import get
from json import load, loads, dump
# from re import findall
import pprint

pp = pprint.PrettyPrinter(indent=4)

source = get('http://www.imdb.com/movies-in-theaters/?ref_=nv_mv_inth_1').text
soup = bs(source, 'lxml')
movies = soup.find_all('table', { 'class': 'nm-title-overview-widget-layout'})

metascore = soup.find("span", { "class" : "metascore" })
genre = soup.find_all("p", { "class" : "cert-runtime-genre" })
title = soup.find_all('td', { "class" : "overview-top" })

title_array = []
genre_array = []

for i in genre:
	try:
		genre_array.append(i.span.text)

	except AttributeError:
		continue

for i in title:
	title_array.append(i.h4.a.text[:-7].lstrip())

def returner(x):
	return x.text

def strip_returner(x):
	return x.text.replace(" ", "")

# genre_append(genre)

meta_score = [strip_returner(i) for i in metascore]
genres = [returner(i) for i in genre]
myobject = {
		"title": title_array,
		"metascore": meta_score,
		"genre": genre_array
	    }

mylist = []

for key, items in myobject.items():
   for i, item in enumerate(items):
       if len(mylist) == i:
           mylist.append({})
       mylist[i][key] = item

with open('imdbobject.json', 'w') as file:
	dump(mylist, file, indent=4, sort_keys=True)

with open('imdbobject.json', 'r+') as imdbfile:
	data = load(imdbfile)

	for i in data:
		genre = i['genre']
		meta = i['metascore']
		title = i['title']

		print("\n Title: {} \n Metascore: {} \n Genre: {}".format(title, meta, genre))