# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

# This scripts generates a report of open security bugs rated with sec-critical
# or sec-high, containing
# * number of open bugs by week.
# * number of opened bugs by week.
# * number of closed bugs by week.
# * resolutions of bugs closed.
#
# The 'open' time
# * starts once 'sec-critical' or 'sec-high' gets added to the bugs' keywords
# * ends once
#   - the bug gets set as resolved
#   - neither 'sec-critical' or 'sec-high' is anymore in the keywords of the bug
#   - the keyword 'stalled' is added (either information to proceed missing like
#     data which can only captured during a violation and which is very rare, or
#     lack of developers to investigate further)
# If multiple affected time ranges are created by those, the time from the first
# time one of the security ratings got added to either the resolution of the bug
# or - if occurred before - the setting of the keyword 'stalled' or removal of
# the security rating from the keywords is regarded as the affected time.

import argparse
import copy
import csv
import datetime
from dateutil.relativedelta import relativedelta
import json
from libmozdata.bugzilla import Bugzilla
from logger import logger
import pytz
import utils

PRODUCTS_TO_CHECK = [
    'Core',
    'DevTools',
    'External Software Affecting Firefox',
    'Firefox',
    'Firefox Build System',
    'Firefox for Android',
    'Firefox for iOS',
    'GeckoView',
    'NSPR',
    'NSS',
    'Testing',
    'Toolkit',
    'WebExtensions',
]

critical = 'sec-critical'
high = 'sec-high'

SEC_RATINGS = [critical, high]

STATUS_GLOBAL_OPEN = ['UNCONFIRMED', 'NEW', 'ASSIGNED', 'REOPENED']
STATUS_GLOBAL_RESOLUTIONS = [
                             '---',
                             'FIXED',
                             'VERIFIED',
                             'INVALID',
                             'WONTFIX',
                             'INACTIVE',
                             'DUPLICATE',
                             'WORKSFORME',
                             'INCOMPLETE',
                             'SUPPORT',
                             'EXPIRED',
                             'MOVED',
                            ]

WFMT = '{}-{:02d}'

# Bugzilla data can be loaded from file
bugzilla_data_loaded = None

# Bugzilla data can be saved froto file
bugzilla_data_to_save = {}

def add_bugzilla_data_to_save(node_path, data):
    # node_path is an array of strings representing the nodes in the JSON to
    # which the data shall be saved. The node has a child 'data' which holds the
    # data.
    node = bugzilla_data_to_save
    for path_step in node_path:
        if not path_step in node:
            node[path_step] = {'data': []}
        node = node[path_step]
    node['data'].append(data)

def get_weeks(start_date, end_date):
    res = []
    weeks_attrs = {}
    while start_date.strftime('%Y-%W') <= end_date.strftime('%Y-%W'):
        y, w, _ = start_date.isocalendar()
        week_start, week_end = utils.get_week_bounds(start_date)
        week_label = WFMT.format(y, w)
        res.append(week_label)
        weeks_attrs[week_label] = {
                                  'week_start' : week_start,
                                  'week_end' : week_end,
                                 }
        start_date += relativedelta(days=7)
    return res, weeks_attrs

def get_bugs():

    def bug_handler(bug_data, other_data):
        if bzdata_save_path:
            add_bugzilla_data_to_save(['sec-critical-high'], bug_data)

        bug_creation_time_str = bug_data['creation_time']
        bug_creation_time = datetime.datetime.strptime(bug_creation_time_str, '%Y-%m-%dT%H:%M:%SZ')
        bug_creation_time = pytz.utc.localize(bug_creation_time)

        sec_important_start = None
        sec_important_end = None
        sec_important_rating = None

        sec_important_times = []
        stalled_added_last = None
        # Look for changes to the 'keywords' field.
        for historyItem in bug_data['history']:
            for change in historyItem['changes']:
                if change['field_name'] == 'keywords':
                    change_time_str = historyItem['when']
                    change_time = datetime.datetime.strptime(change_time_str, '%Y-%m-%dT%H:%M:%SZ')
                    change_time = pytz.utc.localize(change_time)

                    keywords_removed = change['removed'].split(', ')
                    keywords_added = change['added'].split(', ')

                    sec_rating_removed = critical in keywords_removed or high in keywords_removed
                    sec_rating_added = critical in keywords_added or high in keywords_added
                    if sec_rating_removed or sec_rating_added:
                        # Find and set security rating for time range.
                        if sec_rating_removed:
                            if critical in keywords_removed:
                                sec_important_rating = critical
                            elif high in keywords_removed:
                                sec_important_rating = high
                            if not sec_important_start:
                                sec_important_start = max(date_start, bug_creation_time)
                            sec_important_end = change_time
                            sec_important_times.append({
                                                        'rating' : sec_important_rating,
                                                        'start' : sec_important_start,
                                                        'end' : sec_important_end,
                                                      })
                        if sec_rating_added:
                            if critical in keywords_added:
                                sec_important_rating = critical
                            elif high in keywords_added:
                                sec_important_rating = high
                            sec_important_start = change_time
                        # no relevant security added
                        else:
                            sec_important_start = None
                            sec_important_rating = None
                        sec_important_end = None

                    # Bugs which got the 'stalled' keyword added are inactive
                    # and not counted as open (active).
                    stalled_removed = 'stalled' in keywords_removed
                    stalled_added = 'stalled' in keywords_added
                    if stalled_added:
                        stalled_added_last = change_time
                    elif stalled_removed:
                        stalled_added_last = None
        if sec_important_start:
            # Open time range at the end of the investigated time range.
            if not sec_important_end:
                sec_important_end = date_end
            sec_important_times.append({
                                        'rating' : sec_important_rating,
                                        'start' : sec_important_start,
                                        'end' : sec_important_end,
                                      })
        # No keyword changes at all, keyword should have been set when bug got
        # created.
        if len(sec_important_times) == 0:
            keywords = bug_data['keywords']
            sec_important_rating = None
            if critical in keywords:
                sec_important_rating = critical
            elif high in keywords:
                sec_important_rating = high
            else:
                sys.exit('Found neither sec-critical nor sec-high in keywords but expected one of them.')
            sec_important_times.append({
                                        'rating' : sec_important_rating,
                                        'start' : max(date_start, bug_creation_time),
                                        'end' : date_end,
                                      })

        sec_open_end = None
        if critical in bug_data['keywords'] or high in bug_data['keywords']:
            if bug_data['is_open']:
                if 'stalled' in bug_data['keywords']:
                    # Open but stalled bug with a security rating
                    if stalled_added_last:
                        sec_open_end = max(stalled_added_last, date_start)
                    else:
                        # Can only get here if bug got created with 'stalled'
                        # keyword, e.g. if bug got cloned.
                        sec_open_end = max(date_start, bug_creation_time)
                else:
                    sec_open_end = date_end
            else:
                # Bug closed, last time it got resolved used as ended of
                # affected time range.
                last_resolved_str = bug_data['cf_last_resolved']
                last_resolved = datetime.datetime.strptime(last_resolved_str, '%Y-%m-%dT%H:%M:%SZ')
                last_resolved = pytz.utc.localize(last_resolved)
                sec_open_end = min(sec_important_times[-1]['end'], last_resolved)
        else:
            sec_open_end = sec_important_times[0]['start']
        stalled = True if 'stalled' in bug_data['keywords'] else False
        bug_data_to_export = {
                              'id' : bug_data['id'],
                              'rating' : sec_important_times[-1]['rating'],
                              'start' : sec_important_times[0]['start'],
                              'end' : sec_open_end,
                              'stalled' : stalled,
                              'status' : bug_data['status'],
                              'resolution' : bug_data['resolution'],
                            }
        bug_sec_open_ranges.append(bug_data_to_export)

    # Load Bugzilla data from file
    if bzdata_load_path:
        for bug_data in bugzilla_data_loaded['sec-critical-high']['data']:
            other_data = {}
            bug_handler(bug_data, other_data)
    # Load Bugzilla data from Bugzilla server
    else:
        queries = []
        fields = [
                  'id',
                  'creation_time',
                  'status',
                  'is_open',
                  'resolution',
                  'cf_last_resolved',
                  'keywords',
                  'history',
                 ]

        params = {
            'include_fields': fields,
            'product': PRODUCTS_TO_CHECK,
            'j_top' : 'OR',
            # Either the keywords 'sec-critical' or 'sec-high' got removed
            # after the given date.
            'f1' : 'OP',
            'j1' : 'AND_G',
            'f2' : 'keywords',
            'o2' : 'changedfrom',
            'v2' : 'sec-critical',
            'f3' : 'keywords',
            'o3' : 'changedafter',
            'v3' : '',
            'f4' : 'CP',
            'f5' : 'OP',
            'j5' : 'AND_G',
            'f6' : 'keywords',
            'o6' : 'changedfrom',
            'v6' : 'sec-high',
            'f7' : 'keywords',
            'o7' : 'changedafter',
            'v7' : '',
            'f8' : 'CP',
            # Or the bug still has either the keyword 'sec-critical' or
            # 'sec-high' and
            'f9' : 'OP',
            'f10' : 'keywords',
            'o10' : 'anywords',
            'v10' : 'sec-critical, sec-high',
            'f11' : 'OP',
            'j11' : 'OR',
            # ... got resolved after the given date ...
            'f12' : 'cf_last_resolved',
            'o12' : 'changedafter',
            'v12' : '',
            # or hasn't been resolved yet.
            'f13' : 'bug_status',
            'o13' : 'anywords',
            'v13' : 'UNCONFIRMED, NEW, ASSIGNED, REOPENED',
            'f14' : 'CP',
            'f15' : 'CP',
        }

        params['v3'] = date_start_str
        params['v7'] = date_start_str
        params['v12'] = date_start_str

        query = Bugzilla(params,
                         bughandler=bug_handler,
                         bugdata={},
                         timeout=960)
        query.get_data().wait()

def log(message):
    print(message)

def aggregate_to_weekly_reports():
    for bug_sec_data in bug_sec_open_ranges:
        weeks_affected, unused_week_attrs = get_weeks(bug_sec_data['start'], bug_sec_data['end'])
        for week in weeks_affected:
            if week < weeks[0]:
                continue
            if week == weeks_affected[-1] and (bug_sec_data['status'] not in STATUS_GLOBAL_OPEN or bug_sec_data['stalled']):
                continue
            bug_sec_data_dict[bug_sec_data['rating']][week][bug_sec_data['id']] = bug_sec_data

    for sec_rating in bug_sec_data_dict:
        for week_pos in range(len(weeks) - 1, -1, -1):
            week = weeks[week_pos]
            open_by_week[sec_rating][week] = len(bug_sec_data_dict[sec_rating][week])

    for sec_rating in bug_sec_data_dict:
        for week_pos in range(len(weeks) - 1, 0, -1):
            week_this = weeks[week_pos]
            week_prev = weeks[week_pos - 1]
            open_this = set(bug_sec_data_dict[sec_rating][week_this].keys())
            open_prev = set(bug_sec_data_dict[sec_rating][week_prev].keys())
            opened_bugs = open_this - open_prev
            closed_bugs = open_prev - open_this
            opened_by_week[sec_rating][week_this] = len(opened_bugs)
            closed_by_week[sec_rating][week_this] = len(closed_bugs)
            for bug_id in closed_bugs:
                bug_data = bug_sec_data_dict[sec_rating][week_prev][bug_id]
                if bug_data['status'] not in STATUS_GLOBAL_OPEN:
                    resolution = bug_data['resolution']
                    if resolution not in STATUS_GLOBAL_RESOLUTIONS:
                        resolution = 'unknown'
                elif bug_data['stalled']:
                    resolution = 'stalled'
                resolutions_by_week[resolution][week_this] += 1

def write_csv():
    with open('data/security_bugs_report.csv', 'w') as Out:
        writer = csv.writer(Out, delimiter=',')

        date_row = ['Monday date']
        for week_pos in range(len(weeks) - 1, -1, -1):
            w = weeks[week_pos]
            date_row.append(weeks_attrs[w]['week_start'].strftime('%Y-%m-%d'))

        writer.writerow(['Open security bugs by week'])
        writer.writerow(date_row)

        for sec_rating in SEC_RATINGS:
            sec_rating_row = [sec_rating]
            for week_pos in range(len(weeks) - 1, -1, -1):
                w = weeks[week_pos]
                sec_rating_row.append(open_by_week[sec_rating][w])
            writer.writerow(sec_rating_row)
        sec_rating_total_row = ['Total']
        for week_pos in range(len(weeks) - 1, -1, -1):
            w = weeks[week_pos]
            sec_rating_total = 0
            for sec_rating in SEC_RATINGS:
                sec_rating_total += open_by_week[sec_rating][w]
            sec_rating_total_row.append(sec_rating_total)
        writer.writerow(sec_rating_total_row)

        writer.writerow([])
        writer.writerow([])

        writer.writerow(['Opened security bugs by week'])
        writer.writerow(date_row)

        for sec_rating in SEC_RATINGS:
            sec_rating_row = [sec_rating]
            for week_pos in range(len(weeks) - 1, 0, -1):
                w = weeks[week_pos]
                sec_rating_row.append(opened_by_week[sec_rating][w])
            writer.writerow(sec_rating_row)
        sec_rating_total_row = ['Total']
        for week_pos in range(len(weeks) - 1, 0, -1):
            w = weeks[week_pos]
            sec_rating_total = 0
            for sec_rating in SEC_RATINGS:
                sec_rating_total += opened_by_week[sec_rating][w]
            sec_rating_total_row.append(sec_rating_total)
        writer.writerow(sec_rating_total_row)

        writer.writerow([])
        writer.writerow([])

        writer.writerow(['Closed security bugs by week'])
        writer.writerow(date_row)

        for sec_rating in SEC_RATINGS:
            sec_rating_row = [sec_rating]
            for week_pos in range(len(weeks) - 1, 0, -1):
                w = weeks[week_pos]
                sec_rating_row.append(closed_by_week[sec_rating][w])
            writer.writerow(sec_rating_row)
        sec_rating_total_row = ['Total']
        for week_pos in range(len(weeks) - 1, 0, -1):
            w = weeks[week_pos]
            sec_rating_total = 0
            for sec_rating in SEC_RATINGS:
                sec_rating_total += closed_by_week[sec_rating][w]
            sec_rating_total_row.append(sec_rating_total)
        writer.writerow(sec_rating_total_row)

        writer.writerow([])
        writer.writerow([])

        writer.writerow(['Resolutions of security bugs by week'])
        writer.writerow(date_row)

        for resolution in STATUS_GLOBAL_RESOLUTIONS + ['stalled', 'unknown']:
            resolution_row = [resolution]
            for week_pos in range(len(weeks) - 1, 0, -1):
                w = weeks[week_pos]
                resolution_row.append(resolutions_by_week[resolution][w])
            writer.writerow(resolution_row)
        writer.writerow(sec_rating_total_row)

        writer.writerow([])
        writer.writerow(['"stalled" is not a Resolution status but a keyword which is added if either ' + \
                         'information to proceed is missing like data which can only captured during a ' + \
                         'violation and which is very rare, or lack of developers to investigate further.'])
        writer.writerow(['"unknown" is catching resolutions unknown to this script, should be new to ' + \
                         'bugzilla.mozilla.org'])

parser = argparse.ArgumentParser(description='Count security bugs opened and closed by week')
parser.add_argument('--bzdata-load',
                    nargs='?',
                    default=argparse.SUPPRESS,
                    help='Load the Bugzilla data from a local JSON file. If no path is provided '
                         'the program will try to load "sec_bugs_bugzilla_data.json" from the "data" folder.')
parser.add_argument('--bzdata-save',
                    nargs='?',
                    default=argparse.SUPPRESS,
                    help='Save the Bugzilla data to a local JSON file. If no path is provided '
                         'the program will try to save as "sec_bugs_bugzilla_data.json" into the "data" folder.')
args = parser.parse_args()

# Start of time range used by report. Hardcoded default of 1 year.
date_start = datetime.datetime.now() - datetime.timedelta(days = 365)
date_start = pytz.utc.localize(date_start)
date_start_str = date_start.strftime('%Y-%m-%dT%H:%M:%SZ')

# End of time range used by report.
date_end = datetime.datetime.utcnow()
date_end = pytz.utc.localize(date_end)
date_end_str = date_end.strftime('%Y-%m-%dT%H:%M:%SZ')

weeks, weeks_attrs = get_weeks(date_start, date_end)

bzdata_load_path = None
if 'bzdata_load' in args:
    # Load Bugzilla data from file
    if args.bzdata_load:
        # File path provided as command line argument
        bzdata_load_path = args.bzdata_load
    else:
        # No file path provided, use default location
        bzdata_load_path = 'data/sec_bugs_bugzilla_data.json'
    with open(bzdata_load_path, 'r') as bugzilla_data_reader:
        bugzilla_data_loaded = json.load(bugzilla_data_reader)
    log('Loaded Bugzilla data from {}'.format(bzdata_load_path))
    date_start_str = bugzilla_data_loaded['date_start']['data'][0]
    date_start = datetime.datetime.strptime(date_start_str, '%Y-%m-%dT%H:%M:%SZ')
    date_start = pytz.utc.localize(date_start)
    log('Date start from loaded Bugzilla data: {}'.format(date_start_str))
    date_end_str = bugzilla_data_loaded['date_end']['data'][0]
    date_end = datetime.datetime.strptime(date_end_str, '%Y-%m-%dT%H:%M:%SZ')
    date_end = pytz.utc.localize(date_end)
    log('Date end from loaded Bugzilla data: {}'.format(date_end_str))

bzdata_save_path = None
if 'bzdata_save' in args:
    # File path to which Bugzilla data shall be saved
    if args.bzdata_save:
        # File path provided as command line argument
        bzdata_save_path = args.bzdata_save
    else:
        # No file path provided, use default location
        bzdata_save_path = 'data/sec_bugs_bugzilla_data.json'

# Holds the time range in which a bug was considered open (see top of file) and
# had security rating.
bug_sec_open_ranges = []
get_bugs()

bug_sec_data_dict = { sec_rating : { week : {} for week in weeks } for sec_rating in SEC_RATINGS }
report_data = { sec_rating : { week : set() for week in weeks } for sec_rating in SEC_RATINGS }
open_by_week = { sec_rating : { week : 0 for week in weeks } for sec_rating in SEC_RATINGS }
opened_by_week = { sec_rating : { week : 0 for week in weeks } for sec_rating in SEC_RATINGS }
closed_by_week = { sec_rating : { week : 0 for week in weeks } for sec_rating in SEC_RATINGS }
resolutions_by_week = { resolution : { week : 0 for week in weeks } for resolution in STATUS_GLOBAL_RESOLUTIONS }
# 'stalled' is not a Resolution status but a keyword which is added if either
# information to proceed is missing like data which can only captured during a
# violation and which is very rare, or lack of developers to investigate
# further.
resolutions_by_week['stalled'] = { week : 0 for week in weeks }
# 'unknown' is catching resolutions unknown to this script, should be new to
# bugzilla.mozilla.org
resolutions_by_week['unknown'] = { week : 0 for week in weeks }
aggregate_to_weekly_reports()

write_csv()

if bzdata_save_path:
    # Save Bugzilla data to file
    with open('data/sec_bugs_bugzilla_data.json', 'w') as bugzilla_data_writer:
        add_bugzilla_data_to_save(['date_start'], date_start_str)
        add_bugzilla_data_to_save(['date_end'], date_end_str)
        bugzilla_data_writer.write(json.dumps(bugzilla_data_to_save))
        log('Saved Bugzilla data to {}'.format(bzdata_save_path))


import re

import openpyxl
from playwright.sync_api import Page
from auction_scraper_base import browser_context, get_tenure
from auction_scraper_base import get_property_type, prepare_price, get_bedroom

start_url = "https://bidx1.com/en/ireland?division=80&region=1&maxprice=&page=1"

def property_scraper(detail_url_page, auction_house):
    propertyLink = f'https://bidx1.com{auction_house}'
    detail_url_page.goto(propertyLink)
    print(propertyLink)

    image_link = detail_url_page.locator("(//img[contains(@class,'d-block w-100')])[2]")
    print(image_link)

    guidePrice = detail_url_page.locator("xpath=//p[contains(text(),'Guide Price')]").text_content()
    price, currency = prepare_price(guidePrice)

    description = detail_url_page.locator("xpath=(//div[@class='content container'])[1]").text_content()
    print(description)

    address = detail_url_page.locator("xpath=//h2[@class='m-0 order-1 order-lg-0 ']").text_content()
    postal_code = address.split(',')[-1]

    numbers_of_bedrooms = detail_url_page.locator("xpath=//div[@id='elevator-pitch']/ul").text_content()
    number_of_bedrooms = get_bedroom(numbers_of_bedrooms)
    print(number_of_bedrooms)

    tenure = detail_url_page.locator("xpath=(//div[@id='elevator-pitch']//strong)[1]").text_content()

    if number_of_bedrooms is None:
        number_of_bedrooms = get_bedroom(description)

    property_type = get_property_type(description)

    if property_type == "other" and "land" in address.lower():
        property_type = "land"
    if "land" in address.lower():
        address = re.search(r"(?<= to ).*|(?<= of ).*|(?<= at ).*", address, re.IGNORECASE)
        if address:
            address = address.group().strip()

    if not tenure:
        tenure = get_tenure(description)

    data_hash = {
        "price": price,
        "currency_type": currency,
        "picture_link": image_link,
        "property_description": description,
        "property_link": propertyLink,
        "address": address,
        "postal_code": postal_code,
        "auction_venue": "online",
        "source": "auctionhouselondon",
        "property_type": property_type,
        "number_of_bedrooms": number_of_bedrooms,
        "tenure": tenure,
    }

def properties_scraper(page, browser):
    auction_divs = []
    anchor_locator = page.locator("//a[@class='disable-decoration d-flex flex-column flex-fill']")

    for i in range(anchor_locator.count()):
        property_href = anchor_locator.nth(i)
        property_card = property_href.get_attribute("href")
        auction_divs.append(property_card)

    detail_url_page: Page = browser.new_page()

    for auction_house in auction_divs:
        property_scraper(detail_url_page, auction_house)

def run():
    with browser_context(headless=False) as (page, browser):
        page.set_default_navigation_timeout(120000)
        page.goto(start_url)
        properties_scraper(page, browser)

if __name__ == "__main__":
    run()


from .parameter_spec import ParameterSpec

class Macro(object):
    def __init__(self, param_spec: ParameterSpec, body, env):
        self.param_spec = param_spec
        self.body = body
        self.env = env

# true, false (Boolean)
# nil Type
# def
# fn
# do
# macro <
# defn
# defmacro
# if
# when
# unless


import requests
import ssl
# Ignore SSL certificate errors
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE

url = "https://api.covid19api.com/"

country = input("Enter the country for which you want the data (Enter Global for global data)\n")

try:
    # Code for Global Data.
    if country.lower() == 'global':
        url = url + 'summary'  # This is the url from where the data is being extracted.
        response = requests.request("GET", url)  # Get request is sent to the api for getting the data.
        data = response.json()  # data is then stored in json format.
        ans = ''  # Declaring a string ans for storing the final answer in it.
        for i in data['Global']:
            if (i != 'NewConfirmed') & (i != 'NewDeaths') & (i != 'NewRecovered'):
                ans = ans + "\n" + i + ": " + str(data['Global'][i])  # Data extracted from the json data and stored in ans.
        print(ans)

    elif (country.lower() == 'usa') or (country.lower() == 'us') or (
            country.lower() == 'united states of america') or (
            country.lower() == 'the united states of america'):
        url = "https://corona.lmao.ninja/v2/countries/USA"  # api url for extracting data for US.

        payload = {}
        headers = {
            'Cookie': '__cfduid=d200d71060eb7fc135ca56ddd5f37e7041586929559'
        }

        response = requests.request("GET", url, headers=headers, data=payload)  # data being extracted from api using GET method.
        data = response.json()  # response stored in data as json.
        answer = "Total Cases: " + str(data['cases']) + "\n" + "Deaths: " + str(data['deaths']) + "\n" + "Recovered: " + str(data['recovered'])
        print(answer)  # printing the final answer extracted from json format in the string answer.
    else:
        if (country.lower() == 'uk') or (country.lower() == 'england') or (country.lower() == 'great britain'):
            country = 'United Kingdom'
        if country.lower() == 'uae':
            country = 'United Arab Emirates'
        country = country.replace(' ', '-').lower()
        url = url + 'summary'  # url for extracted info for other countries.
        response = requests.request("GET", url)  # data being extracted from api using GET method.
        data = response.json()  # response stored in data as json.
        final = ''
        for i in data['Countries']:
            if i['Country'].lower() == country:
                for j in i:
                    if (j != 'CountryCode') & (j != 'Slug') & (j != 'Date')\
                            & (j != 'NewConfirmed') & (j != 'NewDeaths') & (j != 'NewRecovered'):
                        final = final + '\n' + j + ': ' + str(i[j])
                print(final)  # printing the final answer extracted from json format in the string final.
except:
    print("The data hasn't been updated yet.")


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import functools
import os

import six

from tensorflow.python.client import session as session_lib
from tensorflow.python.distribute import mirrored_strategy
from tensorflow.python.eager import backprop
from tensorflow.python.eager import context
from tensorflow.python.eager import def_function
from tensorflow.python.eager import test
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.framework import test_util
from tensorflow.python.keras.engine import training
from tensorflow.python.keras.layers import core
from tensorflow.python.ops import init_ops
from tensorflow.python.ops import resource_variable_ops
from tensorflow.python.ops import state_ops
from tensorflow.python.ops import template
from tensorflow.python.ops import variable_scope
from tensorflow.python.training import adam
from tensorflow.python.training import checkpoint_management
from tensorflow.python.training import saver as saver_lib
from tensorflow.python.training import training_util
from tensorflow.python.training.tracking import graph_view
from tensorflow.python.training.tracking import tracking
from tensorflow.python.training.tracking import util as trackable_utils

class NonLayerTrackable(tracking.AutoTrackable):

  def __init__(self):
    super(NonLayerTrackable, self).__init__()
    self.a_variable = trackable_utils.add_variable(
        self, name="a_variable", shape=[])

# pylint: disable=not-callable
class MyModel(training.Model):
  """A concrete Model for testing."""

  def __init__(self):
    super(MyModel, self).__init__()
    self._named_dense = core.Dense(1, use_bias=True)
    self._second = core.Dense(1, use_bias=False)
    # We can still track Trackables which aren't Layers.
    self._non_layer = NonLayerTrackable()

  def call(self, values):
    ret = self._second(self._named_dense(values))
    return ret

class CheckpointingTests(test.TestCase):

  @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True)
  def testNamingWithOptimizer(self):
    input_value = constant_op.constant([[3.]])
    model = MyModel()
    # A nuisance Model using the same optimizer. Its slot variables should not
    # go in the checkpoint, since it is never depended on.
    other_model = MyModel()
    optimizer = adam.AdamOptimizer(0.001)
    optimizer_step = training_util.get_or_create_global_step()
    root_trackable = trackable_utils.Checkpoint(
        optimizer=optimizer, model=model, optimizer_step=optimizer_step)
    if context.executing_eagerly():
      optimizer.minimize(
          lambda: model(input_value),
          global_step=optimizer_step)
      optimizer.minimize(
          lambda: other_model(input_value),
          global_step=optimizer_step)
    else:
      train_op = optimizer.minimize(
          model(input_value), global_step=optimizer_step)
      optimizer.minimize(
          other_model(input_value),
          global_step=optimizer_step)
      self.evaluate(trackable_utils.gather_initializers(
          root_trackable))
      self.evaluate(train_op)
    named_variables, serialized_graph, _ = graph_view.ObjectGraphView(
        root_trackable).serialize_object_graph()
    expected_checkpoint_names = (
        # Created in the root node, so no prefix.
        "optimizer_step",
        "model/_second/kernel",
        "model/_named_dense/kernel",
        "model/_named_dense/bias",
        # non-Layer dependency of the model
        "model/_non_layer/a_variable",
        # The optimizer creates two non-slot variables
        "optimizer/beta1_power",
        "optimizer/beta2_power",
        # Slot variables
        "model/_second/kernel/.OPTIMIZER_SLOT/optimizer/m",
        "model/_second/kernel/.OPTIMIZER_SLOT/optimizer/v",
        "model/_named_dense/kernel/.OPTIMIZER_SLOT/optimizer/m",
        "model/_named_dense/kernel/.OPTIMIZER_SLOT/optimizer/v",
        "model/_named_dense/bias/.OPTIMIZER_SLOT/optimizer/m",
        "model/_named_dense/bias/.OPTIMIZER_SLOT/optimizer/v",
    )
    suffix = "/.ATTRIBUTES/VARIABLE_VALUE"
    expected_checkpoint_names = [
        name + suffix for name in expected_checkpoint_names]
    named_variables = {v.name: v for v in named_variables}
    six.assertCountEqual(self, expected_checkpoint_names,
                         named_variables.keys())
    # Check that we've mapped to the right variable objects (not exhaustive)
    self.assertEqual(
        "global_step",
        named_variables["optimizer_step" + suffix].full_name)
    self.assertEqual(
        "my_model/dense_1/kernel",
        named_variables["model/_second/kernel" + suffix].full_name)
    self.assertEqual(
        "my_model/dense/kernel",
        named_variables["model/_named_dense/kernel" + suffix].full_name)
    self.assertEqual(
        "beta1_power",
        named_variables["optimizer/beta1_power" + suffix].full_name)
    self.assertEqual(
        "beta2_power",
        named_variables["optimizer/beta2_power" + suffix].full_name)
    # Spot check the generated protocol buffers.
    self.assertEqual("optimizer",
                     serialized_graph.nodes[0].children[1].local_name)
    optimizer_node = serialized_graph.nodes[serialized_graph.nodes[0].children[
        1].node_id]
    self.assertEqual("beta1_power",
                     optimizer_node.children[0].local_name)
    self.assertEqual("beta1_power",
                     serialized_graph.nodes[optimizer_node.children[0].node_id]
                     .attributes[0].full_name)
    self.assertEqual(
        "my_model/dense/kernel",
        serialized_graph.nodes[optimizer_node.slot_variables[0]
                               .original_variable_node_id]
        .attributes[0].full_name)
    # We strip off the :0 suffix, as variable.name-based saving does.
    self.assertEqual(
        "my_model/dense/kernel/Adam",
        serialized_graph.nodes[optimizer_node.slot_variables[0]
                               .slot_variable_node_id]
        .attributes[0].full_name)
    self.assertEqual(
        "my_model/dense/kernel/Adam:0",
        optimizer.get_slot(
            var=model._named_dense.kernel,
            name="m").name)
    self.assertEqual(
        "model/_named_dense/kernel" + suffix,
        serialized_graph.nodes[
            optimizer_node.slot_variables[0]
            .original_variable_node_id].attributes[0].checkpoint_key)
    self.assertEqual("m", optimizer_node.slot_variables[0].slot_name)
    self.assertEqual(
        "model/_named_dense/kernel/.OPTIMIZER_SLOT/optimizer/m" + suffix,
        serialized_graph.nodes[
            optimizer_node.slot_variables[0]
            .slot_variable_node_id].attributes[0].checkpoint_key)

  @test_util.run_in_graph_and_eager_modes
  def testSaveRestore(self):
    model = MyModel()
    optimizer = adam.AdamOptimizer(0.001)
    root_trackable = trackable_utils.Checkpoint(
        optimizer=optimizer, model=model)
    input_value = constant_op.constant([[3.]])
    if context.executing_eagerly():
      optimizer.minimize(
          lambda: model(input_value))
    else:
      train_op = optimizer.minimize(model(input_value))
      # TODO(allenl): Make initialization more pleasant when graph building.
      root_trackable.save_counter  # pylint: disable=pointless-statement
      self.evaluate(trackable_utils.gather_initializers(
          root_trackable))
      self.evaluate(train_op)
    prefix = os.path.join(self.get_temp_dir(), "ckpt")
    self.evaluate(state_ops.assign(model._named_dense.variables[1], [42.]))
    m_bias_slot = optimizer.get_slot(model._named_dense.variables[1], "m")
    self.evaluate(state_ops.assign(m_bias_slot, [1.5]))
    save_path = root_trackable.save(file_prefix=prefix)
    self.evaluate(state_ops.assign(model._named_dense.variables[1], [43.]))
    self.evaluate(state_ops.assign(root_trackable.save_counter, 3))
    optimizer_variables = self.evaluate(optimizer.variables())
    self.evaluate(state_ops.assign(m_bias_slot, [-2.]))
    # Immediate restoration
    status = root_trackable.restore(save_path=save_path).assert_consumed()
    status.run_restore_ops()
    self.assertAllEqual([42.], self.evaluate(model._named_dense.variables[1]))
    self.assertAllEqual(1, self.evaluate(root_trackable.save_counter))
    self.assertAllEqual([1.5], self.evaluate(m_bias_slot))
    if not context.executing_eagerly():
      return  # Restore-on-create is only supported when executing eagerly
    on_create_model = MyModel()
    on_create_optimizer = adam.AdamOptimizer(
        0.001,
        # Preserve beta1_power and beta2_power when appying gradients so we can
        # test that they've been restored correctly.
        beta1=1.0, beta2=1.0)
    on_create_root = trackable_utils.Checkpoint(
        optimizer=on_create_optimizer, model=on_create_model)
    # Deferred restoration
    status = on_create_root.restore(save_path=save_path)
    status.assert_nontrivial_match()
    status.assert_existing_objects_matched()
    with self.assertRaises(AssertionError):
      status.assert_consumed()
    on_create_model(constant_op.constant([[3.]]))  # create variables
    self.assertAllEqual(1, self.evaluate(on_create_root.save_counter))
    self.assertAllEqual([42.],
                        self.evaluate(
                            on_create_model._named_dense.variables[1]))
    on_create_m_bias_slot = on_create_optimizer.get_slot(
        on_create_model._named_dense.variables[1], "m")
    status.assert_existing_objects_matched()
    with self.assertRaises(AssertionError):
      status.assert_consumed()
    # Optimizer slot variables are created when the original variable is
    # restored.
    self.assertAllEqual([1.5], self.evaluate(on_create_m_bias_slot))
    self.assertAllEqual(optimizer_variables[2:],
                        self.evaluate(on_create_optimizer.variables()))
    dummy_var = resource_variable_ops.ResourceVariable([1.])
    on_create_optimizer.minimize(loss=dummy_var.read_value)
    status.assert_existing_objects_matched()
    status.assert_consumed()
    beta1_power, beta2_power = on_create_optimizer._get_beta_accumulators()
    self.assertAllEqual(optimizer_variables[0], self.evaluate(beta1_power))
    self.assertAllEqual(optimizer_variables[1], self.evaluate(beta2_power))

  # TODO(allenl): Debug garbage created by this test in python3.
  def testDeferredRestorationUsageEager(self):
    """An idiomatic eager execution example."""
    num_training_steps = 10
    checkpoint_directory = self.get_temp_dir()
    checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
    for training_continuation in range(3):
      model = MyModel()
      optimizer = adam.AdamOptimizer(0.001)
      root = trackable_utils.Checkpoint(
          optimizer=optimizer, model=model,
          optimizer_step=training_util.get_or_create_global_step())
      root.restore(checkpoint_management.latest_checkpoint(
          checkpoint_directory))
      for _ in range(num_training_steps):
        # TODO(allenl): Use a Dataset and serialize/checkpoint it.
        input_value = constant_op.constant([[3.]])
        optimizer.minimize(
            lambda: model(input_value),  # pylint: disable=cell-var-from-loop
            global_step=root.optimizer_step)
      root.save(file_prefix=checkpoint_prefix)
      self.assertEqual((training_continuation + 1) * num_training_steps,
                       root.optimizer_step.numpy())

  def testEagerDistributionStrategy(self):
    num_training_steps = 10
    checkpoint_directory = self.get_temp_dir()
    checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")

    def _train_fn(optimizer, model):
      input_value = constant_op.constant([[3.]])
      optimizer.minimize(
          functools.partial(model, input_value),
          global_step=root.optimizer_step)

    for training_continuation in range(3):
      strategy = mirrored_strategy.MirroredStrategy()
      with strategy.scope():
        model = MyModel()
        optimizer = adam.AdamOptimizer(0.001)
        root = trackable_utils.Checkpoint(
            optimizer=optimizer, model=model,
            optimizer_step=training_util.get_or_create_global_step())
        root.restore(checkpoint_management.latest_checkpoint(
            checkpoint_directory))

        for _ in range(num_training_steps):
          strategy.extended.call_for_each_replica(
              functools.partial(_train_fn, optimizer, model))
        root.save(file_prefix=checkpoint_prefix)
        self.assertEqual((training_continuation + 1) * num_training_steps,
                         root.optimizer_step.numpy())

  def testGraphDistributionStrategy(self):
    self.skipTest("b/121381184")
    num_training_steps = 10
    checkpoint_directory = self.get_temp_dir()
    checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")

    def _train_fn(optimizer, model):
      input_value = constant_op.constant([[3.]])
      return optimizer.minimize(
          functools.partial(model, input_value),
          global_step=root.optimizer_step)

    for training_continuation in range(3):
      with ops.Graph().as_default():
        strategy = mirrored_strategy.MirroredStrategy()
        with strategy.scope():
          model = MyModel()
          optimizer = adam.AdamOptimizer(0.001)
          root = trackable_utils.Checkpoint(
              optimizer=optimizer, model=model,
              optimizer_step=training_util.get_or_create_global_step())
          status = root.restore(checkpoint_management.latest_checkpoint(
              checkpoint_directory))
          train_op = strategy.extended.call_for_each_replica(
              functools.partial(_train_fn, optimizer, model))
          with self.session() as session:
            if training_continuation > 0:
              status.assert_consumed()
            status.initialize_or_restore()
            for _ in range(num_training_steps):
              session.run(train_op)
            root.save(file_prefix=checkpoint_prefix)
        self.assertEqual((training_continuation + 1) * num_training_steps,
                         root.optimizer_step.numpy())

  def testUsageGraph(self):
    """Expected usage when graph building."""
    with context.graph_mode():
      num_training_steps = 10
      checkpoint_directory = self.get_temp_dir()
      checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
      for training_continuation in range(3):
        with ops.Graph().as_default():
          model = MyModel()
          optimizer = adam.AdamOptimizer(0.001)
          root = trackable_utils.CheckpointV1(
              optimizer=optimizer, model=model,
              global_step=training_util.get_or_create_global_step())
          input_value = constant_op.constant([[3.]])
          train_op = optimizer.minimize(
              model(input_value),
              global_step=root.global_step)
          checkpoint_path = checkpoint_management.latest_checkpoint(
              checkpoint_directory)
          with self.session(graph=ops.get_default_graph()) as session:
            status = root.restore(save_path=checkpoint_path)
            status.initialize_or_restore(session=session)
            if checkpoint_path is None:
              self.assertEqual(0, training_continuation)
              with self.assertRaises(AssertionError):
                status.assert_consumed()
              with self.assertRaises(AssertionError):
                status.assert_existing_objects_matched()
            else:
              status.assert_consumed()
              status.assert_existing_objects_matched()
            for _ in range(num_training_steps):
              session.run(train_op)
            root.save(file_prefix=checkpoint_prefix, session=session)
            self.assertEqual((training_continuation + 1) * num_training_steps,
                             session.run(root.global_step))
            self.assertEqual(training_continuation + 1,
                             session.run(root.save_counter))

  @test_util.run_in_graph_and_eager_modes
  def testAgnosticUsage(self):
    """Graph/eager agnostic usage."""
    # Does create garbage when executing eagerly due to ops.Graph() creation.
    num_training_steps = 10
    checkpoint_directory = self.get_temp_dir()
    for training_continuation in range(3):
      with test_util.device(use_gpu=True):
        model = MyModel()
        optimizer = adam.AdamOptimizer(0.001)
        root = trackable_utils.Checkpoint(
            optimizer=optimizer, model=model,
            global_step=training_util.get_or_create_global_step())
        manager = checkpoint_management.CheckpointManager(
            root, checkpoint_directory, max_to_keep=1)
        status = root.restore(save_path=manager.latest_checkpoint)
        input_value = constant_op.constant([[3.]])
        train_fn = functools.partial(
            optimizer.minimize,
            functools.partial(model, input_value),
            global_step=root.global_step)
        if not context.executing_eagerly():
          train_fn = functools.partial(self.evaluate, train_fn())
        status.initialize_or_restore()
        for _ in range(num_training_steps):
          train_fn()
        manager.save()
        self.assertEqual((training_continuation + 1) * num_training_steps,
                         self.evaluate(root.global_step))
        self.assertEqual(training_continuation + 1,
                         self.evaluate(root.save_counter))

  # pylint: disable=cell-var-from-loop
  @test_util.run_in_graph_and_eager_modes
  def testWithDefun(self):
    num_training_steps = 2
    checkpoint_directory = self.get_temp_dir()
    checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
    for training_continuation in range(3):
      with test_util.device(use_gpu=True):
        model = MyModel()
        # Don't actually train so we can test variable values
        optimizer = adam.AdamOptimizer(0.)
        root = trackable_utils.Checkpoint(
            optimizer=optimizer, model=model,
            global_step=training_util.get_or_create_global_step())
        checkpoint_path = checkpoint_management.latest_checkpoint(
            checkpoint_directory)
        status = root.restore(save_path=checkpoint_path)
        def train_fn():
          @def_function.function
          def _call_model(x):
            return model(x)
          with backprop.GradientTape() as tape:
            loss = _call_model(constant_op.constant([[3.]]))
          gradients = tape.gradient(loss, model.variables)
          return optimizer.apply_gradients(zip(gradients, model.variables),
                                           global_step=root.global_step)
        if not context.executing_eagerly():
          train_fn = functools.partial(
              self.evaluate, train_fn())
        status.initialize_or_restore()
        for _ in range(num_training_steps):
          train_fn()
        if training_continuation > 0:
          status.assert_consumed()
          self.assertAllClose([[42.]], self.evaluate(model.variables[0]))
        else:
          self.evaluate(model.variables[0].assign([[42.]]))
        root.save(file_prefix=checkpoint_prefix)
        self.assertEqual((training_continuation + 1) * num_training_steps,
                         self.evaluate(root.global_step))
        self.assertEqual(training_continuation + 1,
                         self.evaluate(root.save_counter))
  # pylint: enable=cell-var-from-loop

  def _get_checkpoint_name(self, name):
    root = tracking.AutoTrackable()
    trackable_utils.add_variable(
        root, name=name, shape=[1, 2], dtype=dtypes.float64)
    (named_variable,), _, _ = trackable_utils._serialize_object_graph(
        root, saveables_cache=None)
    with ops.name_scope("root/" + named_variable.name):
      pass  # Make sure we can use this as an op name if we prefix it.
    return named_variable.name

  def testAnonymousVarsInInit(self):

    class Model(training.Model):

      def __init__(self):
        super(Model, self).__init__()
        self.w = resource_variable_ops.ResourceVariable(0.0)
        self.b = resource_variable_ops.ResourceVariable(0.0)
        self.vars = [self.w, self.b]

      def call(self, x):
        return x * self.w + self.b

    with context.eager_mode():
      model = Model()
      optimizer = adam.AdamOptimizer(learning_rate=0.05)
      checkpoint_directory = self.get_temp_dir()
      checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
      checkpoint = trackable_utils.Checkpoint(
          model=model, optimizer=optimizer)
      for _ in range(2):
        checkpoint.save(checkpoint_prefix)
        with backprop.GradientTape() as tape:
          loss = (constant_op.constant(1.)
                  - model(constant_op.constant(1.))) ** 2
        grad = tape.gradient(loss, model.vars)
        optimizer.apply_gradients(
            [(g, v) for g, v in zip(grad, model.vars)])

  @test_util.run_in_graph_and_eager_modes
  def testDeferredSlotRestoration(self):
    checkpoint_directory = self.get_temp_dir()

    root = trackable_utils.Checkpoint()
    root.var = trackable_utils.add_variable(
        root, name="var", initializer=0.)
    optimizer = adam.AdamOptimizer(0.1)
    if context.executing_eagerly():
      optimizer.minimize(root.var.read_value)
    else:
      train_op = optimizer.minimize(root.var)
      # Note that `optimizer` has not been added as a dependency of
      # `root`. Create a one-off grouping so that slot variables for `root.var`
      # get initialized too.
      self.evaluate(trackable_utils.gather_initializers(
          trackable_utils.Checkpoint(root=root, optimizer=optimizer)))
      self.evaluate(train_op)
    self.evaluate(state_ops.assign(root.var, 12.))
    no_slots_path = root.save(os.path.join(checkpoint_directory, "no_slots"))
    root.optimizer = optimizer
    self.evaluate(state_ops.assign(root.var, 13.))
    self.evaluate(state_ops.assign(optimizer.get_slot(name="m", var=root.var),
                                   14.))
    slots_path = root.save(os.path.join(checkpoint_directory, "with_slots"))
    new_root = trackable_utils.Checkpoint()
    # Load the slot-containing checkpoint (deferred), then immediately overwrite
    # the non-slot variable (also deferred).
    slot_status = new_root.restore(slots_path)
    no_slot_status = new_root.restore(no_slots_path)
    with self.assertRaises(AssertionError):
      no_slot_status.assert_consumed()
    new_root.var = trackable_utils.add_variable(
        new_root, name="var", shape=[])
    no_slot_status.assert_consumed()
    no_slot_status.run_restore_ops()
    self.assertEqual(12., self.evaluate(new_root.var))
    new_root.optimizer = adam.AdamOptimizer(0.1)
    slot_status.assert_existing_objects_matched()
    with self.assertRaisesRegexp(AssertionError, "beta1_power"):
      slot_status.assert_consumed()
    self.assertEqual(12., self.evaluate(new_root.var))
    if context.executing_eagerly():
      # Slot variables are only created with restoring initializers when
      # executing eagerly.
      self.assertEqual(14., self.evaluate(
          new_root.optimizer.get_slot(name="m", var=new_root.var)))
    else:
      self.assertIs(new_root.optimizer.get_slot(name="m", var=new_root.var),
                    None)
    if context.executing_eagerly():
      new_root.optimizer.minimize(new_root.var.read_value)
    else:
      train_op = new_root.optimizer.minimize(new_root.var)
      # The slot variable now exists; restore() didn't create it, but we should
      # now have a restore op for it.
      slot_status.run_restore_ops()
      self.assertEqual(14., self.evaluate(
          new_root.optimizer.get_slot(name="m", var=new_root.var)))
      self.evaluate(train_op)
    slot_status.assert_consumed()

  def testManySavesGraph(self):
    """Saves after the first should not modify the graph."""
    with context.graph_mode():
      graph = ops.Graph()
      with graph.as_default(), self.session(graph):
        checkpoint_directory = self.get_temp_dir()
        checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
        obj = trackable_utils.Checkpoint()
        obj.var = variable_scope.get_variable(name="v", initializer=0.)
        obj.opt = adam.AdamOptimizer(0.1)
        obj.opt.minimize(obj.var.read_value())
        self.evaluate(trackable_utils.gather_initializers(obj))
        obj.save(checkpoint_prefix)
        before_ops = graph.get_operations()
        obj.save(checkpoint_prefix)
        self.assertEqual(before_ops, graph.get_operations())

  def testManyRestoresGraph(self):
    """Restores after the first should not modify the graph."""
    with context.graph_mode():
      graph = ops.Graph()
      with graph.as_default(), self.session(graph):
        checkpoint_directory = self.get_temp_dir()
        checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
        obj = trackable_utils.Checkpoint()
        obj.var = variable_scope.get_variable(name="v", initializer=0.)
        obj.opt = adam.AdamOptimizer(0.1)
        obj.opt.minimize(obj.var.read_value())
        self.evaluate(trackable_utils.gather_initializers(obj))
        save_path = obj.save(checkpoint_prefix)
        obj.restore(save_path)
        before_ops = graph.get_operations()
        obj.restore(save_path)
        self.assertEqual(before_ops, graph.get_operations())

  def testMultipleGraphsNonSlotVariables(self):
    with context.graph_mode():
      checkpoint_directory = self.get_temp_dir()
      checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
      optimizer = adam.AdamOptimizer(0.001)
      # Construct a model in one graph
      first_graph = ops.Graph()
      first_session = session_lib.Session(graph=first_graph)
      with first_graph.as_default(), first_session.as_default():
        first_variable = resource_variable_ops.ResourceVariable([1.])
        first_root_trackable = trackable_utils.Checkpoint(
            optimizer=optimizer, variable=first_variable)
        train_op = optimizer.minimize(first_variable.read_value)
        self.evaluate(trackable_utils.gather_initializers(
            first_root_trackable))
        self.evaluate(train_op)
        self.evaluate(first_variable.assign([1.]))
        self.evaluate(optimizer.get_slot(
            var=first_variable, name="m").assign([2.]))
        beta1_power, _ = optimizer._get_beta_accumulators()
        self.evaluate(beta1_power.assign(3.))

      # Save and load in a second graph
      second_graph = ops.Graph()
      with second_graph.as_default(), session_lib.Session(graph=second_graph):
        second_variable = resource_variable_ops.ResourceVariable([1.])
        second_root_trackable = trackable_utils.Checkpoint(
            optimizer=optimizer, variable=second_variable)
        train_op = optimizer.minimize(second_variable.read_value)
        second_root_trackable.restore(None).initialize_or_restore()
        self.evaluate(train_op)
        self.evaluate(second_variable.assign([4.]))
        self.evaluate(optimizer.get_slot(
            var=second_variable, name="m").assign([5.]))
        beta1_power, _ = optimizer._get_beta_accumulators()
        self.evaluate(beta1_power.assign(6.))
        save_path = second_root_trackable.save(checkpoint_prefix)
        self.evaluate(second_variable.assign([7.]))
        self.evaluate(optimizer.get_slot(
            var=second_variable, name="m").assign([8.]))
        beta1_power, _ = optimizer._get_beta_accumulators()
        self.assertAllEqual(6., self.evaluate(beta1_power))
        status = second_root_trackable.restore(save_path)
        status.assert_consumed().run_restore_ops()
        self.assertAllEqual([4.], self.evaluate(second_variable))
        self.assertAllEqual([5.], self.evaluate(optimizer.get_slot(
            var=second_variable, name="m")))
        beta1_power, _ = optimizer._get_beta_accumulators()
        self.assertAllEqual(6., self.evaluate(beta1_power))

      # Check that the first graph is unmolested
      with first_graph.as_default(), first_session.as_default():
        self.assertAllEqual([1.], self.evaluate(first_variable))
        self.assertAllEqual([2.], self.evaluate(optimizer.get_slot(
            var=first_variable, name="m")))
        beta1_power, _ = optimizer._get_beta_accumulators()
        self.assertAllEqual(3., self.evaluate(beta1_power))

  @test_util.run_in_graph_and_eager_modes
  def test_initialize_if_not_restoring(self):
    checkpoint_directory = self.get_temp_dir()
    checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
    optimizer_only_prefix = os.path.join(checkpoint_directory, "opt")
    with test_util.device(use_gpu=True):
      model = MyModel()
      optimizer = adam.AdamOptimizer(0.001)
      root = trackable_utils.Checkpoint(
          model=model,  # Do not save the optimizer with the checkpoint.
          global_step=training_util.get_or_create_global_step())
      optimizer_checkpoint = trackable_utils.Checkpoint(
          optimizer=optimizer)

      checkpoint_path = checkpoint_management.latest_checkpoint(
          checkpoint_directory)
      status = root.restore(save_path=checkpoint_path)
      input_value = constant_op.constant([[3.]])
      train_fn = functools.partial(
          optimizer.minimize,
          functools.partial(model, input_value),
          global_step=root.global_step)
      if not context.executing_eagerly():
        train_fn = functools.partial(self.evaluate, train_fn())
      status.initialize_or_restore()
      self.evaluate([v.initializer for v in optimizer.variables()])
      train_fn()
      model_save_path = root.save(file_prefix=checkpoint_prefix)
      self.evaluate(optimizer.variables()[0].assign(42.))
      optimizer_save_path = optimizer_checkpoint.save(optimizer_only_prefix)

    # Restore into a graph with the optimizer
    with test_util.device(use_gpu=True):
      model = MyModel()
      optimizer = adam.AdamOptimizer(0.001)
      root = trackable_utils.Checkpoint(
          optimizer=optimizer, model=model,
          global_step=training_util.get_or_create_global_step())
      status = root.restore(save_path=model_save_path)
      input_value = constant_op.constant([[3.]])
      train_fn = functools.partial(
          optimizer.minimize,
          functools.partial(model, input_value),
          global_step=root.global_step)
      if not context.executing_eagerly():
        train_fn = functools.partial(self.evaluate, train_fn())
      status.initialize_or_restore()
      train_fn()
      with self.assertRaises(AssertionError):
        status.assert_existing_objects_matched()
      with self.assertRaises(AssertionError):
        status.assert_consumed()

    # Make sure initialization doesn't clobber later restores
    with test_util.device(use_gpu=True):
      model = MyModel()
      optimizer = adam.AdamOptimizer(0.001, beta1=1.0)
      root = trackable_utils.Checkpoint(
          optimizer=optimizer, model=model,
          global_step=training_util.get_or_create_global_step())
      opt_root = trackable_utils.Checkpoint(
          optimizer=optimizer)
      status = root.restore(save_path=model_save_path)
      init_only_optimizer_status = opt_root.restore(save_path=None)
      optimizer_status = opt_root.restore(save_path=optimizer_save_path)
      input_value = constant_op.constant([[3.]])
      train_fn = functools.partial(
          optimizer.minimize,
          functools.partial(model, input_value),
          global_step=root.global_step)
      if not context.executing_eagerly():
        train_fn = functools.partial(self.evaluate, train_fn())
      optimizer_status.run_restore_ops()
      status.initialize_or_restore()
      init_only_optimizer_status.initialize_or_restore()
      train_fn()
      self.assertEqual(42., self.evaluate(optimizer.variables()[0]))

class _ManualScope(tracking.AutoTrackable):

  def __call__(self):
    with variable_scope.variable_scope("ManualScope") as vs:
      self.variable_scope = vs
      with trackable_utils.capture_dependencies(template=self):
        return self._build()

  def _build(self):
    return variable_scope.get_variable(name="in_manual_scope", shape=[])

class TemplateTests(test.TestCase):

  @test_util.run_in_graph_and_eager_modes
  def test_trackable_save_restore(self):

    def _templated():
      v = variable_scope.get_variable(
          "v", shape=[1], initializer=init_ops.zeros_initializer(),
          use_resource=True)
      v2 = variable_scope.get_variable(
          "v2", shape=[1], initializer=init_ops.zeros_initializer(),
          use_resource=True)
      manual = _ManualScope()
      return v, v + 1., v2, manual, manual()

    save_template = template.make_template("s1", _templated)
    v1_save, _, v2_save, manual_scope, manual_scope_v = save_template()
    six.assertCountEqual(self, [
        id(obj) for obj in
        [v1_save, v2_save, manual_scope, manual_scope_v, save_template]
    ], [id(obj) for obj in trackable_utils.list_objects(save_template)])
    manual_dep, = manual_scope._checkpoint_dependencies
    self.assertEqual("in_manual_scope", manual_dep.name)
    self.assertIs(manual_scope_v, manual_dep.ref)
    optimizer = adam.AdamOptimizer(0.0)
    save_root = trackable_utils.Checkpoint(
        my_template=save_template, optimizer=optimizer)
    optimizer.minimize(v1_save.read_value)
    self.evaluate([v.initializer for v in save_template.variables])
    self.evaluate([v.initializer for v in optimizer.variables()])
    self.evaluate(v1_save.assign([12.]))
    self.evaluate(v2_save.assign([14.]))
    checkpoint_directory = self.get_temp_dir()
    checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
    save_path = save_root.save(checkpoint_prefix)

    load_template = template.make_template("s2", _templated)
    load_optimizer = adam.AdamOptimizer(0.0)
    load_root = trackable_utils.Checkpoint(
        my_template=load_template, optimizer=load_optimizer)
    status = load_root.restore(save_path)
    var, var_plus_one, var2, _, _ = load_template()
    load_optimizer.minimize(var.read_value)
    self.assertEqual(3, len(load_template._checkpoint_dependencies))
    self.assertEqual("v", load_template._checkpoint_dependencies[0].name)
    self.assertEqual("v2", load_template._checkpoint_dependencies[1].name)
    self.assertEqual("ManualScope",
                     load_template._checkpoint_dependencies[2].name)
    status.assert_consumed().run_restore_ops()
    self.assertAllEqual([12.], self.evaluate(var))
    self.assertAllEqual([13.], self.evaluate(var_plus_one))
    self.assertAllEqual([14.], self.evaluate(var2))

class CheckpointCompatibilityTests(test.TestCase):

  def _initialized_model(self):
    input_value = constant_op.constant([[3.]])
    model = MyModel()
    optimizer = adam.AdamOptimizer(0.001)
    optimizer_step = training_util.get_or_create_global_step()
    root_trackable = trackable_utils.Checkpoint(
        optimizer=optimizer, model=model, optimizer_step=optimizer_step)
    train_op = optimizer.minimize(
        functools.partial(model, input_value),
        global_step=optimizer_step)
    self.evaluate(trackable_utils.gather_initializers(
        root_trackable))
    self.evaluate(train_op)
    # A regular variable, a slot variable, and a non-slot Optimizer variable
    # with known values to check when loading.
    self.evaluate(model._named_dense.bias.assign([1.]))
    self.evaluate(optimizer.get_slot(
        var=model._named_dense.bias, name="m").assign([2.]))
    beta1_power, _ = optimizer._get_beta_accumulators()
    self.evaluate(beta1_power.assign(3.))
    return root_trackable

  def _set_sentinels(self, root_trackable):
    self.evaluate(root_trackable.model._named_dense.bias.assign([101.]))
    self.evaluate(
        root_trackable.optimizer.get_slot(
            var=root_trackable.model._named_dense.bias, name="m")
        .assign([102.]))
    beta1_power, _ = root_trackable.optimizer._get_beta_accumulators()
    self.evaluate(beta1_power.assign(103.))

  def _check_sentinels(self, root_trackable):
    self.assertAllEqual(
        [1.], self.evaluate(root_trackable.model._named_dense.bias))
    self.assertAllEqual([2.], self.evaluate(
        root_trackable.optimizer.get_slot(
            var=root_trackable.model._named_dense.bias, name="m")))
    beta1_power, _ = root_trackable.optimizer._get_beta_accumulators()
    self.assertAllEqual(3., self.evaluate(beta1_power))

  def _write_name_based_checkpoint(self):
    checkpoint_directory = self.get_temp_dir()
    checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
    with context.graph_mode():
      save_graph = ops.Graph()
      with save_graph.as_default(), self.session(
          graph=save_graph) as session:
        root = self._initialized_model()
        name_saver = saver_lib.Saver()
        return name_saver.save(
            sess=session, save_path=checkpoint_prefix,
            global_step=root.optimizer_step)

  @test_util.run_in_graph_and_eager_modes
  def testLoadFromNameBasedSaver(self):
    """Save a name-based checkpoint, load it using the object-based API."""
    with test_util.device(use_gpu=True):
      save_path = self._write_name_based_checkpoint()
      root = self._initialized_model()
      self._set_sentinels(root)
      with self.assertRaises(AssertionError):
        self._check_sentinels(root)
      object_saver = trackable_utils.TrackableSaver(
          graph_view.ObjectGraphView(root))
      self._set_sentinels(root)
      status = object_saver.restore(save_path)
      if context.executing_eagerly():
        self._check_sentinels(root)
      if context.executing_eagerly():
        status.assert_consumed()
        status.assert_existing_objects_matched()
        status.assert_nontrivial_match()
      else:
        # When graph building, we haven't read any keys, so we don't know
        # whether the restore will be complete.
        with self.assertRaisesRegexp(AssertionError, "not restored"):
          status.assert_consumed()
        with self.assertRaisesRegexp(AssertionError, "not restored"):
          status.assert_existing_objects_matched()
        with self.assertRaisesRegexp(AssertionError, "not restored"):
          status.assert_nontrivial_match()
      status.run_restore_ops()
      self._check_sentinels(root)
      self._set_sentinels(root)
      status = object_saver.restore(save_path)
      status.initialize_or_restore()
      self._check_sentinels(root)
      # Check that there is no error when keys are missing from the name-based
      # checkpoint.
      root.not_in_name_checkpoint = resource_variable_ops.ResourceVariable([1.])
      status = object_saver.restore(save_path)
      with self.assertRaises(AssertionError):
        status.assert_existing_objects_matched()

  def testSaveGraphLoadEager(self):
    checkpoint_directory = self.get_temp_dir()
    checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
    with context.graph_mode():
      save_graph = ops.Graph()
      with save_graph.as_default(), self.session(
          graph=save_graph):
        root = self._initialized_model()
        save_path = root.save(file_prefix=checkpoint_prefix)
    with context.eager_mode():
      root = self._initialized_model()
      self._set_sentinels(root)
      root.restore(save_path).assert_consumed()
      self._check_sentinels(root)

  def testSaveEagerLoadGraph(self):
    checkpoint_directory = self.get_temp_dir()
    checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
    with context.eager_mode():
      root = self._initialized_model()
      save_path = root.save(file_prefix=checkpoint_prefix)
    with context.graph_mode():
      save_graph = ops.Graph()
      with save_graph.as_default(), self.session(
          graph=save_graph):
        root = self._initialized_model()
        self._set_sentinels(root)
        root.restore(save_path).assert_consumed().run_restore_ops()
        self._check_sentinels(root)

if __name__ == "__main__":
  test.main()


from collections import deque

with open("puzzle.txt", "r") as puzzle:
    lines = puzzle.read()

def part_1() -> int:
    deck = deque(maxlen=4)
    input = iter(lines)
    letter_no = 0
    while True:
        if len(deck) == 4 and len(set(deck)) == 4:
            return letter_no
        else:
            deck.append(next(input))
            letter_no += 1

def part_2() -> int:
    input = iter(lines)
    letter_no = 14
    marker = [next(input) for _ in range(14)]
    while True:
        if len(set(marker)) == 14:
            return letter_no
        else:
            del marker[0]
            marker.append(next(input))
            letter_no += 1

print("Part 1 =", part_1())
print("Part 2 =", part_2())


import re
import os
from shop.admin.routes import admin
from flask import redirect, render_template, url_for, flash, request, session
from werkzeug.utils import secure_filename
from shop import db, app, photos
from .models import Brand, Category, Color, add_img, add_product
from .forms import Addproducts
#from shop.products.models import addproducts

def drink(sentence):
    print(f'\n\n{sentence}:\t', sentence, '\n\n\n')

@app.route('/addcat', methods=['GET', 'POST'])
def addcat():
    if 'username' not in session:
        flash('please loggin first', 'danger')
        return redirect(url_for('login'))

    if request.method == 'POST':
        get_brand = request.form.get('category')
        cat = Category(name=get_brand)
        db.session.add(cat)
        flash(f'the category {get_brand} was added to your database', 'success')
        db.session.commit()
        return redirect(url_for('addcat'))
    return render_template('products/addcat.html')

@app.route('/updatecats/<int:id>', methods=['GET', 'POST'])
def updatecategory(id):
    if 'username' not in session:
        flash('please loggin first', 'danger')
        return redirect(url_for('login'))

    #print(f'\n\nid:\t', id, '\n\n\n')
    updatecat = Category.query.get_or_404(id)# models so sql request for getting id
    categories = request.form.get('category')
    #print(f'\n\nfile:\t', categories, '\n\n\n')
    if request.method == 'POST':
        updatecat.name = categories
        flash(f'your category has been updated', 'success')
        db.session.commit()
        return redirect(url_for('category'))

    return render_template('/products/updatebrands.html', title='update category page', updatecat=updatecat)

@app.route('/deletecat/<int:id>', methods=['POST', 'GET'])
def deletecategory(id):
    cat = Category.query.get_or_404(id)
    #drink('delete sent')
    db.session.delete(cat)
    db.session.commit()
    flash(f'the category name: {cat}, was successfully deleted', 'success')
    return redirect(url_for('category'))

@app.route('/addbrand', methods=['GET', 'POST'])
def addbrand():
    if 'username' not in session:
        flash('please loggin first', 'danger')
        return redirect(url_for('login'))

    if request.method == 'POST':
        get_brand = request.form.get('brand')
        brand = Brand(name=get_brand)
        db.session.add(brand)
        flash(f'the brand {get_brand} was added to your database', 'success')
        db.session.commit()
        return redirect(url_for('addbrand'))
    return render_template('products/addbrand.html')

@app.route('/updatebrands/<int:id>', methods=['GET', 'POST'])
def updatebrand(id):
    if 'username' not in session:
        flash('please loggin first', 'danger')
        return redirect(url_for('login'))

    #print(f'\n\nid:\t', id, '\n\n\n')
    updatebrand = Brand.query.get_or_404(id)
    brand = request.form.get('brand')
    #print(f'\n\nfile:\t', brand, '\n\n\n')
    if request.method == 'POST':
        updatebrand.name = brand
        flash(f'your brand has been updated', 'success')
        db.session.commit()
        return redirect(url_for('brands'))

    return render_template('products/updatebrands.html', title='update brand page', updatebrand=updatebrand)

@app.route('/deletebrand/<int:id>', methods=['POST', 'GET'])
def deletebrand(id):
    brand = Brand.query.get_or_404(id)
    db.session.delete(brand)
    db.session.commit()
    flash(f'the brand name: {brand}, was successfully deleted', 'success')
    return redirect(url_for('brands'))

ALLOWED_EXTENSIONS = set(['png', 'jpg', 'jpeg', 'gif'])
def allowed_file(filename):
    return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS

@app.route('/addproduct', methods=['GET', 'POST'])
def upload():
    if 'username' not in session:
        flash('please loggin first', 'danger')
        return redirect(url_for('login'))

    # DB fetching for html selector
    brands = Brand.query.all()
    category = Category.query.all()
    colors = Color.query.all()
    form = Addproducts(request.form)
    images = []# nom des images que je veux associer
    if request.method == 'POST':
        files = request.files.getlist('file[]')
        for file in files:
            if file and allowed_file(file.filename):
                filename = secure_filename(file.filename)
                file.save(os.path.join(app.config ['UPLOAD_FOLDER'], filename))
                images.append(file.filename)
                flash(f"file:\t{filename.lower()}")
                path = f'{os.getcwd()}\\static\\images\\{file.filename}'
                product_name = form.name.data
                #print(f'\n\nfile:\t', path, '\n\n\n')
                adding = add_img(name=file.filename, product_name=product_name, path= path)
                db.session.add(adding)
                db.session.commit()

        name = form.name.data
        price = form.price.data
        discount = form.discount.data
        stock = form.stock.data
        colors = request.form.get('color_name')
        description = form.description.data

        brand = request.form.get('brand')
        category = request.form.get('category')
        #print(f'\n\nfile:\t', brand, '\n\n\n')
        # if file and allowed_file(file.filename):
        try:
            image_1 = request.files.get('image_1')
            image_1_name = secure_filename(image_1.filename)
            image_1.save(os.path.join(app.config ['UPLOAD_FOLDER'], image_1_name))

            image_2 = request.files.get('image_2')
            image_2_name = secure_filename(image_2.filename)
            image_2.save(os.path.join(app.config ['UPLOAD_FOLDER'], image_2_name))

            image_3 = request.files.get('image_3')
            image_3_name = secure_filename(image_3.filename)
            image_3.save(os.path.join(app.config ['UPLOAD_FOLDER'], image_3_name))
        except FileNotFoundError:
            print("no picture into form")

            image_1_name, image_2_name, image_3_name = 'NaN'
            # that work but all or nothing

        #print(f'\n\nfile:\t', files.filename, '\n\n\n')
        adding = add_product(name=name, price=price, discount=discount,
                             stock=stock, colors_id=colors, description=description,
                             brand_id=brand, category_id=category, image_1=image_1_name,
                             image_2=image_2_name, image_3=image_3_name)
        db.session.add(adding)
        db.session.commit()
        flash(f'the product {name} was successfuly added to your database', 'success')

        return render_template('products/addproduct.html')

    return render_template('products/addproduct.html', title='add product page',brands=brands,categories=category, colors=colors)# form=form,  categories=category)

@app.route('/updateproduct/<int:id>', methods=['GET', 'POST'])
def updateproduct(id):
    if 'username' not in session:
        flash('please loggin first', 'danger')
        return redirect(url_for('login'))

    #print(f'\n\nid:\t', id, '\n\n\n')
    brands = Brand.query.all()
    categories = Category.query.all()
    colors = Color.query.all()

    updateproduct = add_product.query.get_or_404(id)
    form = Addproducts(request.form)
    name = request.form.get('name')
    if request.method == 'POST':
        color = request.form.get('color')
        color_id = Color.query.filter_by(name_color=color).first()

        print(color_id)
        updateproduct.color = color_id# I need color id here
        print(request.form.get('color'))
        updateproduct.name = form.name.data
        updateproduct.price = request.form.get('price')
        updateproduct.dicsount = form.discount.data
        updateproduct.stock = form.stock.data

        updateproduct.image_1
        updateproduct.image_2
        updateproduct.image_3

        db.session.commit()
        flash(f'the product {name} was successfuly added to your database', 'success')
        return redirect(url_for('admin'))

    return render_template('products/updateproducts.html', title='update product page',
            updateproduct=updateproduct, brands=brands, categories=categories, colors=colors)

@app.route('/deleteproduct/<int:id>', methods=['POST', 'GET'])
def deleteproduct(id):
    product = add_product.query.get_or_404(id)
    #drink('delete sent')
    db.session.delete(product)
    db.session.commit()
    flash(f'the product name: {product}, was successfully deleted', 'success')
    return redirect(url_for('admin'))

@app.route('/addcolor', methods=['GET', 'POST'])
def add_color():
    if 'username' not in session:
        flash('please loggin first', 'danger')
        return redirect(url_for('login'))

    if request.method == 'POST':
        get_color = request.form.get('name_color')

        color = Color(name_color=get_color)
        print(request.form)
        db.session.add(color)
        db.session.commit()
        flash(f'the color {get_color} was successfuly added to your database', 'success')

        return render_template('products/add_colors.html')

    return render_template('products/add_colors.html', title='add color page')# form=form,  categories=category)


from turtle import Turtle
FONT1 = ("Courier", 24, "normal")
FONT2 = ("Courier", 40, "normal")

class Scoreboard(Turtle):
    def __init__(self):
        self.level = 1
        super().__init__()

    def write_level(self):
        self.penup()
        self.hideturtle()
        self.color('black')
        self.goto(-240, 240)
        self.clear()
        self.write(f'Level: {self.level}', font=FONT1)

    def game_over(self):
        self.goto(0, 0)
        self.color('red')
        self.write('GAME OVER', align='center', font=FONT2)


'''
File to launch the voltage and injection error scans.

Author: Birk Emil Karlsen-Bæck
'''

# Imports -------------------------------------------------------------------------------------------------------------
import numpy as np
import argparse
import os

# Arguments -----------------------------------------------------------------------------------------------------------
parser = argparse.ArgumentParser(description="This file launches parameter scans in RF voltage and injection energy "
                                             "error.")

parser.add_argument("--emittance", '-em', type=int, default=1,
                    help="Option to either have small emittance (True) or nominal (False).")
parser.add_argument("--intensity", '-in', type=float, default=9.0,
                    help="Intensity of the injected bunch, default is 9.0e9.")
parser.add_argument("--scan_mode", '-sm', type=int, default=1,
                    help="Different parameter scan modes.")

args = parser.parse_args()

# Parameters ----------------------------------------------------------------------------------------------------------
N_t = 30000
EMIT = args.emittance
N_p = args.intensity
IMP = 0

if args.scan_mode == 1:
    Vs = np.array([0.5, 1.0, 1.5])
    Es = np.linspace(0, 50, 11)
elif args.scan_mode == 2:
    Vs = np.array([1.5, 4, 5, 6, 7, 8, 9, 10, 11, 12])
    Es = np.linspace(0, 100, 21)

print(f'Launching {len(Vs) * len(Es)} simulations...')
print(f'Injection energy error: Max = {Es[-1]:.1f} MeV, Min = {Es[0]:.1f} MeV, Step = {Es[2] - Es[1]:.1f} MeV')
print(f'RF voltage: Max = {Vs[-1]:.1f} MV, Min = {Vs[0]:.1f} MV, Step = {Vs[2] - Vs[1]:.1f} MV')
print()

current_working_directory = os.getcwd() + '/'
sim_file = 'launch_simulation.py'

for i in range(len(Vs)):
    for j in range(len(Es)):
        argument_str_i = f'-nt {N_t} -em {EMIT} -im {IMP} -in {N_p} -ee {Es[j]} -rv {Vs[i]}'
        os.system(f'python {current_working_directory}{sim_file} {argument_str_i}')

print('\nDone!')


import dearpygui.dearpygui as dpg
from objects.sensor import Sensor

class SensorHandler:
    def __init__(self, sensor_list):
        self.sensor_list = sensor_list

    def create_sensor(self, sender, data):
        name = dpg.get_value(self.name_c)
        reward = dpg.get_value(self.reward_c)
        data_size = dpg.get_value(self.data_size_c)
        x_cord_s = dpg.get_value(self.x_cord_s_c)
        y_cord_s = dpg.get_value(self.y_cord_s_c)
        hovering_cost = dpg.get_value(self.hovering_cost_c)

        sensor = Sensor(name, reward, data_size, x_cord_s, y_cord_s, hovering_cost)
        self.sensor_list.append(sensor)
        print(f"Sensor {sensor} added")

        dpg.configure_item(item=self.sensor_combo, items=self.sensor_list)

        self.redraw_simulation()
        return self.sensor_list

    def delete_sensor(self, sender, data):
        selected_sensor_str = dpg.get_value(item=self.sensor_combo)
        for sensor in self.sensor_list:
            if str(sensor) == selected_sensor_str:
                selected_sensor = sensor
                break
        else:
            print("No sensor selected")
            return

        self.sensor_list.remove(selected_sensor)
        print(f"Sensor {selected_sensor} removed")

        dpg.configure_item(item=self.sensor_combo, items=self.sensor_list)
        dpg.set_value(item=self.sensor_combo, value="")

        self.redraw_simulation()
        return self.sensor_list

    def set_redraw_simulation(self, redraw_simulation):
        self.redraw_simulation = redraw_simulation

    def create_add_sensor_window(self, width, height, pos):
        with dpg.window(label="Add Sensor", width=width, height=height, pos=pos):
            self.name_c = dpg.add_input_text(label="Name", default_value="pX")
            self.reward_c = dpg.add_input_int(label="Reward", default_value=50)
            self.data_size_c = dpg.add_input_int(label="Data Size", default_value=10)
            self.x_cord_s_c = dpg.add_input_int(label="X", default_value=100)
            self.y_cord_s_c = dpg.add_input_int(label="Y", default_value=100)
            self.hovering_cost_c = dpg.add_input_int(
                label="Hovering Cost", default_value=5
            )
            dpg.add_button(label="Add Sensor", callback=self.create_sensor)

    def create_sensor_list_window(self, width, height, pos):
        with dpg.window(label="Sensor List", width=width, height=height, pos=pos):
            self.sensor_combo = dpg.add_combo(label="Sensor", items=self.sensor_list)
            dpg.add_button(label="Delete Sensor", callback=self.delete_sensor)

    def refresh_sensor_combo(self):
        dpg.configure_item(item=self.sensor_combo, items=self.sensor_list)
        dpg.set_value(item=self.sensor_combo, value="")


from math import inf
from typing import List

class Solution:
    def findLongestChain(self, pairs: List[List[int]]) -> int:
        pairs.sort(key=lambda x: x[1])
        ans, cur = 0, -inf
        for l, r in pairs:
            if cur < l:
                ans += 1
                cur = r
        return ans


"""empty message

Revision ID: 4d1af98d3ffe
Revises: 7d94df3bbf97
Create Date: 2020-03-02 22:16:16.992025

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '4d1af98d3ffe'
down_revision = '7d94df3bbf97'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('users', sa.Column('flash_id', sa.String(length=64), nullable=True))
    op.drop_constraint('users_person_id_key', 'users', type_='unique')
    op.create_unique_constraint(None, 'users', ['flash_id'])
    op.drop_column('users', 'person_id')
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('users', sa.Column('person_id', sa.VARCHAR(length=64), autoincrement=False, nullable=True))
    op.drop_constraint(None, 'users', type_='unique')
    op.create_unique_constraint('users_person_id_key', 'users', ['person_id'])
    op.drop_column('users', 'flash_id')
    # ### end Alembic commands ###


import torch
import torch.nn as nn
import os, sys
import cv2
import numpy as np
import tqdm
from PIL import Image
from pathlib import Path
from time import gmtime, strftime
# Import test.py from repo
sys.path.insert(1, '/home/alex/facade/SSIW/Test_Minist/tools/')
import test as t
from utils.get_class_emb import create_embs_from_names
from utils.segformer import get_configured_segformer

from torch.utils.data import Dataset, DataLoader, random_split
from torch.cuda.amp import autocast, GradScaler
from torch.optim.lr_scheduler import StepLR
import torchvision.transforms as transforms

import torch.optim as optim

class CMPDataset(Dataset):
    def __init__(self, root_dir, transform=None, resize=True):
        self.root_dir = root_dir
        self.transform = transform
        dataset_files = os.listdir(root_dir)
        self.file_names = sorted(set([Path(file).stem for file in dataset_files]))
        self.resize = resize
        self.original_sizes = {}

        # From imagenet, as in paper
        self.normalize = transforms.Normalize(
            mean=[0.485, 0.456, 0.406],
            std=[0.229, 0.224, 0.225]
        )

    def __len__(self):
        return len(self.file_names)

    def __getitem__(self, index):
        img_path = os.path.join(self.root_dir, self.file_names[index] + ".jpg")
        img = cv2.imread(img_path, -1)[:, :, ::-1]

        label_path = os.path.join(self.root_dir, self.file_names[index] + ".png")
        label = Image.open(label_path)
        label = np.array(label)

        h, w = img.shape[:2]
        self.original_sizes[index] = (h, w)

        if self.resize:
            img = cv2.resize(img, (520, 520), interpolation=cv2.INTER_LINEAR)
            label = cv2.resize(label, (520, 520), interpolation=cv2.INTER_NEAREST)

        # Apply transformations
        if self.transform is not None:
            transformed = self.transform(image=img, mask=label)
            img = transformed['image']
            label = transformed['mask']

        img = self.normalize(img)

        # Convert label to tensor and return image and label
        label = torch.from_numpy(label).long()
        img = torch.from_numpy(np.ascontiguousarray(img)).to(torch.float32)
        return img, label

    def get_batch(self, indices):
        # Load images and labels for the batch
        batch_images = []
        batch_labels = []
        for index in indices:
            img, label = self.__getitem__(index)
            print(f'img: {img.shape}')
            print(f'label: {label.shape}')
            # print(f'manual transpose of img: {np.transpose(img, (2,0,1)).shape}')
            # img = np.transpose(img, (2, 0, 1))
            # print('transpose done')
            print(f'new img shape {img.shape}')
            batch_images.append(img)
            batch_labels.append(label)

        print(batch_images)
        batch_images = torch.stack(batch_images, dim=0)
        print(f'b_images shape: {batch_images.shape}')
        batch_labels = torch.stack(batch_labels, dim=0)

        return batch_images, batch_labels

    def get_original_size(self, index):
        return self.original_sizes[index]

def get_prediction_tau(embs, gt_embs_list, tau):
    prediction = []
    logits = []
    B, _, _, _ = embs.shape
    for b in range(B):
        score = embs[b, ...]
        score = score.unsqueeze(0)
        emb = gt_embs_list
        emb = emb / emb.norm(dim=1, keepdim=True)
        score = score / score.norm(dim=1, keepdim=True)
        score = score.permute(0, 2, 3, 1) @ emb.t()
        # [N, H, W, num_cls] You maybe need to remove the .t() based on the shape of your saved .npy
        score = score / tau
        score = score.permute(0, 3, 1, 2)  # [N, num_cls, H, W]
        prediction.append(score.max(1)[1])
        logits.append(score)
    if len(prediction) == 1:
        prediction = prediction[0]
        logit = logits[0]
    else:
        prediction = torch.cat(prediction, dim=0)
        logit = torch.cat(logits, dim=0)
    return logit

class PredictionCriterion(nn.Module):
    def __init__(self):
        super().__init__()
        self.cross_entropy = nn.CrossEntropyLoss(ignore_index=255)
        # self.temperature = nn.Parameter(torch.tensor([1.0]), requires_grad=True).to(device)

    def forward(self, emb, gt_embs_list, labels):
        logits = t.get_prediction(emb, gt_embs_list)
        loss = self.cross_entropy(logits, labels - 1)
        return loss

def evaluate(model, criterion, test_loader, device):
    model.eval()
    correct = 0
    total_loss = 0
    with torch.no_grad():
        for images, labels in test_loader:
            images = images.permute(0, -1, 1, 2).to(torch.float32)
            images = images.to(device)
            labels = labels.to(device)

            emb, _, _ = model(inputs=images, label_space=['universal'])
            total_loss += criterion(emb, gt_embs_list, labels).item()

        total_loss /= len(test_loader)
        print('Test set: Average loss: {:.4f}'.format(total_loss))
        return total_loss

if __name__ == '__main__':

    device = 'cuda' if torch.cuda.is_available() else 'cpu'

    # Set up data
    dataset_dir = '/home/alex/facade/CMP_facade'
    with open(os.path.join(dataset_dir, 'label_names.txt')) as file:
        class_dict = {line.split()[0]: line.split()[1] for line in file}
    class_labels = list(class_dict.values())
    dataset = CMPDataset(os.path.join(dataset_dir, 'base'), resize=True)

    # Define sentences:
    sentences = {
        'background': 'the part of a picture, scene, or design that forms a setting for the main figures or objects, or appears furthest from the viewer.',
        'facade': 'the principal front of a building, that faces on to a street or open space.',
        'window': 'an opening in the wall or roof of a building or vehicle, fitted with glass in a frame to admit light or air and allow people to see out.',
        'door': 'a hinged, sliding, or revolving barrier at the entrance to a building, room, or vehicle, or in the framework of a cupboard.',
        'cornice': 'which is generally any horizontal decorative moulding that crowns a building or furniture element for example, the cornice over a door or window',
        'sill': 'a decorative (raised) panel or stripe under a window',
        'balcony': 'a platform projecting from the wall of a building, supported by columns or console brackets, and enclosed with a balustrade, usually above the ground floor.',
        'blind': 'a type of functional window covering used to obstruct light, made of hard or soft material; i.e. shutters, roller shades, wood blinds, standard vertical, and horizontal blinds',
        'deco': 'a piece of original art, paintings, reliefs, statues.',
        'molding': 'a horizontal decorative stripe across the facade, possibly with a repetitive texture pattern, used to cover transitions between surfaces or for decoration',
        'pillar': 'a vertical decorative stripe across the facade which is made of stone, or appearing to be so.',
        'shop': 'a building or part of a building where goods or services are sold or advertised.'}

    # For consistency with paper include this preamble
    def add_preamble(x): return f'This is a picture of a {x}, '
    sentences = {k: add_preamble(k)+v for (k,v) in sentences.items()}

    gt_embs_list = create_embs_from_names(class_labels, other_descriptions=sentences).float()
    gt_embs_list = gt_embs_list.to(device)
    id_to_label = {i: v for i, v in enumerate(class_labels)}
    id_to_label[255] = 'unlabel'

    num_model_classes = 512 # to initially load in the weights
    model = get_configured_segformer(num_model_classes, criterion=None, load_imagenet_model=False)
    model = torch.nn.DataParallel(model)
    ckpt_path = '/home/alex/facade/SSIW/Test_Minist/models/segformer_7data.pth'
    checkpoint = torch.load(ckpt_path, map_location='cpu')['state_dict']
    ckpt_filter = {k: v for k, v in checkpoint.items() if 'criterion.0.criterion.weight' not in k}
    model.load_state_dict(ckpt_filter, strict=False)

    for name, param in model.named_parameters():
        param.requires_grad = True
        #if 'auxi_net' not in name and 'head' not in name:
        #    param.requires_grad = False

    model = model.to(device)

    batch_size = 8
    train_size = int(0.9 * len(dataset))
    test_size = len(dataset) - train_size
    train_dataset, test_dataset = random_split(dataset, [train_size, test_size])
    train_loader = DataLoader(train_dataset, batch_size, shuffle=True)
    test_loader = DataLoader(test_dataset, batch_size, shuffle=False)

    optimizer = optim.Adam(model.parameters(), lr=1e-1)
    criterion = PredictionCriterion()
    scheduler = StepLR(optimizer, step_size=20, gamma=0.01)
    accumulation_steps = 3

    min_loss = float('inf')
    losses = []

    train_losses = []
    batch_losses = []
    eval_losses = []

    model.train()

    scaler = GradScaler()
    save_itr = 5
    save_dir = strftime("%Y%m%d%H%M", gmtime())

    for epoch in range(100):

        for batch, (images, labels) in tqdm.tqdm(enumerate(train_loader), total=len(train_loader)):

            # N, C, , ,

            mean, std = get_imagenet_mean_std()

            images = images.permute(0, -1, 1, 2).to(torch.float32)
            images = images.half().to(device)
            labels = labels.to(device)

            # optimizer.zero_grad()

            with autocast():
                emb, _, _ = model(inputs=images, label_space=['universal'])
                loss = criterion(emb, gt_embs_list, labels)
                loss = loss / accumulation_steps

            scaler.scale(loss).backward()

            if (batch + 1) % accumulation_steps == 0:
                print('Update loss')
                scaler.step(optimizer)
                scaler.update()
                optimizer.zero_grad()
                batch_losses.append(loss.item())

            print(f" Epoch :{epoch} Images {batch*batch_size}/{train_size}. Loss: {loss.item()} Lr: {scheduler.get_last_lr()}")

        scheduler.step()
        train_losses.append(loss.item())
        eval_loss = evaluate(model, criterion, test_loader, device)
        eval_losses.append(eval_loss)

        if epoch + 1 % save_itr == 0:
            if not os.path.exists(save_dir):
                os.makedirs(save_dir)
            print(f'Save to {save_dir}')
            torch.save(model.state_dict(), os.path.join(save_dir, f"model_FT_320_weights_epoch{epoch + 1}.pt"))

            import csv
            with open(os.path.join(save_dir, 'epoch_losses.csv'), 'w', newline='') as f:
                writer = csv.writer(f)
                writer.writerow(['train_loss', 'eval_loss'])
                for train_loss, eval_loss in zip(train_losses, eval_losses):
                    writer.writerow([train_loss, eval_loss])

            with open(os.path.join(save_dir, 'batch_loss.csv'), 'w', newline='') as f:
                writer = csv.writer(f)
                writer.writerow(['train_loss', 'eval_loss'])
                for batch_losses, eval_loss in zip(batch_losses, eval_losses):
                    writer.writerow([batch_losses, eval_loss])


from django.db import models
from user_profile.models import User
import uuid
from django_comments_xtd.signals import should_request_be_authorized
from django.dispatch import receiver

class Post(models.Model):
    id           = models.UUIDField(primary_key=True, editable=False, default=uuid.uuid4)
    user      = models.ForeignKey(User, on_delete=models.CASCADE)
    description  = models.CharField(max_length=150, null=True)
    cover_image  = models.ImageField(upload_to='coverimg/')
    audio_field  = models.FileField(upload_to='audio/')
    created_at   = models.DateTimeField(auto_now_add=True, null=True)
    def __str__(self):
        if len(self.description) > 15:
            return f"{self.user} posted a tale with description as {self.description[0:15]}..."
        return f"{self.user} posted a tale with description as {self.description}"

@receiver(should_request_be_authorized)
def my_callback(sender, comment, request, **kwargs):
    if (request.user and request.user.is_authenticated):
        return True


from django.conf.urls import patterns, url
import views, api

urlpatterns = patterns('',

    #Tasks
    url(r'^task/$', views.task_list, name="tasks"),
    url(r'^task/(\d+)$', views.task_detail, name="task_detail"),
    url(r'^task/add$', views.task_create, name="task_add"),
    url(r'^task/edit/(\d+)$', views.task_update, name="task_edit"),
    url(r'^task/delete/(\d+)$', views.task_delete, name="task_delete"),
    url(r'^task/data$', views.task_data, name="task_data"),
    url(r'^task/history/$', views.task_history, name="task_history"),
    url(r'^task/history/data$', views.task_history_data, name="task_history_data"),
    url(r'^task/history/output/(\d+)$', views.task_output, name="task_output"),
    url(r'^task/history/log/(\d+)$', views.task_log, name="task_log"),

    #Joblist
    url(r'^joblist/$', views.joblist_list, name="joblists"),
    url(r'^joblist/(\d+)$', views.joblist_detail, name="joblist_detail"),
    url(r'^joblist/add$', views.joblist_create, name="joblist_add"),
    url(r'^joblist/edit/(\d+)$', views.joblist_update, name="joblist_edit"),
    url(r'^joblist/delete/(\d+)$', views.joblist_delete, name="joblist_delete"),
    url(r'^joblist/data/(\d+)$', views.joblist_jobs_data, name="joblist_jobs_data"),
    url(r'^joblist/data/\d+/(\d+)/$', views.joblist_job_detail_data, name="joblist_job_detail_data"),

    #Job
    url(r'^job/$', views.job_list, name="jobs"),
    url(r'^job/(\d+)$', views.job_detail, name="job_detail"),
    url(r'^job/add$', views.job_create, name="job_add"),
    url(r'^job/edit/(\d+)$', views.job_update, name="job_edit"),
    url(r'^job/delete/(\d+)$', views.job_delete, name="job_delete"),
    url(r'^job/data$', views.job_data, name="job_data"),

    url(r'^api$', api.api, name="api"),
    #Search
    url(r'^search/$', views.search, name="search"),

)