load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")

def maybe(repo_rule, name, **kwargs):
    """Defines a repository if it does not already exist.
    """
    if name not in native.existing_rules():
        repo_rule(name = name, **kwargs)

def github_archive(name, repo_name, commit, sha256 = None, kind = "zip"):
    """Defines a GitHub commit-based repository rule."""
    project = repo_name[repo_name.index("/"):]
    http_archive(
        name = name,
        sha256 = sha256,
        strip_prefix = "{project}-{commit}".format(project = project, commit = commit),
        urls = [u.format(commit = commit, repo_name = repo_name, kind = kind) for u in [
            "https://mirror.bazel.build/github.com/{repo_name}/archive/{commit}.{kind}",
            "https://github.com/{repo_name}/archive/{commit}.{kind}",
        ]],
    )

def POM_com_rule_repositories():
    """Defines external repositories for CFS Bazel rules.
    These repositories must be loaded before calling external.bzl%POM_dependencies.
    """
    maybe(
        http_archive,
        name = "rules_java",
        url = "https://github.com/bazelbuild/rules_java/archive/973a93dd2d698929264d1028836f6b9cc60ff817.zip",
        sha256 = "a6cb0dbe343b67c7d4f3f11a68e327acdfc71fee1e17affa4e605129fc56bb15",
        strip_prefix = "rules_java-973a93dd2d698929264d1028836f6b9cc60ff817",
    )

    maybe(
        http_archive,
        name = "rules_proto",
        sha256 = "e4fe70af52135d2ee592a07f916e6e1fc7c94cf8786c15e8c0d0f08b1fe5ea16",
        strip_prefix = "rules_proto-97d8af4dc474595af3900dd85cb3a29ad28cc313",
        url = "https://github.com/bazelbuild/rules_proto/archive/97d8af4dc474595af3900dd85cb3a29ad28cc313.zip",
    )

    maybe(
        http_archive,
        name = "build_bazel_rules_nodejs",
        sha256 = "b6670f9f43faa66e3009488bbd909bc7bc46a5a9661a33f6bc578068d1837f37",
        urls = ["https://github.com/bazelbuild/rules_nodejs/releases/download/1.3.0/rules_nodejs-1.3.0.tar.gz"],
    )

    maybe(
        http_archive,
        name = "rules_jvm_external",
        sha256 = "e5b97a31a3e8feed91636f42e19b11c49487b85e5de2f387c999ea14d77c7f45",
        strip_prefix = "rules_jvm_external-2.9",
        urls = ["https://github.com/bazelbuild/rules_jvm_external/archive/2.9.zip"],
    )

    maybe(
        http_archive,
        name = "rules_python",
        sha256 = "e5470e92a18aa51830db99a4d9c492cc613761d5bdb7131c04bd92b9834380f6",
        strip_prefix = "rules_python-4b84ad270387a7c439ebdccfd530e2339601ef27",
        urls = [
            "https://mirror.bazel.build/github.com/bazelbuild/rules_python/archive/4b84ad270387a7c439ebdccfd530e2339601ef27.tar.gz",
            "https://github.com/bazelbuild/rules_python/archive/4b84ad270387a7c439ebdccfd530e2339601ef27.tar.gz",
        ],
    )


import requests # Handles http requests
import base64 # base64 encoding for sending BLOBs to template as text
import sys # for debug flushes
import math # used for floor() and ceil() functions
import time # used for time.sleep() to delay after loading web page
import hashlib # used for a checksum stored in the Image table
from io import BytesIO # Handle binary data to save img_data to database
from .models import Image, Search # Search and Image models (objects for database)
from PIL import Image as PILImage # For raster based image manipulation
from django.shortcuts import render, redirect # For rendering templates with context data and returning HTTP responses
from django.conf import settings # To allow access to constants set in settings.py
from bs4 import BeautifulSoup # For parsing html content
from urllib.parse import urljoin, urlparse # For combining relative references to full URL
from django.http import HttpResponse # For determining HttpResponse types
from django.utils import timezone # For displaying timezone
from selenium import webdriver # for webscraping and screencapturing
from selenium.webdriver.common.by import By

# Used to output debug statements to the console as the program runs
def debug(str):
    print(f"DEBUG: {str}")
    sys.stdout.flush()
    return

# Handler function to get_web_response for index view
# Allows extraction of the final URL and the associated response, or handles and logs any errors that occur
# during the process.
def get_web_response_handler(request, url_with_scheme, url_entered):
    # Get HTML content of the URL, but handle redirects manually, to update url to new location
    try:
        response = requests.get(url_with_scheme, allow_redirects=False)
        response.raise_for_status()
    except requests.exceptions.RequestException as e:
        debug(f"Error trying to get {url_with_scheme}: {e}")
        return None, None, f"Failed to get page: {e}"

    debug(f"in index(), returned from requests.get with response={response}")

    if response.status_code in {200,201}: # if status ok or created (kinda ok)
        url = url_entered
    elif response.status_code in {301,302,307,308}: # temporary or permanent redirect
        url = response.headers['Location']
        parsed_url = urlparse(url)
        scheme = parsed_url.scheme if parsed_url.scheme else 'http' # Figure out scheme (e.g. http)
        url = f"{scheme}://{parsed_url.netloc}{parsed_url.path}" # Rebuild URL

        try:
            response = requests.get(url)
            response.raise_for_status()
        except requests.exceptions.RequestException as e:
            debug(f"Error trying to redirect to {response.headers['Location']}: {e}")
            return None, None, f"Failed to redirect from {url_entered}, to {url}: {e}"

        if response.status_code in {200,201}: # if redirected get() status ok or created (kinda ok)
            debug(f"Successfully redirected to {url}")
        else:
            debug(f"Failed redirect to get {url}, status code {response.status_code}")
            return None, None, f"Failed redirect to get {url_entered} HttpResponse:{response.status_code}"
    else:
        debug(f"Status code {response.status_code} trying to get {url_entered}")
        return render(request, 'fail.html',{'error_message': f"Could not get {url_entered} HttpResponse:{response.status_code}"})
    return url, response, None

# The purpose of this function is to handle the srcset attribute of an img tag, extract the URL-size pairs, and select
# the URL with the largest size that passes certain checks. It ensures that images with excessive sizes are not chosen
# and that the selected URL is a valid image.
def pick_an_image_from_srcset(image_srcset, page_url):
    # an img srcset in html will list URLs of the same image in different sizes,
    # separated by commas, to allow picking the best size for a layout.
    # if we pick an image that's many megabytes, it chokes saving to the database.
    # so if the size is capped in retrieve_and_validate_img_handlr(), and we pick the
    # biggest image url in the that wasn't filtered out. if there are none we skip this image.
    # it can be slow checking a lot of image sizes...if speed were more crucial, maybe
    # check just the first and last urls, as they're probably in ascending or descending
    # order by size, so one or the other might fit our criteria.

    debug(f"In pick_an_image_from_srcset(), image_srcset={image_srcset} page_url={page_url}")

    biggest_size = 0
    biggest_url = None

    # Brute force way to parse image_srcset, first break as left and right of ' ',
    # then if there's another comma, take just the portion right of the comma.
    # That handles srcsets that have commas within the URLs, not to be confused
    # with commas separating URL-size pairs, as in:
    #  "https://www.google.com?TYPE=1,DIR 1x, https://www.google.com?TYPE=2,DIR 2x"
    while ' ' in image_srcset:
        url_to_check = image_srcset.split(' ')[0]
        image_srcset = after_substr(image_srcset,' ') # set it to everything after the ' '
        if ',' in image_srcset:
            image_srcset = after_substr(image_srcset,',').strip() # now set it to everything after the ','

        # Join the web page URL prefix to the image URL if the image URL is a relative link
        if not url_to_check.startswith('http') and not url_to_check.startswith('data:'):
            url_to_check = urljoin(page_url, url_to_check)

        response_content, content_type = retrieve_and_validate_img_handler(url_to_check)
        if response_content is not None:
            image_size_bytes = len(response_content)

            debug(f"url_to_check={url_to_check[:40]}... was size={image_size_bytes}")
            # Update the biggest area and URL if necessary
            if image_size_bytes > biggest_size:
                biggest_size = image_size_bytes
                biggest_url = url_to_check

    if biggest_url is None:
        debug(f"None of the images were suitable")
        return

    img_url = biggest_url
    debug(f"Chose best size from srcset, img_url = {img_url}, size={biggest_size}")
    return img_url

# return everything after a given substring in a string
def after_substr(string, substring):
    index = string.find(substring)
    if index != -1:
        return string[index + len(substring):]
    else:
        return ""

# Retrieve the image data from the URL
def retrieve_and_validate_img_handler(img_url):

    # Don't store images in database if over max size, currently 1024576.
    # It can handle bigger, but may affect performance at some level,
    # and that seems like a reasonable limit.
    maximum_size_to_save = 2000000

    if img_url.split('/')[0] == 'data:image':
        # If img_url is simply data, like data:image/x-png;base64,iVBORw0KGgoAAAANSUh...,
        # then that contains all the data we need without retrieving it from the web.

        debug(f"Using data:image from URL")

        if len(img_url) > maximum_size_to_save: # Return None if too big
            return None, None

        # img_url is of the form "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgA..."

        content_type = after_substr(img_url,':').split(';')[0] # Extract "image/png" between the first ':' and first ';'
        base64_string = after_substr(img_url,',') # Extract the base64-encoded data after the first comma

        image_bytes = base64.b64decode(base64_string) # Convert the base64 string to bytes
        return image_bytes, content_type
    else:
        try:
            response = requests.get(img_url)
            response.raise_for_status()
        except Exception as e:
            debug(f"Error retrieving image {img_url}: {e}")
            return None, None

        if len(response.content) > maximum_size_to_save: # Return None if too big
            return None, None

        content_type = response.headers.get('content-type')
        if not content_type.startswith("image/"): # if not an image type of content, skip it
        #    debug(f"Invalid image content type for {img_url}: {content_type}")
            return None, None # Not image content_type (maybe "text/html") so skip to next loop iterator

        debug(f"retrieve_and_validate_img_handler() returning content_type: {content_type}")
        return response.content, content_type

# The purpose of this function is to handle the saving of image data into the database. It converts
# the image data into a `BytesIO` object, generates a unique identifier for the image, creates an `Image`
# object, and saves it to the database.
def database_save_handler(image_data, search, img_url, content_type):
    debug(f"In database_save_handler, passed image_data, search (.id={search.id}), img_url={img_url}, content_type={content_type}")

    try:
        img_data = BytesIO(image_data)
    except Exception as e:
        debug(f"Error setting img_data to BytesIO() for {img_url}: {e}")
        return
#    debug(f"got img_data from BytesIO, img_data={img_data}")

    unique_search_image = str(search.id) + '+' + hashlib.md5(image_data).hexdigest() # search_id + 32-character checksum of image data

    try:
        img_obj = Image(search=search, url=img_url[:255], image=img_data.getvalue(), content_type=content_type[:64], unique_search_image = unique_search_image[:64])
#        debug(f"did Image() call")
        img_obj.save()
#        debug(f"did img_obj.save")
    except Exception as e:
        debug(f"Error saving image to database: {e}")
        return
    return True

# Retrieves and stores image from 'image_url' in database Image table, associating it with a search in Search table.
# 'page_url' is the web page from which the 'image_url' was found, needed in case 'image_URL' is a relative link.
def store_image_from_url_in_database(search,image_url,page_url):

    # Check parameters sent to function
    if search is None or image_url is None or image_url == '' or page_url is None:
        return

    # Join the web page URL prefix to the image URL if the image URL is a relative link
    if not image_url.startswith('http'):
        image_url = urljoin(page_url, image_url)

    # Call handler function for retrieving and validating img
    response_content, content_type = retrieve_and_validate_img_handler(image_url)

    # If we got a response, call handler function for saving objects to database
    if response_content:
        if len(image_url) > 255:
            image_url = image_url[:255]
        database_save_handler(response_content, search, image_url, content_type)
    return

# The purpose of this function is to use a webdriver to load a web page, capture a screenshot of the page,
# and process specific elements (img and svg) to save image URLs or image data to the database.
def scrape_page_with_webdriver(search,url):

    debug(f"starting scrape_page_with_webdriver(url), url={url}")

    # Set Chrome webdriver options and instantiate the driver
    try:
        options = webdriver.ChromeOptions()   # Options for Chrome webdriver
        options.add_argument('--headless')    # Run in headless mode (no GUI)
        options.add_argument('--disable-gpu') # disable GPU usage (avoids some bugs)
        driver = webdriver.Chrome(executable_path=settings.CHROME_DRIVER_EXECUTABLE_LOCATION, options=options)
    except Exception as e:
        debug(f"Exception opening Chrome webdriver: {e}")

    # Set Firefox webdriver options and instantiate the driver
    # try:
    #     options = webdriver.FirefoxOptions() # Options for Firefox webdriver
    #     options.headless = True              # Run in headless mode (no GUI)
    #     options.binary_location = FIREFOX_BROWSER_EXECUTABLE_LOCATION # Actual firefox location
    #     driver = webdriver.Firefox(executable_path=FIREFOX_DRIVER_EXECUTABLE_LOCATION, options=options)
    # except Exception as e:
    #     debug(f"Exception opening Firefox webdriver: {e}")

    # Get URL using web driver
    debug(f"Load {url} with webdriver")
    try:
        driver.get(url)
    except Exception as e:
        debug(f"Exception opening URL {url} with webdriver: {e}")
        return

    debug(f"Loaded {url} in webdriver")

    # Set window size for webdriver (virtual window size, since it's operating in 'headless' mode)
    browser_width = 1920
    browser_height = 6000
    try:
        driver.set_window_size(browser_width, browser_height)
    except Exception as e:
        debug(f"Exception setting virtual window size webdriver: {e}")
        return

    # Sleep to give page time to load
    time.sleep(3)

    # NOTE: Tried using this to wait until Selenium reported the web elements were
    # all visible, but didn't seem to work. Also tried with presence_of_all_elements.
    # Just using 3 second delay for now.
    #
    # Wait for all elements to be visible
    #
    # (from dependencies required for this):
    # from selenium.webdriver.support.ui import WebDriverWait
    # from selenium.webdriver.support import expected_conditions
    # from selenium.common.exceptions import TimeoutException
    # from selenium.webdriver.common.by import By
    #
    # try:
    #     images = WebDriverWait(driver, 10).until(expected_conditions.visibility_of_all_elements_located((By.TAG_NAME, 'img')))
    # except TimeoutException:
    #     return render(request, 'fail.html', {'error_message': f"Timed out waiting for web page elements to load"})

    screen_png = driver.get_screenshot_as_png() # saves screenshot of entire page
    screen_whole = PILImage.open(BytesIO(screen_png)) # uses PIL library to open image in memory

    element_tags_to_process = ('img','svg') # WebElement tag_names to process

    # For loop does one pass processing all 'img' element types, then a second for all 'svg' element
    # types. Type 'img' provide URLs we can fetch, while 'svg' is used for inline SVG instructions on
    # a web page (draw circle at x,y, etc.), so in that case we crop the area from a screenshot
    for element_tag in element_tags_to_process:
#        elements = driver.find_elements_by_tag_name(element_tag)
        elements = driver.find_elements(By.TAG_NAME, element_tag)
        debug(f"Number of elements found: {len(elements)}")

        # Iterate over the web elements and store images if suitable
        for element in elements:
            try:
                element_displayed = element.is_displayed()

                # Skip elements that aren't displayed, have no location, or have width or height of zero
                if not element_displayed or not element.location or \
                not element.size or element.size['height'] == 0 or element.size['width'] == 0:
                    continue

                if element_tag == 'img':
                    image_src = element.get_attribute('src')
                    image_srcset = element.get_attribute('srcset')
                elif element_tag == 'svg':
                    image_src = None
                    image_srcset = None
            except Exception as e: # (likely StaleElementReferenceException, but could be timeout or something else)
                debug("Got a stale element or other error processing WebElements from Selenium")
                continue

            debug(f"IMG SRCSET element name={element.tag_name} text={element.text} size = {element.size} location = {element.location}")
            if element_tag == 'img':
                # Add url from image_src (if any) to image_srcset (if any), to compare image_srcset images all together

                if image_src and image_src != '':
                    if image_srcset and image_srcset == '':
                        image_srcset = image_src + ' 1x'
                    else:
                        image_srcset = image_src + ' 1x,' + image_srcset

                debug(f"           checking {image_srcset}")

                if image_srcset == '':
                    debug(f"Skipping img element loop (img={img})")
                    continue  # Skip this image tag since it has no 'src' or 'srcset' attributes
                else:
                    image_src = pick_an_image_from_srcset(image_srcset,url)
                    store_image_from_url_in_database(search, image_src, url)

            elif element_tag == 'svg':
                debug(f"           element={element}")

                # Calculate crop parameters. floor() and ceil() to round to integers in case it comes back float.
                left = math.floor(element.location['x'])
                top = math.floor(element.location['y'])
                right = left + math.ceil(element.size['width'])
                bottom = top + math.ceil(element.size['height'])
                # debug(f"           left={left} top={top} right={right} bottom={bottom}")

                if bottom > browser_height or right > browser_width:
                    debug("            Image to crop is off the page")
                    continue

                screen_cropped = screen_whole.crop( (left, top, right, bottom) ) # get cropped subset of image
                debug(f"           Cropped")

                # Create a BytesIO object to store the image data as bytes
                image_buffer = BytesIO()

                # Save the cropped image to the BytesIO buffer
                screen_cropped.save(image_buffer, format='PNG')

                # Get the bytes value from the BytesIO buffer
                image_data = image_buffer.getvalue()

                database_save_handler(image_data, search, '(screen shot)', 'image/x-png')

                # Note to self:
                # To print first 40 characters of binary data for debugging, b64encode it:
                # {base64.b64encode(screen_cropped)[:40]}
                # {(base64.b64decode(screen_cropped.make_blob())[:40])}

    debug(f"quitting driver, went through {len(elements)} elements")
    # Close the browser instance
    try:
        driver.quit()
    except Exception as e: # (likely StaleElementReferenceException, but could be timeout or something else)
        debug("Got a stale element or other error processing WebElements from Selenium")
        return
    debug(f"quit driver")

    return

# The purpose of this function is to retrieve Search and Image objects from the database, calculate the number
# of searches and images, and render an HTML template with the information to be displayed in the browser.
def home_page(request):
    try:
        searches = Search.objects.all()
        images = Image.objects.all()
    except Exception as e: # (likely StaleElementReferenceException, but could be timeout or something else)
        debug(f"Error in past_searches() retrieving searches or images from database: {e}")
        return render(request, 'fail.html', {'error_message': f"Error retrieving searches/images from database: {e}"})
    return render(request, 'index.html', {'number_of_searches': len(searches), 'number_of_images': len(images)})

# The purpose of this function is to handle form submission, retrieve the URL entered by the user,
# perform web scraping operations using `BeautifulSoup`, and interact with the database to store the
# scraped data. It also calls other functions to handle HTTP requests, parse image tags, and perform
# additional web scraping using a web driver.
def scrape_web_page(request):
    debug(f"Starting scrape_web_page(request), request={request}")

    if request.method == 'POST':

        url_entered = request.POST['url'] # url_entered gets URL user entered in the index.html form
        # Note that we store url_entered in the search database just as the user entered it, even if
        # it's just "google.com", and we wind up storing images from "https://www.google.com"

        parsed_url = urlparse(url_entered) # Parse the URL entered by the user to get the scheme
        scheme = parsed_url.scheme if parsed_url.scheme else 'http' # Figure out scheme (e.g. http)
        url_with_scheme = f"{scheme}://{parsed_url.netloc}{parsed_url.path}" # Rebuild URL

        url, response, error = get_web_response_handler(request,url_with_scheme, url_entered) # Call handler function for getting web response
        debug(f"get_web_response_handler() returned url={url}, response={response}, error={error}")
        if error:
            return render(request, 'fail.html', {'error_message': error})

        # We can retrieve a web page, so save the search URL (as the user entered it) in the Searches database
        try:
            search = Search.objects.create(url=url_entered) # Save the search instance
            search.save()
        except Exception as e:
            debug(f"Failure inserting Search record: {e}")
            return render(request, 'fail.html', {'error_message': f"Unable to insert search in database.html: {e}"})

        soup = BeautifulSoup(response.content, 'html.parser')  # Parse HTML content with BeautifulSoup
        img_tags = soup.find_all('img') # Extract all the image URLs from the HTML content

        for img in img_tags:
            img_str = str(img) # img by itself is an object, and we may want to use its string representation

            debug(f"checking img_str={img_str}")

            # Some sites use data-gl-src, data-gl-srcset, data-getimg, data-hi-res-src, data-full-url, full-src, and
            # a variety of other non-standard alternatives to src and srcset in image tags. (Example: usatoday.com).
            # So if we don't find ' src' or ' srcset', we'll use these variants instead if they're present.

            # Some img tags have a single src= and a multi-image srcset=, so find the single image url, and
            # add it to a multi-image srcset url, so we pick an appropriately sized image from all hte candidates

            if ' src="' in img_str: # priority if it has a space before it, in case of multiple src attributes
                single_image_url = after_substr(img_str,' src="').split('"')[0]
            elif 'src="' in img_str:
                single_image_url = after_substr(img_str,'src="').split('"')[0]
            elif 'url="' in img_str:
                single_image_url = after_substr(img_str,'url="').split('"')[0]
            elif 'img="' in img_str:
                single_image_url = after_substr(img_str,'img="').split('"')[0]
            else:
                single_image_url = ''

            if ' srcset="' in img_str: # priority if it has a space before it, in case of multiple srcset attributes
                multi_image_url = after_substr(img_str,'srcset="').split('"')[0]
            elif 'srcset="' in img_str:
                multi_image_url = after_substr(img_str,'srcset="').split('"')[0]
            else:
                multi_image_url = ''

            if single_image_url != '':
                if multi_image_url == '':
                    multi_image_url = single_image_url + ' 1x'
                else:
                    multi_image_url = single_image_url + ' 1x,' + multi_image_url

            if multi_image_url == '':
                debug(f"Skipping img in img_tags loop (img={img})")
                continue  # Skip this image tag since it has no 'src' or 'srcset' attributes

            image_url = pick_an_image_from_srcset(multi_image_url,url)

            # Store the image at 'image_url' in Images table, with search data
            debug(f"In img_tags loop, about to store img_url = {image_url} from img tag={img}")
            store_image_from_url_in_database(search,image_url,url)

        debug(f"Done with img_tags loop and beautiful soup scraping, about to call scrape_page+with_webdriver")
        debug(f"_____________________________________________________________________________________________")
        debug(f"_____________________________________________________________________________________________")
        scrape_page_with_webdriver(search,url)

        return redirect('success', id=search.id)
    return render(request, 'scrape_web_page.html')

# The purpose of this function is to serve images to the client by retrieving the image object
# from the database based on the provided `image_id`.
# It then returns an HTTP response with the image data and appropriate content_type, allowing the
# client to display the image in the browser or use it in other applications that consume image data.
def myimage(request, image_id):
    image = Image.objects.get(pk=image_id)
    return HttpResponse(image.image, content_type="image/jpeg")

# The purpose of this function is to retrieve images from the database, process them by extracting
# filenames and generating data URIs, and render a template to display the images in a web page.
def show_all_images(request):
    images = Image.objects.all()
    images = add_template_data_to_image(images)
    return render(request, 'show_all_images.html', {'images': images}, )

# The purpose of this function is to retrieve past searches and images from the database,
# adjust the timestamp to the local timezone, and render a template to display the searches
# along with the number of images in a web page.
def past_searches(request):
    # Get all past searches, format local timestamp, and send to template
    try:
        searches = Search.objects.all()
        images = Image.objects.all()
    except Exception as e: # (likely StaleElementReferenceException, but could be timeout or something else)
        debug(f"Error in past_searches() retrieving searches or images from database: {e}")
        return render(request, 'fail.html', {'error_message': f"Error retrieving searches/images from database: {e}"})

    for search in searches:
        local_tz = timezone.get_current_timezone()
        local_dt = search.timestamp.astimezone(local_tz)
        search.timestampadjuster = local_dt.strftime('%Y-%m-%d %H:%M:%S')

    return render(request, 'past_searches.html', {'searches': searches, 'number_of_images': len(images)}) # Render list of searches to template

# Takes an Image record retrieved from database, and adds two fields to each
# image record to display in the web page template:
# * image.filename (extracted from the URL, minus the http & domain name.
# * image.image_data_uri the image itself in data-formatted base64 encoding that
#   can be displayed with an <img> tag in the template.
def add_template_data_to_image(images):
    max_filename_length = 60 # max filename we display

    for image in images:
        if image.url.startswith("data"):
            # if starts with 'data:' instead of 'http:' or 'https:', just use url as a pseudo-filename
            image.filename = image.url[:40]
        else:
            # sometimes the url contains ? and other extraneous data after the filename, so strip everything after ?
            filename = image.url.split('?')[0]
            # split remaining url by / and pick the last (-1) element, which is just the filename
            filename = filename.split('/')[-1]

            # Make a max-60-character filename to display as caption to template.
            # If it's say 100 characters ending in .jpeg, it will truncate the part before
            # and add (...) but keep the .jpeg. If it doesn't have a period in it, it just
            # truncates it 5 characters before the size limit and adds (...).
            if filename == '':
                image.filename = '(no filename)' # if there was no text between the last / in the url and the ?, just include the whole url
            elif filename.rfind('.') == -1 and len(filename) > max_filename_length: # if no period in filename & > max chars, truncate to 26 chars + '(...)'
                image.filename = filename[:max_filename_length-5] + '(...)'
            elif filename.rfind('.') > 0 and len(filename) > max_filename_length: # if period & prefix of filename is > max chars, truncate prefix + '(...)'
                image.filename = filename.split('.')[0][:max_filename_length-9] + '(...).' + after_substr(filename,'.')[:4]
            else:
                image.filename = filename

        image.image_data_uri = f"data:{image.content_type};base64,{base64.b64encode(image.image).decode('utf-8')}"

    return images

# Show a page for a given past search, including its URL & timestamp, and images stored
def past_search(request):
    try:
        # Check if an id parameter was sent (e.g. http://127.0.0.1:8000/past_searches?id=5)
        search_id_for_page = request.GET['id']
    except KeyError:
        # No id sent to this as an attribute to the URL
        return render(request, 'fail.html', {'error_message': "No search id sent to past_search.html"})

    try:
        search = Search.objects.get(id=search_id_for_page)
    except Search.DoesNotExist:
        return render(request, 'fail.html',{'error_message': f"Search ID {search_id_for_page} not found"})

    local_tz = timezone.get_current_timezone()
    local_dt = search.timestamp.astimezone(local_tz)
    search.timestamp_local = local_dt.strftime('%Y-%m-%d %H:%M:%S')

    images = Image.objects.filter(search_id=search_id_for_page)

    # Add filename and image_data_uri attribute for each image to send to success.html
    images = add_template_data_to_image(images)

    # Return past_search.html with data to render it (images, search.url, search_timestamp_formatted)
    debug(f"Returning past_search.html")
    return render(request, 'past_search.html', {'images': images, 'search_url': search.url, 'search_timestamp': search.timestamp_local})

# Show success.html after storing images for user's requested URL
def success(request, id):
    debug(f"in success(), id={id} ")

    # Retrieve the Search record with the id of the search we just performed
    try:
        search = Search.objects.get(id=id)
    except Search.DoesNotExist:
        return render(request, 'fail.html', {'error_message': "Unexpected problem retrieving images from search"},)

    # Format the timestamp from the search as the local timezone
    local_tz = timezone.get_current_timezone()
    local_dt = search.timestamp.astimezone(local_tz)
    search_timestamp_formatted = local_dt.strftime('%Y-%m-%d %H:%M:%S')

    # Retrieve all the Image records with the id of the search we just performed
    images = Image.objects.filter(search_id=id)

    # Add filename and image_data_uri attribute for each image to send to success.html
    images = add_template_data_to_image(images)

    # Return success.html with data to render it (images, search.url, search_timestamp_formatted)
    debug(f"Returning success.html")
    return render(request, 'success.html', {'images': images, 'search_url': search.url, 'search_timestamp': search_timestamp_formatted})


import tkinter as tk
from tkinter import ttk
import sql_command as sql
from screens.BackButton import BackButton

class ShowCustomerScreen(tk.Frame):
    def __init__(self, parent, *args, **kwargs):
        super().__init__(parent, *args, **kwargs)
        self.parent = parent

        self.back_container = tk.Frame(self)
        self.back_container.pack(fill="x")

        self.back_to_menu_button = BackButton(
            self.back_container, command=self.back_to_menu)

        self.back_to_menu_button.pack(side=tk.LEFT)

        self.table = ttk.Treeview(self, columns=(
            1, 2, 3, 4, 5, 6, 7), show="headings")
        self.table.column(1, anchor=tk.CENTER, width=100, stretch=True)
        self.table.column(2, anchor=tk.CENTER, width=100, stretch=True)
        self.table.column(3, anchor=tk.CENTER, width=100, stretch=True)
        self.table.column(6, anchor=tk.CENTER, width=10, stretch=True)
        self.table.column(7, anchor=tk.CENTER, width=60, stretch=False)
        self.table.heading(1, text="ID",)
        self.table.heading(2, text="CCCD")
        self.table.heading(3, text="Phone")
        self.table.heading(4, text="Email")
        self.table.heading(5, text="Username")
        self.table.heading(6, text="Vip Point")
        self.table.heading(7, text="Class")
        self.table.pack(expand=True, fill="both")

    def before_switch_handler(self):
        self.show_customer_table()

    def back_to_menu(self):
        self.parent.switch_to("MenuScreen")

    def show_customer_table(self):
        for row in self.table.get_children():
            self.table.delete(row)
        if (self.parent.pg_connection != None):
            with self.parent.pg_connection.cursor() as cur:
                cur.execute(sql.show_customer)
                rows = cur.fetchall()
                for row in rows:
                    self.table.insert('', 'end', values=row)


import sys
r = sys.stdin.readline

def Binom(n,k):
	C = [[0 for i in range(j)] for j in range(n+1)]
	print(C)
	for i in range(n+1):
		for j in range(min(i,k)):
			if (j==0)|(j==i): C[i][j] = 1
			else: C[i][j] = C[i-1][j-1] + C[i-1][j]
	return C[n][k]

while True:
	n, k = r().strip().split(" ")
	n = int(n)
	k = int(k)
	print(Binom(n,k))


from app.db import DBS
from app.settings.log import logger

async def get_number(idmonopolia: int):
    logger.info(f"SMS-SENDER: поиск номера для заказа {idmonopolia}")

    QUERY_GET_PHONE = (
        "SELECT phone "
        "FROM mircomf4_epool.ep_zakaz_mon4tk "
        f'WHERE idmonopolia="{idmonopolia}";'
    )
    mysql_read = DBS["mysql"]
    phone = (await mysql_read.fetch_one(QUERY_GET_PHONE))[0]
    logger.info(f"SMS-SENDER: найден номер {phone} для заказа {idmonopolia}")

    res = ""
    try:
        res = await _validate_phone(idmonopolia, phone)
        is_found = True
    except AssertionError:
        is_found = False

    if not is_found:
        logger.info(f"SMS-SENDER: пробуем взять номер из другой базы")
        QUERY_GET_PHONE = (
            "SELECT phone_delivery "
            "FROM mircomf4_epool.ep_zakaz "
            f'WHERE idmonopolia="{idmonopolia}";'
        )
        phone = (await mysql_read.fetch_one(QUERY_GET_PHONE))[0]
        res = await _validate_phone(idmonopolia, phone)

    logger.info(f"SMS-SENDER: взят номер из базы {res} для заказа {idmonopolia}")

    return res

async def _validate_phone(idmonopolia, phone):
    if len(phone) == 11 and phone[:2] == "89":
        res = phone

    elif len(phone) == 12 and phone[:3] == "+79":
        res = "89" + phone[3:]

    elif len(phone) == 10 and phone[0] == "9":
        res = "8" + phone

    else:
        logger.warning(f"SMS-SENDER: номер {phone} невалидный для заказа {idmonopolia}")
        raise AssertionError("Номер невалиден")

    logger.info(f"SMS-SENDER: номер прошел валидацию {phone} для заказа {idmonopolia}")
    return res


"""
Given an array of integers arr and an integer value k,
return the total amount of unique, contiguous, subarrays
that sum to k in arr.

Input: [3, 7, -4, -2, 1, 5] k = 3
Output: 2
Explanation: 2 subarrays sum up to 3

                      i j
[3, 7, -4, -2, 1, 5] [0,0]
[3, 7, -4, -2, 1, 5] [1,2]
"""

from typing import List

class Solution:
    def brute_force(self, arr: List[int], k: int) -> int:
        """
        Check every subarray
        Complexity
        ----
        Time : O(N^2)
        Space : O(N^2)
        """
        seen = set()
        for i in range(len(arr)):
            sub_array_sum = 0
            sub_array = []
            for j in range(i, len(arr)):
                sub_array_sum += arr[j]
                sub_array.append(arr[j])
                if sub_array_sum == k and tuple(sub_array) not in seen:
                    seen.add(tuple(sub_array))
        return len(seen)

    def cumulative_sum(self, arr: List[int], k: int) -> int:
        """
        We can use the difference of cumulative sums. Cumulative sums
        are often helpful when dealing with subarray sums

        Complexity
        ----
        Time : O(N^2)
        Space : O(N) not including output
        """
        # Declaring a sums array for storing the sum
        sums = []
        sums.append(arr[0])
        # Storing the cumulative sum in the sums array
        for idx in range(1, len(arr)):
            sums.append(sums[idx - 1] + arr[idx])

        count = 0
        # Now we can check sub array sums by simply finding the difference between cumulative sums
        for start in range(0, len(arr)):
            for end in range(start, len(arr)):
                temp_sum = sums[end]

                # this corresponds to the sub array ranging from start to end
                if start != 0:
                    temp_sum -= sums[start - 1]

                if temp_sum == k:
                    count += 1

        return count

    def empty_gaps(self, arr: List[int], k: int) -> int:
        """
        Complexity
        ----
        Time : O(N)
        Space : O(N)
        """
        m = {}  # how many times have we seen a sum in the past?
        s = 0
        count = 0
        m[0] = 1

        for idx in range(0, len(arr)):
            # keep track of the cumulative sum
            s += arr[idx]
            """
            If we have seen (s-k) before, that means the cumulative sum of the
            subarray between the current index and that index which yielded a
            cumulative sum of (s-k) will be k, which is our target
            """
            if (s - k) in m:
                count += m[s - k]
            """
            We need to keep track of how many times we have seen each cumulative
            sum because if we have seen it more then once then there is more then one
            sub array with the target sum that we need to count
            """
            if s in m:
                m[s] += 1
            # if we have not seen this sum add one occurence to the map
            else:
                m[s] = 1
        return count

print(Solution().brute_force([3, 7, -4, -2, 1, 5], 3))
print(Solution().cumulative_sum([3, 7, -4, -2, 1, 5], 3))
print(Solution().empty_gaps([3, 7, -4, -2, 1, 5], 3))


import PySimpleGUI as sg
import soundboardplay

#row/column declaration
ROWS,COL = soundboardplay.qtyRowCol()

#Button Layout for PysimpleGUI
layout = [[sg.Button('+', size=(13, 10), key=(i,j), pad=(0,0)) for j in range(COL)] for i in range(ROWS)]
window = sg.Window('Soundboard.tx', layout)

#creates dictionary with list of files and positions from zip object
dicOfFiles = dict(soundboardplay.assignFiles(COL))

while True:
    # The Event Loop
    event, values = window.read()
    #print(event, values)
    if event == sg.WIN_CLOSED or event == 'Exit':
        break
    if event in dicOfFiles.keys():
        soundboardplay.playSound(dicOfFiles[event])
        #print(str(soundboardplay.lengthOfSound(dicOfFiles[event])))
    else:
        print('no file assigned')

window.close()


import copy
#git push 1
#recursive brute-force approach

#Project Architecture:

'''
board setup with an array of strings

board =[ "6..874.1.",
         "..9.36...",
         "...19.8..",
         "7946.....",
         "..1.894..",
         "...41..69",
         ".7..5..9.",
         ".539.76..",
         "9.2.61.47"   ]

dots representing open squares

Strategy:

proc Solve:
    fill in all the "obvious cells until we run out"
    if the puzzle is solved, return true
    if there is a contradiction, return false

    #Landing here means we start guessing
    Find the first empty cell
    possibilities < get all possibilities at empty cell

    for each possibility in possibilities:
        fill in the possibility at the cell
        recursively run Solve()
        if it returns, True, return True
        if it returns Fasce, undo all changes made
    return false

'''

board =[
        "6..874.1.",
         "..9.36...",
         "...19.8..",
         "7946.....",
         "..1.894..",
         "...41..69",
         ".7..5..9.",
         ".539.76..",
         "9.2.61.47"   ]

def main():
    global board #convert every line of text into a list since python strings are immutable

    for idx,line in enumerate(board):
    #gives the index of the current line as well as contents
        board[idx] = list(line)
        #Converts every character into an actual array item

    solve()
    printBoard()

def solve():
    global board

    try:
        fillAllObvious()
    except:
        return False
    if isComplete():
        return True

    i,j = 0,0
    for rowIdx,row in enumerate(board):
        for colIdx,col in enumerate(row):
            if col == ".":
                i,j = rowIdx, colIdx

    possibilites = getPossibilities(i,j)
    for value in possibilites:
        snapshot = copy.deepcopy(board)

        board[i][j] = value
        result = solve()
        if result == True:
            return True
        else:
            board = copy.deepcopy(snapshot)

    return False

def fillAllObvious():
    global board             #nested for loop gives different values of i and j we will use this to check each cell

    while True:
        somethingChanged = False
        for i in range(0,9):
            for j in range(0,9):
                possibilites = getPossibilities(i,j) #func checks possibilites in cell with coordinates then return the items back
                if possibilites == False:
                    continue
                if len(possibilites) == 0:
                    raise RuntimeError("NO moves left")
                if len(possibilites) == 1:
                    board[i][j] = possibilites[0]
                    somethingChanged = True

        if somethingChanged == False:
            return

def getPossibilities(i,j):
    global board
    if board[i][j] != ".":
        return False

                                                      #using {} to create a set
    possibilites = {str(n) for n in range(1,10)}  #using conveinient python comprehension

    for val in board[i]:
        possibilites -= set(val)  #rows

    for idx in range(0,9):
        possibilites -= set( board[idx][j] ) #columns

    iStart = (i // 3) * 3
    jStart = (j // 3) * 3

    subboard = board[iStart:iStart+3]
    for idx,row in enumerate(subboard):
        subboard[idx] = row[jStart:jStart+3]

    for row in subboard:
        for col in row:
            possibilites -= set(col)

    return list(possibilites)

#Clean print function
def printBoard():
    global board
    for row in board:
        for col in row:
            print(col, end = "")
        print("")

def isComplete():
    for row in board:
        for col in row:
           if (col == "."):
                return False

    return True

main()
        #print(list(line))
        #So printing this returns

'''
    ['6', '.', '.', '8', '7', '4', '.', '1', '.']
    ['.', '.', '9', '.', '3', '6', '.', '.', '.']
    ['.', '.', '.', '1', '9', '.', '8', '.', '.']
    ['7', '9', '4', '6', '.', '.', '.', '.', '.']
    ['.', '.', '1', '.', '8', '9', '4', '.', '.']
    ['.', '.', '.', '4', '1', '.', '.', '6', '9']
    ['.', '7', '.', '.', '5', '.', '.', '9', '.']
    ['.', '5', '3', '9', '.', '7', '6', '.', '.']
    ['9', '.', '2', '.', '6', '1', '.', '4', '7']

    We now have lists, allowing us to modify contents
    Main Idea: Convert List of Strings into a Lists of Lists
'''


from collections import deque
class Solution:
    def racecar(self, target: int) -> int:
        queue = deque([(0, 1, 0)])
        visited = set()

        while queue:
            node, speed, dist = queue.popleft()
            if node == target:
                return dist

            visited.add((node, speed))
            if (node + speed, speed * 2) not in visited:
                queue.append((node + speed, speed * 2, dist + 1))
            if (node + speed > target and speed > 0) or (node + speed < target and speed < 0):
                if speed > 0:
                    iter_speed = -1
                else:
                    iter_speed = 1
                if (node, iter_speed) not in visited:
                    queue.append((node, iter_speed, dist + 1))

        return -1


from flask import Blueprint, render_template
from flask import Flask, url_for, json, jsonify
from flask_login import login_required, current_user
import os
import datetime
import time

views = Blueprint('views', __name__)

@views.route('/')
# To see home page only when logged in.
@login_required
def home():
    return render_template('home.html', user=current_user)

@views.route('/_updatetruckdata', methods=['GET'])
def updatetruckdata():
    truck_data = json.load(open('website/static/data/truck_data.json'))
    trucks_today = []
    trucks_15 = []
    trucks_30 = []
    now = datetime.datetime.now()
    week = ["S", "M", "T", "W", "T", "F", "S"]

    y = json.dumps(truck_data)
    dict = json.loads(y)
    i = 0
    for truck in truck_data:
        truck_day = dict[i].get("Days")
        arrival = dict[i].get("Arrival")
        departure = dict[i].get("Departure")
        day_index = now.strftime("%w")
        curr_day = week[int(day_index)]

        curr_hour = int(now.strftime("%H"))
        curr_min = int(now.strftime("%M"))
        curr_sec = int(now.strftime("%S"))
        # current time is expressed as the number of seconds that have passed since 12am
        curr_in_sec = curr_hour*3600 + curr_min*60 + curr_sec

        if arrival[1] == ":":
            arrival_hour = int(arrival[0])
            arrival_min = int(arrival[2:4])
        else:
            arrival_hour = int(arrival[0:2])
            arrival_min = int(arrival[3:5])

        if departure[1] == ":":
            dep_hour = int(departure[0])
            dep_min = int(departure[2:4])
        else:
            dep_hour = int(departure[0:2])
            dep_min = int(departure[3:5])

        arrival_sec = int(arrival[-2:])
        # truck arrival and departure time is expressed as the number of seconds that have passed since 12am
        arrival_in_sec = arrival_hour*3600 + arrival_min*60 + arrival_sec

        dep_sec = int(departure[-2:])
        dep_in_sec = dep_hour*3600 + dep_min*60 + dep_sec

        current = False
        if day_index == "1" or day_index == "3" or day_index == "5":
            if curr_day in truck_day:
                current = True
        elif day_index == "2":
            if truck_day[1] == "T":
                current = True
        elif day_index == "4":
            if truck_day[3] == "T":
                current = True
        elif day_index == "6":
            if truck_day[5] == "S":
                current = True
        elif day_index == "0":
            if truck_day[6] == "S":
                current = True

        to_arrive = False
        to_depart = False
        if current == True:
            # check if a truck's arrival time is within 2 hours (expressed in seconds) of the current time
            # of if the departure time hasn't passed yet
            if (curr_in_sec <= arrival_in_sec) and (arrival_in_sec <= curr_in_sec + 7200):
                to_arrive = True
            if (curr_in_sec >= arrival_in_sec) and (curr_in_sec < dep_in_sec):
                to_depart = True
            if (to_arrive == True) or (to_depart == True):
                trucks_today.append(dict[i])
                # checking if the truck arrives within 30 min
                if (curr_in_sec <= arrival_in_sec) and (arrival_in_sec <= curr_in_sec + 1800):
                    trucks_30.append(True)
                else:
                    trucks_30.append(False)
                # checking if the truck departs within 15 min
                if (curr_in_sec <= dep_in_sec) and (dep_in_sec <= curr_in_sec + 900):
                    trucks_15.append(True)
                else:
                    trucks_15.append(False)

        i += 1
    return jsonify(today = trucks_today, fifteen = trucks_15, thirty = trucks_30)


from tkinter import Frame, Scale, Label, IntVar

d_frame_cnf = dict( bg='#4D4D4D', padx=5, pady=5, relief='raised', bd=1)
d_label_name_cnf = dict( font='Calibri 18 bold', fg='white', relief='flat')
d_label_value_cnf = dict( font='Calibri 16 bold', fg='white', relief='flat')
d_scale_cnf = dict( bd=1, troughcolor='#787878', highlightthickness=0,activebackground='#707070', showvalue=False, length=40)

def ri(v): return int(round(v))

class KeyCode:
    LEFT = 37
    UP = 38
    RIGHT = 39
    DOWN = 40

class FrameSlider(Frame):

    class Orientation:
        VERTICAL = 'v'
        HORIZONTAL = 'h'

    def __init__(self, master, _id, name='', range_=(0,10), value=0, orientation=Orientation.HORIZONTAL, wide=30, **kw):
        super().__init__(master=master, cnf=d_frame_cnf, **kw)

        self._id = _id
        self._orientation = orientation
        self.var = self.__create_var(range_=range_, value=value)
        self.__create_name_label(name=name)
        self.__create_value_label(range_=range_)
        self._slider = self.__create_slider(range_=range_, wide=wide)
        self.__set_bindings()

        self.callback = None

    def __create_var(self, range_, value):
        var = IntVar(value=max(range_[0],value))
        self.__trace_tag = var.trace_add(mode='write', callback=self.__on_change_int_var)
        return var

    def __create_name_label(self, name):
        label = Label(master=self, text=name, bg=self['background'], cnf=d_label_name_cnf)
        d_pack = dict( side='bottom', pady=(5,0)) if self._orientation == self.Orientation.VERTICAL else dict(side='left', padx=(0,5))
        label.pack(**d_pack)

    def __create_value_label(self, range_):
        width = len(str(max(range_)))
        label = Label(master=self, textvariable=self.var, bg=self['background'], width=width, cnf=d_label_value_cnf)
        d_pack = dict( side='top', pady=(0,5)) if self._orientation == self.Orientation.VERTICAL else dict(side='right', padx=(5,0))
        label.pack(**d_pack)

    def __create_slider(self, range_, wide):
        from_, to = range_ if (self._orientation ==  self.Orientation.HORIZONTAL) else range_[::-1]
        scale = Scale(master=self, from_=from_, to=to, orient=self._orientation, width=wide, variable=self.var, cnf=d_scale_cnf)
        scale.config(bg=self['background'], )
        side, fill = ('top','y') if self._orientation == self.Orientation.VERTICAL else ('right','x')
        scale.pack(side=side, fill=fill, expand=True)
        return scale

    def __set_bindings(self):
        self._slider.bind('<Button-1>', self.__on_click_left)

    # [ M E T H O D S ]

    def set_callback(self,on_change):
        self.callback = on_change

    def set_value(self,value):
        self.var.trace_remove(mode='write', cbname=self.__trace_tag)
        self.var.set(value=value)
        self.__trace_tag = self.var.trace_add(mode='write', callback=self.__on_change_int_var)

    # [ C A L L B A C K S ]

    def __on_change_int_var(self,a,b,c):
        if self.callback:
            self.callback(_id=self._id, value=self.var.get())

    def __on_click_left(self, event):
        self._slider.focus_force()


import numpy as np
import matplotlib.pyplot as plt
import scipy.stats as stats
import seaborn as sns

# x= np.linspace(-1.6, -1.3, 250)

plt.rcParams["figure.figsize"] = [4.50, 3.50]
plt.rcParams["figure.autolayout"] = True

for chain in range(5):

    sam_all = np.load('robertson_dreamzs_5chain_sampled_params_chain_%d_10000.npy' % chain)
    sam_all = sam_all[int(len(sam_all)/2):]
    if chain == 0:
        sam_0 = np.array([x[0] for x in sam_all])
        sam_1 = np.array([x[1] for x in sam_all])
        sam_2 = np.array([x[2] for x in sam_all])
    else:
        sam_0 = np.append(sam_0, np.array([x[0] for x in sam_all]))
        sam_1 = np.append(sam_1, np.array([x[1] for x in sam_all]))
        sam_2 = np.append(sam_2, np.array([x[2] for x in sam_all]))

print(len(sam_0))
print(len(sam_1))
print(len(sam_2))

# scale_factor = 1

plt.figure('sam0')
# n, bins, patches = plt.hist(sam_0, bins= 60, density = True, color='blue', label='histogram_0')
# gaussian_kde_z0 = stats.gaussian_kde(sam_0)
# plt.plot(bins, gaussian_kde_z0(bins) * scale_factor, color= 'springgreen', linewidth= 3, label='kde')
sns.distplot(sam_0, color='springgreen')

plt.figure('sam1')
# n, bins, patches = plt.hist(sam_1, bins= 60, density = True, color='pink', label='histogram_1')
# gaussian_kde_z1 = stats.gaussian_kde(sam_1)
# plt.plot(bins, gaussian_kde_z1(bins) * scale_factor, color= 'yellow', linewidth= 5, label='kde')
sns.distplot(sam_1, color='yellow')

plt.figure('sam2')
# n, bins, patches = plt.hist(sam_2, bins= 60, density = True, color='black', label='histogram_2')
# gaussian_kde_z2 = stats.gaussian_kde(sam_2)
# plt.plot(bins, gaussian_kde_z2(bins) * scale_factor, color= 'magenta', linewidth= 7, label='kde')
sns.distplot(sam_2, color='magenta')

plt.legend(loc=0)
plt.title('Density vs log of Parameter Value')
plt.xlabel('log of parameter value')
plt.ylabel('density')
plt.show()

# sam_0 = np.array([x[0] for x in sam_all if x[0] >= -1.6 and x[0] <= -1.3])
# sam_1 = np.array([x[1] for x in sam_all if x[1] >= 0 and x[1] <= 15])
# sam_2 = np.array([x[2] for x in sam_all if x[2] >= 1 and x[2] <= 7])
#binwidth = (-1.3 - -1.6) / 230
# scale_factor = 1 #len(sam) * binwidth

# R_sqr= -1.6 * np.log(np.array(sam)
# vu = -1.3 * np.pi * np.array(sam)
# s1 = np.sqrt(R_sqr) * np.cos(vu)
# s2 = np.sqrt(R_sqr) * np.sin(vu)
#
# fig = plt.figure(figsize = (12, 4))
# for ind_subplot, zi,  col in zip ((1, 2), (s1, s2), ('crimson', 'dodgerblue')):
#     lalo = fig.add_subplot(1, 2, ind_subplot)
#     plt.hist(zi, bins=40, range=(-1.6, -1.3), color='blue', label='histogram')
#
#     bindwidth =  4 / 250
#     scale_factor = len(zi) * bindwidth
#
#     gaussian_kde_zi = stats.gaussian_kde(s1)
#     plt.plot(x, gaussian_kde_zi(x) * scale_factor, color= 'springgreen', linewidth= 3, label='kde')
#
# plt.show()
    # std_zi = np.std(zi)
    # mean_zi = np.mean(zi))
    # plt.plot(x, stats.norm.pdf((x-mean_zi/std_zi)*scale_factor, color= 'black'))

# Finding the maximun and minimum number of values in the data
# max(sam)
# min(sam)
# print(max(sam))
# print(min(sam))

    # plt.xlim(xmin = -1.6, xmax = -1.3)
    # plt.legend(prop = {'size': 16}, title = 'Map')
    # plt.title('population')
    # plt.set_xlabel('percentage')
    # plt.set_ylabel('number')
    #bindwidth = ((0.8351350471426552-(-3.1179278912226955) / 250))

# print(len(sam))
# print("square root", mt.sqrt)
#print(value_counts(sam))

#logarithm
# logps = np.load('robertson_dreamzs_5chain_logps_chain_0_10000.npy')
# logps = [x[0] for x in logps]
# plt.plot(logps)
# plt.xlabel('iteration')
# plt.ylabel('log likelihood')
# plt.show()

#logarithm
# logps = np.load('robertson_dreamzs_5chain_logps_chain_1_10000.npy')
# logps = [x[0] for x in logps]
# plt.plot(logps)
# plt.xlabel('iteration')
# plt.ylabel('log likelihood')
# plt.show()

#sampled_params_chain_0_10000

# density = gaussian_kde(sam)
# plt.hist(sam)
# plt.plot(density(sam), bins=300,)

# plt.style.use('ggplot')

# sam = np.load('robertson_dreamzs_5chain_sampled_params_chain_0_10000.npy')
# sam = [x[0] for x in sam]
# plt.legend(prop={'size': 16}, title = 'Map')
# plt.title('population')
# plt.xlabel('percentage')
# plt.ylabel('number')
# plt.style.use('ggplot')
# gaussian_kde = gaussian_kde(sam)
# plt.hist(sam, bins=210)
# plt.plot(sam, gaussian_kde, bins=300, color='springgreen', linewidth=3, label='kde' )
# plt.show()

#TESTING
# Rsqure = -2 * np.log(sam)
# sam1= np.sqrt(Rsqure)
# bindwidth = 8 / 40
# scale_factor = len(sam1) * bindwidth
#n, x, _ = plt.hist(sam, bins=450, density=True, color='g', edgecolor='k')
#plt.plot(x,gaussian_kde(sam)(x))
# xvalues = np.linspace(-1.6, -1.2,1000)
#plt.xlim(xmin=-1.6, xmax = -1.2)

#sampled_params_chain_0_10000
# sam = np.load('robertson_dreamzs_5chain_sampled_params_chain_1_10000.npy')
# sam = [x[0] for x in sam]
# print(sam)
# plt.xlabel('percentage')
# plt.ylabel('number')
# plt.title('population')
# plt.style.use('ggplot')
# plt.hist(sam, bins=200, color='r', edgecolor='g')
# plt.xlim(xmin=-2, xmax = -1)
# plt.show()


import os
from typing import *

import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
import torch
from IPython.display import Image, display, clear_output
from sklearn.manifold import TSNE
from torch import Tensor
from torch.distributions import Normal
from torchvision.utils import make_grid

def plot_2d_latents(ax, qz, z, y):
    z = z.to('cpu')
    y = y.to('cpu')
    scale_factor = 2
    batch_size = z.shape[0]
    palette = sns.color_palette()
    colors = [palette[l] for l in y]

    # plot prior
    prior = plt.Circle((0, 0), scale_factor, color='gray', fill=True, alpha=0.1)
    ax.add_artist(prior)

    # plot data points
    mus, sigmas = qz.mu.to('cpu'), qz.sigma.to('cpu')
    mus = [mus[i].numpy().tolist() for i in range(batch_size)]
    sigmas = [sigmas[i].numpy().tolist() for i in range(batch_size)]

    posteriors = [
        plt.matplotlib.patches.Ellipse(mus[i], *(scale_factor * s for s in sigmas[i]), color=colors[i], fill=False,
                                       alpha=0.3) for i in range(batch_size)]
    for p in posteriors:
        ax.add_artist(p)

    ax.scatter(z[:, 0], z[:, 1], color=colors)

    ax.set_xlim([-3, 3])
    ax.set_ylim([-3, 3])
    ax.set_aspect('equal', 'box')

def plot_latents(ax, z, y):
    z = z.to('cpu')
    palette = sns.color_palette()
    colors = [palette[l] for l in y]
    z = TSNE(n_components=2).fit_transform(z)
    ax.scatter(z[:, 0], z[:, 1], color=colors)

def make_vae_plots(vae, x, y, outputs, training_data, CASP12_data,TS115_data,CB513_data, figsize=(18, 6)):
    fig, axes = plt.subplots(1, 3, figsize=figsize, squeeze=False)

    # plot the observation
    #axes[0, 0].set_title(r'Observation $\mathbf{x}$')
    #plot_samples(axes[0, 0], x)

    # plot posterior samples
    #axes[0, 2].set_title(
    #    r'Reconstruction $\mathbf{x} \sim p_\theta(\mathbf{x} | \mathbf{z}), \mathbf{z} \sim q_\phi(\mathbf{z} | \mathbf{x})$')
    #px = outputs['px']
    #x_sample = px.sample().to('cpu')
    #plot_samples(axes[0, 2], x_sample)

    # plot ELBO
    ax = axes[0, 0]
    ax.set_title(r'ELBO: $\mathcal{L} ( \mathbf{x} )$')
    ax.plot(training_data['elbo'], label='Training')
    ax.plot(CASP12_data['elbo'], label='CASP12')
    ax.plot(TS115_data['elbo'], label='TS115')
    ax.plot(CB513_data['elbo'], label='CB513')
    ax.legend()

    # plot KL
    ax = axes[0, 1]
    ax.set_title(r'$\mathcal{D}_{\operatorname{KL}}\left(q_\phi(\mathbf{z}|\mathbf{x})\ |\ p(\mathbf{z})\right)$')
    ax.plot(training_data['kl'], label='Training')
    ax.plot(CASP12_data['kl'], label='CASP12')
    ax.plot(TS115_data['kl'], label='TS115')
    ax.plot(CB513_data['kl'], label='CB513')
    ax.legend()

    # plot NLL
    ax = axes[0, 2]
    ax.set_title(r'$\log p_\theta(\mathbf{x} | \mathbf{z})$')
    ax.plot(training_data['log_px'], label='Training')
    ax.plot(CASP12_data['log_px'], label='CASP12')
    ax.plot(TS115_data['log_px'], label='TS115')
    ax.plot(CB513_data['log_px'], label='CB513')
    ax.legend()

    # plot prior samples
    #axes[2, 0].set_title(r'Samples $\mathbf{x} \sim p_\theta(\mathbf{x} | \mathbf{z}), \mathbf{z} \sim p(\mathbf{z})$')
    #px = vae.sample_from_prior(batch_size=x.size(0))['px']
    #x_samples = px.sample()
    #plot_samples(axes[2, 0], x_samples)

    # plot interpolations samples
    #axes[2, 1].set_title(
    #    r'Latent Interpolations: $\mathbf{x} \sim p_\theta(\mathbf{x} | t \cdot \mathbf{z}_1 + (1-t) \cdot \mathbf{z}_2), \mathbf{z}_1, \mathbf{z}_2 \sim p(\mathbf{z}), t=0 \dots 1$')
    #plot_interpolations(axes[2, 1], vae)

    # plot samples (sampling from a grid instead of the prior)
    #if vae.latent_features == 2:
    #    axes[2, 2].set_title(
    #        r'Samples: $\mathbf{x} \sim p_\theta(\mathbf{x} | \mathbf{z}), \mathbf{z} \sim \operatorname{grid}(-3:3, -3:3)$')
    #    px = vae.sample_from_prior(batch_size=x.size(0))['px']
    #    x_samples = px.sample()
    #    plot_grid(axes[2, 2], vae)

    # display
    plt.tight_layout()
    #plt.close(fig)
    #clear_output(wait=True)

    plt.show()

    #


import logging
import os
import platform
import sys
from datetime import datetime

import prettytable
from airtest.core.api import exists
from airtest.core.cv import Template

from airrun.config import DefaultConfig

logger = logging.getLogger(__name__)

def show_info_as_table(keys: list, values: list) -> None:
    # keys list; values list(dict) or dict
    table = prettytable.PrettyTable()
    if isinstance(values, list):
        table.field_names = keys
        for v in values:
            if isinstance(v, list):
                row = v
            elif isinstance(v, dict):
                row = v.values()
            table.add_row(row)
    elif isinstance(values, dict):
        table.field_names = ['key', 'value']
        for v in keys:
            row = [v, values.get(v)]
            table.add_row(row)
    logger.info('\n{}'.format(table))

def bytes_to_utf8_gbk(data):
    try:
        result = data.decode('utf-8')
        return result
    except Exception:
        result = data.decode('gbk')
        return result

def deal_with_python_version(data):
    if str(sys.version_info.major) == '3':
        if isinstance(data, list):
            result = [bytes_to_utf8_gbk(d) for d in data]
        else:
            result = bytes_to_utf8_gbk(data)
        return result
    else:
        return data

def time_now_format() -> str:
    date_time_now = datetime.now().strftime('%Y%m%d-%H.%M.%S')
    return date_time_now

def system():
    return platform.system()

def make_log_path(device_name: str = "device") -> str:
    time_now = time_now_format()
    log_path = f"{DefaultConfig.LOG_PATH_ROOT}/{device_name}/{time_now}"
    if not os.path.exists(log_path):
        os.makedirs(log_path)
    return log_path

def make_report_path(log_path: str = "") -> str:
    report_path = f"{log_path}_report"
    if not os.path.exists(report_path):
        os.makedirs(report_path)
    return report_path

def exists_check(template):
    if isinstance(template, Template):
        return exists(template)
    else:
        return template.exists()


"""https://www.codewars.com/kata/path-finder-number-1-can-you-reach-the-exit/train/python"""

"""v1 - First try, working pretty good: explores 500x500 maze in ~ 5 seconds

- coordinates are represented as (x, y), use in reverse order for indexing
- paths are tuples of tuples, signifying that we're not modifying any existing paths
- lambdas take 1 pair of coordinates and return their neighbor
- process:
  - we start outside the "entrance" at (0, -1), from where the only possible NSEW step is (0, 0)
    (this is so that on a 1x1 maze, (0, 0) can be "found" as an exit)
  - for each path, we explore possible extensions, and
    - return True if extension is end of maze
    - skip if it's outside board, has negative index, or already visited
    - if empty, add to visited and save as new path
    - if wall, add to visited (this is done passively, not even mentioned in the code)
  - all previous paths are discarded, replaced with list of new
  ↳ repeat until exit is found or all options are depleted
"""
def path_finder(maze):
    next_north = lambda last: (last[0],   last[1]-1)
    next_east  = lambda last: (last[0]+1, last[1])
    next_south = lambda last: (last[0],   last[1]+1)
    next_west  = lambda last: (last[0]-1, last[1])

    maze = maze.split('\n')
    paths = [((0, -1), )]
    visited = set()
    while paths:
        new_paths = []
        for path in paths:
            for extension in (next_north, next_east, next_south, next_west):
                ext = extension(path[-1])
                if all(n >= 0 for n in ext) and ext not in visited:
                    try:
                        if ext == (len(maze[0])-1, len(maze)-1): return True
                        if maze[ext[1]][ext[0]] == '.': new_paths.append(path + (ext, ))
                        visited.add(ext)
                    except IndexError: continue
        paths = new_paths
    return False

"""Top CW solution

- much faster than mine
- seems to use variable names that come from algorithmic terminology
- I have no idea how it works, even though it shares some traits with mine
- I'm sure I'll understand once I go deeper into algorithms
"""
# def path_finder(maze):
#     g = maze.splitlines()
#     end, bag = len(g[0]) -1 + len(g) * 1j - 1j, {0}
#     grid = {x + y * 1j for y,l in enumerate(g) for x,c in enumerate(l) if '.' == c}
#     while bag:
#         if end in bag: return True
#         grid -= bag
#         bag = grid & set.union(*({z + 1j ** k for k in range(4)} for z in bag))
#     return False


import random, sys

print('ROCK, PAPER, SCISSORS')

# These variables keep track of the number of wins, losses, and ties.
wins = 0
losses = 0
ties = 0

while True: # This is the main game loop.
    print()
    print('%s Wins, %s Losses, %s Ties' % (wins, losses, ties))
    print()
    while True: #This is the player input loop.
        print('Enter R for rock, P for paper, or S for scissors.')
        print('To quit, enter Q.')
        print()

        player_move = input()
        print()
        if player_move == 'Q':
            sys.exit() # This terminates the program.
        if player_move == 'R' or player_move == 'P' or player_move == 'S':
            break # This breaks the player input loop.

    # This displays what the player chose.
    if player_move == 'R':
        print('You throw out rock!')
    elif player_move == 'P':
        print('You throw out paper!')
    elif player_move == 'S':
        print('You throw out paper!')

    # This displays what the computer chose.
    random_number = random.randint(1,3)
    if random_number == 1:
        computer_move = 'R'
        print('Computer used rock!')
    if random_number == 2:
        computer_move = 'P'
        print('Computer used paper!')
    if random_number == 3:
        computer_move = 'S'
        print('Computer used scissors!')

    print()

    # This displays and records the score.
    if player_move == computer_move:
        print('You tied!')
        ties += 1
    elif player_move == 'R' and computer_move == 'S':
        print('You win!')
        wins += 1
    elif player_move == 'P' and computer_move == 'R':
        print('You win!')
        wins += 1
    elif player_move == 'S' and computer_move == 'P':
        print('You win!')
        wins += 1
    elif player_move == 'R' and computer_move == 'P':
        print('You lose!')
        losses += 1
    elif player_move == 'P' and computer_move == 'S':
        print('You lose!')
        losses += 1
    elif player_move == 'S' and computer_move == 'R':
        print('You lose!')
        losses += 1


#
# Author: Yudong Lin
#
# extract potential useful features from gufi
#

import pandas as pd

def extract_and_add_gufi_features(_df: pd.DataFrame) -> pd.DataFrame:
    from pandarallel import pandarallel  # type: ignore

    pandarallel.initialize(verbose=1)

    def _split_gufi(x: pd.DataFrame) -> pd.Series:
        from datetime import datetime

        information: list = x["gufi"].split(".")
        gufi_flight_number: str = information[0]
        gufi_flight_major_carrier: str = gufi_flight_number[:3]
        gufi_flight_destination_airport: str = information[2]
        gufi_flight_date: datetime = datetime.strptime(
            "_".join((information[3], information[4], information[5][:2])), "%y%m%d_%H%M_%S"
        )
        # gufi_flight_FAA_system: str = information[6]
        gufi_timestamp_until_etd = int((gufi_flight_date - x["timestamp"]).seconds / 60)
        return pd.Series(
            [
                # gufi_flight_number,
                gufi_flight_major_carrier,
                gufi_flight_destination_airport,
                gufi_timestamp_until_etd,
                # gufi_flight_date,
                # gufi_flight_FAA_system,
            ]
        )

    _df[
        [
            # "gufi_flight_number",
            "airline",
            "gufi_flight_destination_airport",
            "gufi_timestamp_until_etd",
            # "gufi_flight_date",
            # "gufi_flight_FAA_system",
        ]
    ] = _df.parallel_apply(lambda x: _split_gufi(x), axis=1)

    return _df


from fastapi import APIRouter, Body, Path, status

from app.usecases import pokemon as pokemon_ucase
from common.type import PokemonNumberStr

from .mapper import PokemonRequestMapper, PokemonResponseMapper
from .schema import CreatePokemonRequest, PokemonResponse, UpdatePokemonRequest

router = APIRouter()

@router.post('/pokemons', status_code=status.HTTP_201_CREATED)
async def create_pokemon(body: CreatePokemonRequest) -> PokemonResponse:
    create_pokemon_data = PokemonRequestMapper.create_request_to_entity(body)
    created_pokemon = await pokemon_ucase.create_pokemon(create_pokemon_data)

    return PokemonResponseMapper.entity_to_response(created_pokemon)

@router.get('/pokemons/{no}')
async def get_pokemon(no: str = Path(..., description=PokemonNumberStr.__doc__)) -> PokemonResponse:
    no = PokemonNumberStr(no)
    pokemon = await pokemon_ucase.get_pokemon(no)

    return PokemonResponseMapper.entity_to_response(pokemon)

@router.get('/pokemons')
async def get_pokemons() -> list[PokemonResponse]:
    pokemons = await pokemon_ucase.get_pokemons()

    return list(map(PokemonResponseMapper.entity_to_response, pokemons))

@router.patch('/pokemons/{no}')
async def update_pokemon(
    no: str = Path(..., description=PokemonNumberStr.__doc__),
    body: UpdatePokemonRequest = Body(...),
) -> PokemonResponse:
    no = PokemonNumberStr(no)
    update_pokemon_data = PokemonRequestMapper.update_request_to_entity(body)
    update_pokemon_data.validate_no_not_in_evolutions(no)
    updated_pokemon = await pokemon_ucase.update_pokemon(no, update_pokemon_data)

    return PokemonResponseMapper.entity_to_response(updated_pokemon)

@router.delete('/pokemons/{no}', status_code=status.HTTP_204_NO_CONTENT)
async def delete_pokemon(no: str = Path(..., description=PokemonNumberStr.__doc__)):
    no = PokemonNumberStr(no)
    await pokemon_ucase.delete_pokemon(no)


import pandas as pd
import gradio as gr
from sklearn.linear_model import LinearRegression
from sklearn.metrics import r2_score
from sklearn.preprocessing import OneHotEncoder
from sklearn.compose import make_column_transformer
from sklearn.pipeline import make_pipeline
from sklearn.model_selection import train_test_split

car = pd.read_csv("Cleaned Car.csv")

X = car.drop(columns='Price')
y = car['Price']

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

ohe = OneHotEncoder()
ohe.fit(X[['name', 'company', 'fuel_type']])
var = ohe.categories_
column_trans = make_column_transformer((OneHotEncoder(categories=ohe.categories_), ['name', 'company', 'fuel_type']),
                                       remainder='passthrough')

lr = LinearRegression()
pipe = make_pipeline(column_trans, lr)
pipe.fit(X_train, y_train)

y_pred = pipe.predict(X_test)

r2_score(y_test, y_pred)

def predict_car_price(Car_Model_Name, Car_Company_Name, Car_Purchase_Year, Nos_of_KM_driven, Fuel_Type):
    a = Car_Model_Name
    b= Car_Company_Name
    c = Car_Purchase_Year
    d = Nos_of_KM_driven
    e = Fuel_Type
    pipe = make_pipeline(column_trans, lr)
    y_pred = pipe.predict(
        pd.DataFrame([[a, b, c, d, e]], columns=['name', 'company', 'year', 'kms_driven', 'fuel_type']))

    return y_pred

app = gr.Interface(
    fn=predict_car_price,
    inputs=['text', 'text', 'number', 'number', 'text'], outputs=['number'],
    title="Car Price Predictor",
    description="This Model will predict the price of a old Car.")

app.launch()


#! /usr/bin/python

class LinkNode(object):

    def __init__(self, val, next = None):
        self.val = val
        self.next = next

class Solution(object):

    def merge_k_sorted_link_list(self, lists) -> LinkNode:
        # merge sort
        return self.merge(lists, 0, len(lists)-1)

    def merge(self, lists, i, j) -> LinkNode:
        if i == j:
            return lists[i]
        mid = (i+j) // 2
        left = self.merge(lists, i, mid)
        right = self.merge(lists, mid+1, j)
        return self.merge_sorted_link_list(left, right)

    def merge_sorted_link_list(self, l1: LinkNode, l2: LinkNode):
        if not l1:
            return l2
        if not l2:
            return l1
        if l1.val < l2.val:
            l1.next = self.merge_sorted_link_list(l1.next, l2)
            return l1
        else:
            l2.next = self.merge_sorted_link_list(l1, l2.next)
            return l2

def print_link_list(head: LinkNode):
    print(head.val, end="")
    while head.next:
        print("->", end="")
        head = head.next
        print(head.val, end="")
    print()

if __name__ == "__main__":
    n3 = LinkNode(3, None)
    n2 = LinkNode(2, n3)
    n1 = LinkNode(1, n2)
    print_link_list(n1)

    n6 = LinkNode(5, None)
    n5 = LinkNode(3, n6)
    n4 = LinkNode(2, n5)
    print_link_list(n4)

    n9 = LinkNode(7, None)
    n8 = LinkNode(5, n9)
    n7 = LinkNode(3, n8)
    print_link_list(n7)

    lists = [n1, n4, n7]
    head = Solution().merge_k_sorted_link_list(lists=lists)
    print_link_list(head)


from server.models.statusCodes import HttpStatusCodes
from server.responseHelpers import get_serializable_block
from typing import List, Tuple
from core.blockchainConstants import Block
from server.models.endpoints import BlockchainEndpoints
from server.models.syncer import Syncer
import requests

class BlockSync(Syncer):
    def __init__(self, peer_nodes: List[str], path=BlockchainEndpoints.BROADCAST_BLOCK) -> None:
        super().__init__(peer_nodes, path)

    def broadcast(self, block: Block) -> Tuple[bool, bool]:
        marked_for_conflict = False
        for node in self.peer_nodes:
            url = self.get_url(node)

            try:
                response = requests.post(
                    url, json={'block': get_serializable_block(block)})
                if response.status_code == HttpStatusCodes.CONFLICT:
                    marked_for_conflict = True
                if response.status_code >= HttpStatusCodes.BAD_REQUEST and response.status_code < 600:
                    return (False, marked_for_conflict)
            except requests.exceptions.ConnectionError:
                continue

        return (True, marked_for_conflict)


import jellyfish as jf
from unidecode import unidecode

from Relatorio import Relatorio
from IdentificarParametros import IdentificarParametros
#from app import app #ao importar app a classe para de funcionar

class Interpretador:

    def __init__(self, desejoUsuario):
        self.desejoUsuario = unidecode(desejoUsuario.upper())

    def __str__(self):
        return "Objeto '{0}' da classe Interpretador".format(self.desejoUsuario)

    def calcular_grau_de_similaridade_frases(self, nomeRelatorio=""):
        """calcula o grau de simililaridade entre duas frases, o desejo do
        usuario e nome do relatorio disponivel"""
        resultadoComparacao = float(jf.levenshtein_distance(
            self.desejoUsuario,
            nomeRelatorio))
        return resultadoComparacao

    def procurar_relatorio_satisfatorio(self):
        """Retorno o comando sql que o usuário deseja"""
        relatorio = Relatorio().getListaComandoRelatorio()

        #setando indices para auxiliar na busca
        listaRelatorio = []
        listaComandos = []
        listaLevenshtein = []
        #item[0] = nome do relatorio
        #item[1] = select para gerar o relatorio
        for item in relatorio.items():
            listaRelatorio.append(item[0])
            listaComandos.append(item[1])
            listaLevenshtein.append(self.calcular_grau_de_similaridade_frases(item[0]))

        #procurar relatorio mais similiar
        value, indice = min((val, idx) for (idx, val) in enumerate(listaLevenshtein))
        #app.logger.debug("Pontuação: {0}, Desejo: {1}, Encontrado: {2}, Comando: {3}".format(value, self.desejoUsuario, listaRelatorio[indice], listaComandos[indice]))

        return listaComandos[indice]

    def indetificar_parametos_ano(self):
        desejUsr = self.desejoUsuario
        idPar = IdentificarParametros(desejUsr).indetificarParametosAno()
        return idPar

#novaInter = Interpretador(desejoUsuario="Relátorio de vendas por valor em 20")
#print(novaInter.desejoUsuario)
#print(novaInter.procurar_relatorio_satisfatorio())
#print(novaInter.indetificar_parametos_ano())


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
PubMed Article Search and Download Python Script (Ver 0.1)
"""

# import modules
import sys
import os
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import argparse
from tqdm import tqdm
import requests
import time
import urllib.request
from urllib.parse import quote
from selenium.webdriver.common.action_chains import ActionChains
from PIL import Image
import textwrap
import logging
import pytextrank
import spacy
# modules for pdf generation
from reportlab.lib.styles import ParagraphStyle
from reportlab.lib.styles import getSampleStyleSheet
from reportlab.pdfgen import canvas
from reportlab.lib.pagesizes import letter, A4
from reportlab.pdfbase.ttfonts import TTFont
from reportlab.pdfbase import pdfmetrics
from reportlab.lib import colors
from reportlab.lib.styles import getSampleStyleSheet
from reportlab.platypus import Paragraph
from reportlab.platypus import SimpleDocTemplate
#

class pmExt:
    def __init__(self):
        pass

    def ext_article(self, keywords, itemN):
        # make directory
        self.create_dir(root_dir, keywords)
        title_list = []
        doi_list = []
        abstract_list = []
        abstract_summary_list = []
        abstract_keyword_list = []
        citation_list = []
        figure_list = []
        # scrolling google browser
        try:
            # browser extraction part ####
            chrome_opt = webdriver.ChromeOptions()
            chrome_opt.add_argument('--disable-gpu')
            pathx = "/Users/uksu/Downloads/chromedriver"
            browser = webdriver.Chrome(executable_path=pathx,options=chrome_opt)
            ActionChains(browser).key_down(Keys.CONTROL).click().key_up(Keys.CONTROL).perform()
            url = 'https://pubmed.ncbi.nlm.nih.gov/?term=' + quote(keywords.encode('utf-8')) + '&size=20'
            browser.get(url)
            time.sleep(1)
            element = browser.find_element_by_tag_name("body")

            # Searching environment setting
            elementx = browser.find_element_by_xpath('//*[@id="static-filters-form"]/div/div[1]/div[1]/ul/li[1]/label')
            browser.execute_script("arguments[0].click();", elementx)
            time.sleep(1)
            elementx = browser.find_element_by_xpath('//*[@id="static-filters-form"]/div/div[1]/div[1]/ul/li[2]/label')
            browser.execute_script("arguments[0].click();", elementx)
            time.sleep(1)
            elementx = browser.find_element_by_xpath('//*[@id="static-filters-form"]/div/div[1]/div[1]/ul/li[3]/label')
            browser.execute_script("arguments[0].click();", elementx)
            time.sleep(1)

            # click first article
            elementx = browser.find_element_by_xpath('//*[@id="search-results"]/section/div[1]/div/article[1]/div[2]/div[1]/a')
            browser.execute_script("arguments[0].click();", elementx)
            first_article = browser.find_element_by_xpath('//*[@id="adjacent-navigation"]/div[2]/a/span[2]')
            browser.execute_script("arguments[0].click();", first_article)
            time.sleep(1)
            first_article = browser.find_element_by_xpath('//*[@id="adjacent-navigation"]/div[2]/a/span[2]')
            browser.execute_script("arguments[0].click();", first_article)
            time.sleep(1)
            ## extract first article
            # title_list
            t_path = '//*[@id="full-view-heading"]/h1'
            title_link = browser.find_element_by_xpath(t_path)
            title_list_tmp = title_link.text
            title_list.append(title_list_tmp)

            # citation list
            try:
                c_path = '//*[@id="citedby"]/h2/em[1]'
                citation_link = browser.find_element_by_xpath(c_path)
                citation_list_tmp = citation_link.text
                citation_list.append(citation_list_tmp)
            except:
                print("!! No citation of this paper !!")
                citation_list_tmp = '0'
                citation_list.append(citation_list_tmp)

            # doi_list
            try:
                d_path = '//*[@id="full-view-identifiers"]/li[2]/span/a'
                doi_link = browser.find_element_by_xpath(d_path)
                doi_list_tmp = doi_link.get_attribute('href')
                doi_list.append(doi_list_tmp)
            except:
                doi_list_tmp = "No DOI!!"
                doi_list.append(doi_list_tmp)

            ## abstract_list
            a_path = '//*[@id="enc-abstract"]/p'
            abstract_link = browser.find_element_by_xpath(a_path)
            abstract_list_tmp = abstract_link.text
            abstract_list.append(abstract_list_tmp)

            # abstract_summary
            # load a spaCy model and set the environment
            nlp = spacy.load("en_core_web_sm")
            # logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
            # logger = logging.getLogger("PyTR")
            tr = pytextrank.TextRank(logger=None)
            nlp.add_pipe(tr.PipelineComponent, name="textrank", last=True)
            doc = nlp(abstract_list_tmp)
            # temporary abstract_summary_list
            abstract_summary_list_tmp = [];
            for ab_summary in doc._.textrank.summary(limit_phrases=1, limit_sentences=2):
                abstract_summary_list_tmp.append(ab_summary)
            abstract_summary_list.append(str(abstract_summary_list_tmp))
            # temporary abstract_summary_list
            abstract_keyword_list_tmp = [];
            for phrase in doc._.phrases[:5]: ## top 5 keywords
                abstract_keyword_list_tmp.append(phrase)
            abstract_keyword_list.append(str(abstract_keyword_list_tmp))

            # figure_list
            j = 1
            while j < 10:
                try:
                    f_path = '//*[@id="slides-container"]/figure['+ str(j) + ']/a'
                    figure_link = browser.find_element_by_xpath(f_path)
                    figure_list_tmp = figure_link.get_attribute('href')
                    figure_list.append(figure_list_tmp)
                except:
                    print("!! No more figures in this paper !!")
                    figure_list.append('no_fig')
                    # figure_list.remove('no_fig')
                j += 1
            time.sleep(1)
            # go to next article
            next_article = browser.find_element_by_xpath('//*[@id="adjacent-navigation"]/div[2]/a/span[2]')
            browser.execute_script("arguments[0].click();", next_article)
            time.sleep(1)
        except Exception as e:
            print(e)
        ## main run
        i = 0
        while i < itemN - 1:
            # title_list
            t_path = '//*[@id="full-view-heading"]/h1'
            title_link = browser.find_element_by_xpath(t_path)
            title_list_tmp = title_link.text
            title_list.append(title_list_tmp)

            # citation list
            try:
                c_path = '//*[@id="citedby"]/h2/em[1]'
                citation_link = browser.find_element_by_xpath(c_path)
                citation_list_tmp = citation_link.text
                citation_list.append(citation_list_tmp)
            except:
                print("!! No citation of this paper !!")
                citation_list_tmp = '0'
                citation_list.append(citation_list_tmp)

            # doi_list
            try:
                d_path = '//*[@id="full-view-identifiers"]/li[2]/span/a'
                doi_link = browser.find_element_by_xpath(d_path)
                doi_list_tmp = doi_link.get_attribute('href')
                doi_list.append(doi_list_tmp)
            except:
                doi_list_tmp = "No DOI!!"
                doi_list.append(doi_list_tmp)

            # abstract_list
            a_path = '//*[@id="enc-abstract"]/p'
            abstract_link = browser.find_element_by_xpath(a_path)
            abstract_list_tmp = abstract_link.text
            abstract_list.append(abstract_list_tmp)

            # load a spaCy model and set the environment
            nlp = spacy.load("en_core_web_sm")
            # logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
            # logger = logging.getLogger("PyTR")
            tr = pytextrank.TextRank(logger=None)
            nlp.add_pipe(tr.PipelineComponent, name="textrank", last=True)
            doc = nlp(abstract_list_tmp)
            # temporary abstract_summary_list
            abstract_summary_list_tmp = [];
            for ab_summary in doc._.textrank.summary(limit_phrases=1, limit_sentences=2):
                abstract_summary_list_tmp.append(ab_summary)
            abstract_summary_list.append(str(abstract_summary_list_tmp))
            # temporary abstract_summary_list
            abstract_keyword_list_tmp = [];
            for phrase in doc._.phrases[:5]: ## top 5 keywords
                abstract_keyword_list_tmp.append(phrase)
            abstract_keyword_list.append(str(abstract_keyword_list_tmp))

            # figure_list
            j = 1
            while j < 10:
                try:
                    f_path = '//*[@id="slides-container"]/figure['+ str(j) + ']/a'
                    figure_link = browser.find_element_by_xpath(f_path)
                    figure_list_tmp = figure_link.get_attribute('href')
                    figure_list.append(figure_list_tmp)
                except:
                    print("!! No more figures in this paper !!")
                    figure_list.append('no_fig')
                    # figure_list.remove('no_fig')

                j += 1
            # go to next article
            try:
                next_article = browser.find_element_by_xpath('//*[@id="adjacent-navigation"]/div[3]/a/span[2]')
                browser.execute_script("arguments[0].click();", next_article)
                time.sleep(0.5)
                i += 1
            except:
                print("### End of the searching pages ###")
                break
        browser.quit()
        return title_list, citation_list, doi_list, abstract_list, abstract_summary_list, abstract_keyword_list, figure_list

    def create_dir(self, root_dir, name):
        try:
            if not os.path.exists(root_dir):
                os.makedirs(root_dir)
                time.sleep(0.2)
                path = (name)
                sub_directory = os.path.join(root_dir, path)
                if not os.path.exists(sub_directory):
                    os.makedirs(sub_directory)
            else:
                path = (name)
                sub_directory = os.path.join(root_dir, path)
                if not os.path.exists(sub_directory):
                    os.makedirs(sub_directory)

        except OSError as e:
            if e.errno != 17:
                raise
            pass
        return

# HELP Section
parser = argparse.ArgumentParser(description='## Search and Extract Papers using PubMed Engine ##', formatter_class=argparse.RawDescriptionHelpFormatter,
                                 epilog='''\
version history:
    [Ver 0.21]       bug fixed for duplicated articles over limit
    [ver 0.20]       added an abstract summary as an output using TextRank algorithm
    [ver 0.10]       release of this script (2020.08.01)

++ Copyright at uschoi@nict.go.jp / qtwing@naver.com ++
''')
# parser.add_argument("Keyword", help="Keywords without space",)
# parser.add_argument("Item_number", help="Searching item number")
# parser.add_argument("URL_output.txt", help="URL text files")
parser.add_argument('--version', action='version', version='Version 0.1')
parser.parse_args()

# assign input arguments
print("+++++++++++++++++++++++++++++++++++++++++++++")
keyword = input('## Enter the keywords:  ')
print("+++++++++++++++++++++++++++++++++++++++++++++")
itemN = input('## How many articles do you want ?:  ')
print("+++++++++++++++++++++++++++++++++++++++++++++")

# root directiory
root_dir = "PubMed_Reports/"

# MAIN RUN
response = pmExt
title_list, citation_list, doi_list, abstract_list, abstract_summary_list, abstract_keyword_list, figure_list = response().ext_article(keyword, int(itemN))

# set path
path = root_dir + keyword

# link downloads
print("")
print(" +++++++++++++++++++++++++++++++++++++ NOW Processing IS STARTING +++++++++++++++++++++++++++++++++++++ ")
print("")
print(" [ Now downloading article information !! ] ")
# with open(os.path.join('./' + root_dir + '/', keyword + '.txt'), 'w', encoding="utf-8") as f:
#     pbar = enumerate(tqdm(title_list))
#     for item_ind, item in pbar:
#         f.write("%s\n" % "      ")
#         f.write("[ Article" + "-" + '%04d' %(item_ind + 1) + " ]: " + "%s\n" % item)
#         f.write("Citation" + ": " + "%s\n" % textwrap.fill(citation_list[item_ind], width=150))
#         f.write("doi" + ": " + "%s\n" % textwrap.fill(doi_list[item_ind], width=150))
#         f.write("%s\n" % textwrap.fill(abstract_list[item_ind], width=150))
#         f.write("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++")
#         f.write("%s\n" % "      ")
#         time.sleep(0.01)

# set the template for pdf output
report = SimpleDocTemplate(os.path.join('./' + root_dir + '/' + "pmExt_ArticleReport-" + keyword + '.pdf'), pagesize=A4)
report_style = getSampleStyleSheet()
# report_style.list()
report_style.add(ParagraphStyle(name='Paragraph', spaceAfter=10))
report_style.add(ParagraphStyle(name='content_title',
                          fontFamily='Helvetica',
                          fontSize=12,
                          leading=15,
                          textColor=colors.HexColor("#2E2D30")))
report_style.add(ParagraphStyle(name='content_citation',
                          fontFamily='Helvetica',
                          fontSize=10,
                          textColor=colors.HexColor("#D65720")))
report_style.add(ParagraphStyle(name='content_doi',
                          fontFamily='Times-Roman',
                          fontSize=11,
                          leading=15,
                          textColor=colors.HexColor("#285DC9")))
report_style.add(ParagraphStyle(name='content_line',
                          fontFamily='Times-Roman',
                          fontSize=11,
                          leading=15,
                          textColor=colors.HexColor("#050000")))
report_style.add(ParagraphStyle(name='content_summary',
                          fontFamily='Times-Roman',
                          fontSize=11,
                          leading=15,
                          textColor=colors.HexColor("#1A6304")))
report_style.add(ParagraphStyle(name='content_keyword',
                          fontFamily='Times-Roman',
                          fontSize=11,
                          leading=15,
                          textColor=colors.HexColor("#A10613")))
# title
report_title = Paragraph("pmExt Reports <ver 0.22>", report_style['Heading1'])

# main run
pbar = enumerate(tqdm(title_list))
contents = []
contents.append(report_title)

# run loop
for item_ind, item in pbar:
    # article number
    paragraph_1 = Paragraph(
        ("[ Article" + "-" + '%04d' %(item_ind + 1) + " ]"),
        report_style['Heading2']
    )
    # title
    paragraph_2= Paragraph(
        item,
        report_style['content_title']
    )
    # citation number
    paragraph_3 = Paragraph(
        ("Citation:  " + "%s\n" % citation_list[item_ind]),
        report_style['content_citation']
    )
    # doi link
    paragraph_4 = Paragraph(
        ("DOI:  " + "%s\n" % doi_list[item_ind]),
        report_style['content_doi']
    )
    # abstract
    paragraph_5 = Paragraph(
        abstract_list[item_ind],
        report_style['BodyText']
    )
    contents.append(paragraph_1)
    contents.append(paragraph_2)
    contents.append(paragraph_3)
    contents.append(paragraph_4)
    contents.append(paragraph_5)
report.build(contents)
print(" ")

print(" [ Now downloading abstract summary !! ] ")
# with open(os.path.join('./' + root_dir + '/', keyword + '_abstract_summary' + '.txt'), 'w', encoding="utf-8") as f:
#     pbar = enumerate(tqdm(title_list))
#     for item_ind, item in pbar:
#         f.write("%s\n" % "      ")
#         f.write("[ Article" + "-" + '%04d' %(item_ind + 1) + " ]: " + "%s\n" % item)
#         f.write("Citation" + ": " + "%s\n" % textwrap.fill(citation_list[item_ind], width=150))
#         f.write("doi" + ": " + "%s\n" % textwrap.fill(doi_list[item_ind], width=150))
#         f.write("%s\n" % "      ")
#         f.write("%s\n" % textwrap.fill(abstract_summary_list[item_ind], width=150))
#         f.write("%s\n" % "      ")
#         f.write('# Keywords  ' + "%s\n" % textwrap.fill(abstract_keyword_list[item_ind]))
#         f.write("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++")
#         f.write("%s\n" % "      ")
#         time.sleep(0.01)
#
# set the template for pdf output
summary = SimpleDocTemplate(os.path.join('./' + root_dir + '/' + "pmExt_AbstractSummary-" + keyword + '.pdf'), pagesize=A4)
# title
summary_title = Paragraph("pmExt Reports (Abstract Summary)", report_style['Heading1'])

# main run
pbar = enumerate(tqdm(title_list))
contents_sum = []
contents_sum.append(summary_title)

# run loop
for item_ind, item in pbar:
    # article number
    paragraph_1 = Paragraph(
        ("[ Article" + "-" + '%04d' %(item_ind + 1) + " ]"),
        report_style['Heading2']
    )
    # title
    paragraph_2= Paragraph(
        item,
        report_style['content_title']
    )
    # citation number
    paragraph_3 = Paragraph(
        ("Citation:  " + "%s\n" % citation_list[item_ind]),
        report_style['content_citation']
    )
    # doi link
    paragraph_4 = Paragraph(
        ("DOI:  " + "%s\n" % doi_list[item_ind]),
        report_style['content_doi']
    )
    # abstract summary
    paragraph_5 = Paragraph(
        "+++++++++++++++++++++++++++ Auto-Summary ++++++++++++++++++++++++++++",
        report_style['content_line']
    )
    # abstract summary
    paragraph_6 = Paragraph(
        abstract_summary_list[item_ind],
        report_style['content_summary']
    )
    # abstract summary
    paragraph_7 = Paragraph(
        "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++",
        report_style['content_line']
    )
    # abstract keywords
    paragraph_8 = Paragraph(
        abstract_keyword_list[item_ind],
        report_style['content_keyword']
    )
    contents_sum.append(paragraph_1)
    contents_sum.append(paragraph_2)
    contents_sum.append(paragraph_3)
    contents_sum.append(paragraph_4)
    contents_sum.append(paragraph_5)
    contents_sum.append(paragraph_6)
    contents_sum.append(paragraph_7)
    contents_sum.append(paragraph_8)
summary.build(contents_sum)

# save figures
print(" ")
print(" [ Now downloading article figures !! ] ")
pbar2 = enumerate(tqdm(figure_list))
indN = len(title_list) + 1

n_tmp = 0
ind2_tmp = 1
fig_mark = 0
nofig_mark = 0
for item_ind2, item2 in pbar2:
    if 'no_fig' not in item2:
        filename = "Article" + "_" + '%04d' % (n_tmp + 1) + "_Fig" + '%02d' % (ind2_tmp) + ".jpg"
        urllib.request.urlretrieve(item2, os.path.join(path, filename))
        fig_mark += 1
    else:
        nofig_mark += 1
        ind2_tmp = 0
    mark_diff = item_ind2 - (nofig_mark+fig_mark)
    ind2_tmp = ind2_tmp + 1
    n_tmp = round((item_ind2 - ind2_tmp + 1 - mark_diff)/9)

print("")
print(" +++++++++++++++++++++++++++++++++++++ NOW EXTRACTION FINISHED ++++++++++++++++++++++++++++++++++++++++ ")
print("")

#