import numpy as np
import BBMM
import unittest

np.random.seed(0)
lengthscale = 1.0 + np.random.random()
variance = 1.0 + np.random.random()

n = 3
d = 2
N = 100

def y(X):
    Y = np.sum(np.sin(X[:, 0:d]), axis=1)
    Y_grad = [np.sum(X[:, d * (i + 1):d * (i + 2)] * np.cos(X[:, d * (i + 1):d * (i + 2)]), axis=1) for i in range(n)]
    return np.concatenate([Y] + Y_grad)[:, None]

np.random.seed(0)
X = np.random.random((N, (n + 1) * d))
Y = y(X)

ref = np.array([4.07525802e+01, 1.25747855e+01, 8.88520198e-03])
ref_factor = np.array([6.06428980e+01, 1.32487578e+01, 8.63056515e-01, 8.36132526e-03])
ref_order = np.array([5.01755444e+00, 2.73303199e+00, 1.00000000e-10, 3.33174327e-02])
ref_factor_order = np.array([5.02491805e+00, 2.73296982e+00, 8.65480936e-01, 1.00000000e-10, 1.88639965e-02])

class Test(unittest.TestCase):
    def _run(self, optfactor, split_type, ref):
        stationary_kernel = BBMM.kern.RBF()
        stationary_kernel.set_lengthscale(lengthscale)
        stationary_kernel.set_variance(variance)
        kernel = BBMM.kern.FullDerivative(stationary_kernel, n, d, optfactor=optfactor, likelihood_split_type = split_type)
        noise = [1e-4] * len(kernel.splits)
        gp = BBMM.GP(X, Y, kernel, noise)
        gp.optimize(messages=False)
        err = (gp.params - ref)/ref
        print(ref)
        print(gp.params)
        self.assertTrue(np.max(np.abs(err)) < 1e-5)

    def test(self):
        self._run(False, 'same', ref)

    def test_factor(self):
        self._run(True, 'same', ref_factor)

    def test_order(self):
        self._run(False, 'order', ref_order)

    def test_factor_order(self):
        self._run(True, 'order', ref_factor_order)

if __name__ == '__main__':
    unittest.main()


import requests
import re
from bs4 import BeautifulSoup as soup
import json
from time import sleep
from subprocess import run
import argparse

def get_links(pages,keys):
	links=[]
	for page in range(pages):
		url = 'https://images-api.nasa.gov/search?page={p}&media_type=image&q={k}'.format(p=page,k=keys)
		r= requests.get(url).json()

		try:
			items = r['collection']['items']
			for item in items:
				links.append(item['links'][0]['href'])
		except:
			try:
				sleep(5)
				continue
			except:
				print(page)
				break
	return links

def download_img(links,keys):
	for link in links:
		cmd='googleimagesdownload --keywords {k}  -x {l}'.format(k=keys,l=link.replace('~thumb','~orig'))
		run(cmd,shell=False)

if __name__ == '__main__':
	parse = argparse.ArgumentParser()
	parse.add_argument("-k",'--keywords',type=str,required=True,help='keywords')
	#parse.add_argument("-i", "--image_directory", type=str, required=False,help="specify a directory inside of the output directorydefault=keywords")
	#parse.add_argument("-o", "--output", type=str, required=False,help="output directory(default=downloads)")
	parse.add_argument("-p",'--pages',type=int, required=True,help="num of pages")

	args = parse.parse_args()
	keys = args.keywords
	pages = args.pages
	#output = args.output
	#folder_name =args.image_directory

	links = get_links(pages,keys)
	download_img(links,keys)


from sqlalchemy import create_engine
import pandas as pd
import psycopg2

def getEngine():
	try:
		connection = psycopg2.connect(dbname='eicu', user='', password='')
		# sorry for this, we need to set schema this way
		pd.read_sql_query("""
		    set search_path to eicu;
		    SELECT COUNT(*) FROM patient;
		    """, con=connection)
		return connection
	except:
		print("Unable to connect to the database")
		return None

def getConnection():
	try:
		connection = getEngine()
		return connection
	except:
		print("Unable to connect to the database")
		return None


"""
Define Report Class to handle html reports
"""

# imports

# classes

class Report:
    """
    A class to create, update, modify and save reports with pretty stuff
    """
    def __init__(self, path=None, string=""):
        """
        path is the place where the report is written/updated
        """
        self.path = path
        self.string = string

    def init(self, subject, date_and_time, version,
             cmd):
        """
        Init the report with html headers and various information on the report
        """
        intro_part1 = """<!DOCTYPE html>

CVRmap: individual report

Summary

BIDS participant label: sub-001

Date and time: 2022-06-01 13:54:41.301278

CVRmap version: dev

Command line options:

            """
        intro_part2 = """

        """
        self.string.join(intro_part1 + str(cmd) + '\n' + intro_part2)
        with open(self.path, "w") as f:
            f.write('<!DOCTYPE html>\n')
            f.write('\n')
            f.write('\n')
            f.write('
Connectomix: individual report
\n')
            f.write('
Summary
\n')
            f.write('
\n')
            f.write('
\n')
            f.write(
                '
BIDS participant label: sub-%s
\n' % subject)
            f.write('
Date and time: %s
\n' % date_and_time)
            f.write('
Connectomix version: %s
\n' % version)
            f.write('
Command line options:\n')
            f.write('
\n')
            f.write('\n')
            f.write(str(cmd) + '\n')
            f.write('\n')
            f.write('
\n')
            f.write('
\n')
            f.write('
\n')
            f.write('
\n')

    def add_section(self, title):
        """
        Add a section to the report
        """
        with open(self.path, "a") as f:
            f.write('
%s
\n' % title)

    def add_subsection(self, title):
        """
        Add a subsection to the report
        """
        with open(self.path, "a") as f:
            f.write('
%s
\n' % title)

    def add_sentence(self, sentence):
        """
        Add a sentence to the report
        """
        with open(self.path, "a") as f:
            f.write('%s
\n' % sentence)

    def add_image(self, fig):
        """
        Add an image path to the report
        if fig is a str, then it is interpreted as a path to the image to save.
        If a plotly figure, uses the .to_html argument to embded the image
        Args:
            fig: str or Figure object from plotly
        """
        import base64
        # with open(self.path, "ab") as f:
        #   f.write(b'')
        #   f.write(b'
')

        with open(self.path, "a") as f:
            if type(fig) is str:
                import os.path as path
                # make the path relative to report location
                _report_dir = path.dirname(self.path)
                _fig = path.relpath(fig, _report_dir)
                f.write('

')
            else:
                f.write(fig.to_html(full_html=False))
            f.write('
')

    def finish(self):
        """
        Writes the last lines of the report to finish it.
        """
        with open(self.path, "a") as f:
            f.write('\n')
            f.write('\n')

# functions

def build_report(outputs, entities, args, version):
    from datetime import datetime

    # init
    report = Report(
        outputs['report'])
    report.init(subject=entities['subject'],
                date_and_time=datetime.now(),
                version=version, cmd=args.__dict__)

    # create figures for report

    report.add_subsection(title='Matrix')
    report.add_sentence(
        sentence="Connectivity matrix for denoising strategy: %s" % entities['denoising'])
    report.add_image(outputs['matrix'])

    # finish the report
    report.finish()


import os
import pickle

import jieba
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import LatentDirichletAllocation
from bs4 import BeautifulSoup

# from htds.dataset.service.sdk import *
from preprocess import *

def lda_train(corpus,
              max_features=5000,
              n_topics=300):
    """
    Compute the topic distribution of the given docs with LDA model.
    :param corpus: the docs after pre_processing, like ['我 喜欢 苹果'， '美丽 古老的 国度']
    :param tfidf: TfidfVectorizer instance
    :return: The list of the topic distributions of the docs
    """
    tfidf = TfidfVectorizer(max_features)
    tfidf_matrix = tfidf.fit_transform(corpus)
    with open('./data/model/tfidf.pickle', 'wb') as handle:
        pickle.dump(tfidf, handle, protocol=pickle.HIGHEST_PROTOCOL)
    print("tfidf dumped!")
    print('word frequency matrix: ', tfidf_matrix.toarray())
    # n_components
    lda = LatentDirichletAllocation(n_components=n_topics, random_state=123456)
    res = lda.fit_transform(tfidf_matrix)
    print("LDA training finished!")
    with open('./data/model/lda.pickle', 'wb') as handle:
        pickle.dump(lda, handle, protocol=pickle.HIGHEST_PROTOCOL)
    print("LDA model dumped successfully!")
    print("Topic distribution on training dataset", res)
    return res

def lda_train_1(corpus, n_topics=2):
    """
    Compute the topic distribution of the given docs with LDA model.
    :param corpus: the docs after pre_processing, like ['我 喜欢 苹果'， '美丽 古老的 国度']
    :param tfidf: TfidfVectorizer instance
    :return: The list of the topic distributions of the docs
    """
    tfidf = TfidfVectorizer()
    tfidf_matrix = tfidf.fit_transform(corpus)
    print("tfidf dumped!")
    print('word frequency matrix: ', tfidf_matrix.toarray())
    # n_components
    lda = LatentDirichletAllocation(n_components=n_topics, random_state=123456)
    res = lda.fit_transform(tfidf_matrix)
    print("Topic distribution on training dataset", res)
    return tfidf_matrix, lda

if __name__ == '__main__':
    corpus = pre_process('data')
    res, lda = lda_train_1(corpus)


# -*- coding: utf-8 -*-

import itertools
import glob
import os

# Format expected by setup.py and doc/source/conf.py: string of form "X.Y.Z"
_version_major = 1
_version_minor = 6
_version_micro = 0
_version_extra = ''

# Construct full version string from these.
_ver = [_version_major, _version_minor]
if _version_micro:
    _ver.append(_version_micro)
if _version_extra:
    _ver.append(_version_extra)

__version__ = '.'.join(map(str, _ver))

CLASSIFIERS = ["Development Status :: 3 - Alpha",
               "Environment :: Console",
               "Intended Audience :: Science/Research",
               "License :: OSI Approved :: MIT License",
               "Operating System :: OS Independent",
               "Programming Language :: Python",
               "Topic :: Scientific/Engineering"]

# Description should be a one-liner:
description = "Scilpy: diffusion MRI tools and utilities"
# Long description will go up on the pypi page
long_description = """
Scilpy
========
Scilpy is a small library mainly containing small tools and utilities
to quickly work with diffusion MRI. Most of the tools are based
on or wrapper of the Dipy_ library.

.. _Dipy: http://dipy.org

License
=======
``scilpy`` is licensed under the terms of the MIT license. See the file
"LICENSE" for information on the history of this software, terms & conditions
for usage, and a DISCLAIMER OF ALL WARRANTIES.

All trademarks referenced herein are property of their respective holders.

Copyright (c) 2012--, Sherbrooke Connectivity Imaging Lab [SCIL],
Université de Sherbrooke.
"""

NAME = "scilpy"
MAINTAINER = "Arnaud Boré"
MAINTAINER_EMAIL = "arnaud.bore@gmail.com"
DESCRIPTION = description
LONG_DESCRIPTION = long_description
URL = "https://github.com/scilus/scilpy"
DOWNLOAD_URL = ""
LICENSE = "MIT"
AUTHOR = "The SCIL developers"
AUTHOR_EMAIL = ""
PLATFORMS = "OS Independent"
MAJOR = _version_major
MINOR = _version_minor
MICRO = _version_micro
VERSION = __version__
LEGACY_SCRIPTS = filter(lambda s: not os.path.basename(s) == "__init__.py",
                        glob.glob("scripts/legacy/*.py"))
SCRIPTS = filter(lambda s: not os.path.basename(s) == "__init__.py",
                 glob.glob("scripts/*.py"))

PREVIOUS_MAINTAINERS = ["Jean-Christophe Houde"]


import re

from .by import By
from hyperiontf.typing import (
    IncorrectLocatorException,
    HyperionUIException,
    NoSuchElementException,
)
from hyperiontf.typing import LocatorStrategies

import time
from typing import Any, Sized, cast, Dict, Union

from hyperiontf.configuration import config
from hyperiontf.logging import getLogger

logger = getLogger("Element")

class LocatableElement:
    """
    Represents a locatable element within a web page or application. This class is responsible for locating and providing
    access to a specific element or a collection of elements based on the provided locator strategy.

    The `LocatableElement` class follows the single responsibility principle, separating the responsibility of locating
    elements from other aspects of the class.

    Attributes:
        parent (object): The parent element or container that contains this element.
        locator (By | dict): The locator strategy used to identify the element. It can be a `By` object or a dictionary
                             of locators categorized by platform, operating system, and viewport.
        name (str): A descriptive name for the element used for logging and identification purposes.

    Public Properties:
        platform (str): The platform of the element, determined by the document holder's platform.
        viewport (str): The viewport of the element, determined by the document holder's viewport.
        os (str): The operating system of the element, determined by the root element's operating system.
        root (object): The root element or container of the current element.
        document_holder (object): The document holder that contains this element.

    Methods:
        find_itself(retries: Optional[int] = config.element.search_attempts):
            Searches and locates the element based on the provided locator strategy.

    Private Methods:
        _fetch_elements_item(actual_locator: By | dict, retries: Optional[int] = config.element.search_attempts):
            Searches and locates multiple elements based on the provided locator strategy.
        _exec_search(actual_locator: By | dict, retries: Optional[int] = config.element.search_attempts):
            Executes the element search based on the provided locator strategy.
        _fetch_scope_object(actual_locator): Fetches the scope object for the element search based on the locator.
        _fetch_actual_locator(locator: Optional[By | dict] = None) -> By:
            Fetches the actual locator for the element search, considering platform, operating system, and viewport
            specific locators.
    """

    # Define a sentinel object to represent "not searched yet"
    _NOT_SEARCHED_YET = object()

    def __init__(self, parent, locator: By | dict, name: str):
        """
        Initialize a LocatableElement instance.

        Args:
            parent (object): The parent element or container that contains this element.
            locator (By | dict): The locator strategy used to identify the element. It can be a `By` object or a
                                 dictionary of locators categorized by platform, operating system, and viewport.
            name (str): A descriptive name for the element used for logging and identification purposes.
        """
        self.parent = parent
        self._locator = locator
        self._element_adapter: Any = self._NOT_SEARCHED_YET
        if bool(re.match("^\d+$", name)):
            self.__full_name__ = f"{parent.__full_name__}[{name}]"
        else:
            self.__full_name__ = f"{parent.__full_name__}.{name}"

    @property
    def element_adapter(self) -> Any:
        """
        Get the element adapter representing the located element or elements.

        The `element_adapter` property performs an automatic search once someone tries to interact with the object.
        If the element is not searched yet, it will execute `find_itself()` method to locate the element.

        Returns:
            object or list: The element adapter representing the located element or elements.
        """
        if self._element_adapter is not self._NOT_SEARCHED_YET:
            return self._element_adapter

        # perform automatic search once someone tries to interact with object
        self.find_itself()
        return self._element_adapter

    @property
    def platform(self) -> str:
        """
        Get the platform of the element.

        The platform is determined by the document holder's platform. The platform specifies the environment in which the
        element is located, such as 'web', 'mobile', or 'desktop'.

        Returns:
            str: The platform of the element.
        """
        # do not use root as root platform may be one, but it has webview, so for that document platform will be
        # different
        return self.document_holder.platform

    @property
    def viewport(self) -> str:
        """
        Get the viewport of the element.

        The viewport is determined by the document holder's viewport. The viewport represents the size and layout of the
        viewable area in which the element is displayed, such as 'xs', 'sm', 'md', 'lg', or 'xl'.

        Returns:
            str: The viewport of the element.
        """
        # do not use root as root viewport may be bigger than its built-ins, like iFrames and WebViews, and their
        # dimensions are not necessarily equal to the app one
        return self.document_holder.viewport

    @property
    def os(self) -> str:
        """
        Get the operating system of the element.

        The operating system is determined by the root element's operating system. It represents the underlying operating
        system on which the application or web page is running.

        Returns:
            str: The operating system of the element.
        """
        return self.root.os

    @property
    def root(self) -> Any:
        """
        Get the root element or container of the current element.

        Returns:
            object: The root element or container.
        """
        if hasattr(self.parent, "root"):
            return self.parent.root

        return self.parent

    @property
    def document_holder(self):
        """
        Get the document holder that contains this element.

        The document holder is the top-level element or container that encapsulates the entire page or application.

        Returns:
            object: The document holder that contains this element.
        """
        node = self.parent
        while hasattr(node, "parent"):
            if hasattr(node, "context_manager") or hasattr(node, "content_manager"):
                return node
            node = node.parent
        return node

    @property
    def contains_multiple_elements(self):
        """
        Check if the Elements instance contains multiple elements.

        Returns:
            bool: True if the Elements instance contains multiple elements, False otherwise.
        """
        return hasattr(self, "_elements_cache")

    @property
    def is_multi_child(self):
        """
        Get a boolean value indicating whether this SingleElement is a child of a MultipleElement.

        A SingleElement can be part of a MultipleElement, which represents a collection of elements.
        If this property returns True, it means the SingleElement is part of a MultipleElement;
        otherwise, it is not associated with a MultipleElement.

        Returns:
            bool: True if this SingleElement is a child of a MultipleElement, False otherwise.
        """
        if hasattr(self.parent, "contains_multiple_elements"):
            return self.parent.contains_multiple_elements

        return False

    def find_itself(self, retries: int = config.element.search_attempts):
        """
        Searches and locates the element based on the provided locator strategy.

        Args:
            retries (int, optional): The number of retries for element search in case of failure. Defaults to
                                     `config.element.search_attempts`.

        Raises:
            NoSuchElementException: If the element is not found after the specified number of retries.
        """
        if hasattr(self.parent, "__resolve__"):
            self.parent.__resolve__()

        if self._locator is not None:
            actual_locator = self._fetch_actual_locator(self._locator)
        elif self._locator is None and hasattr(self, "default_locator"):
            actual_locator = self._fetch_actual_locator(self.default_locator)
        else:
            raise Exception("No Locator")

        # applicable only for Multiple Element children
        if actual_locator.by == LocatorStrategies.ELEMENTS_ITEM:
            self._fetch_elements_item(actual_locator)
        else:
            self._exec_search(actual_locator, retries)

    def _fetch_elements_item(
        self,
        actual_locator: Any,
        retries: int = config.element.search_attempts,
    ):
        """
        Searches and locates multiple elements based on the provided locator strategy.

        Args:
            actual_locator (By | dict): The actual locator strategy used to identify the elements.
            retries (int, optional): The number of retries for element search in case of failure. Defaults to
                                     `config.element.search_attempts`.

        Raises:
            NoSuchElementException: If the element is not found after the specified number of retries.
        """
        try:
            self._element_adapter = self.parent.element_adapter[
                int(actual_locator.value)
            ]
        except IndexError:
            self.parent.find_itself()
            if retries > 0:
                return self.find_itself(retries - 1)
            else:
                raise NoSuchElementException(
                    f"[{self.parent.__full_name__}] have only {len(self.parent)} items, but"
                    f" requested item index is {actual_locator.value}"
                )

    def _exec_search(
        self,
        actual_locator: By | dict,
        retries: int = config.element.search_attempts,
    ):
        """
        Executes the element search based on the provided locator strategy.

        Args:
            actual_locator (By | dict): The actual locator strategy used to identify the element.
            retries (int, optional): The number of retries for element search in case of failure. Defaults to
                                     `config.element.search_attempts`.

        Raises:
            NoSuchElementException: If the element is not found after the specified number of retries.
        """
        # If elements require resolving the window handle/context/content before the search itself,
        # then its parent must implement the __resolve__ method. Otherwise, no extra actions are needed,
        # and we can directly fire the find call.

        search_scope = self._fetch_scope_object(actual_locator)
        # do not try to search child element for error elements
        if isinstance(search_scope, HyperionUIException):
            raise search_scope

        logger.debug(f"[{self.__full_name__}] Searching element by {actual_locator}")
        try:
            # Elements cache is a specific attribute for the Elements class family, checking by class will cause a
            # cyclomatic import error since Elements class is also a child of the LocatableElement class.
            if self.contains_multiple_elements:
                self._element_adapter = cast(
                    Sized, search_scope.find_elements(actual_locator)
                )
                # For an array of elements, we need to know the count, so log it.
                logger.debug(
                    f"[{self.__full_name__}] {len(self._element_adapter)} elements found"
                )
                # Normally, find elements does not raise any exception.
                if len(self._element_adapter) == 0 and retries > 0:
                    time.sleep(config.element.search_retry_timeout)
                    return self.find_itself(retries - 1)
            else:
                self._element_adapter = search_scope.find_element(actual_locator)
        except HyperionUIException as uie:
            # During retries, different exceptions may occur, so for debug purposes, we need to log them all.
            logger.debug(
                f"[{self.__full_name__}] Exception intercepted! {uie.__class__.__name__}: {uie}"
            )
            if retries > 0:
                if (
                    uie.__class__.__name__ == "StaleElementReferenceException"
                    and hasattr(self.parent, "find_itself")
                ):
                    if self.parent.is_multi_child:
                        self.parent.parent.find_itself()
                    self.parent.find_itself()
                else:
                    time.sleep(config.element.search_retry_timeout)
                return self.find_itself(retries - 1)
            else:
                self.automation_adapter = NoSuchElementException(
                    f"[{self.__full_name__}] Element was not found!\n"
                    f"Locator: {actual_locator}"
                )

    def _fetch_scope_object(self, actual_locator):
        """
        Fetches the scope object for the element search based on the locator.

        The scope object is determined based on the locator's search scope. If the search scope is 'document', the root
        element's automation adapter is returned. Otherwise, if the parent element has an automation adapter, it is
        returned. Otherwise, the parent element's element adapter is returned.

        Explanation:
        This method needs to check if self.parent is an instance of the IFrame class. However, IFrame is a child class of
        LocatableElement. Placing the import of IFrame at the top of the file would lead to a cyclic import issue.

        Cyclic imports occur when two or more modules depend on each other, directly or indirectly. In this case, if
        IFrame is imported at the top of the file that contains LocatableElement, and IFrame imports LocatableElement,
        it creates a circular dependency that Python cannot resolve.

        To avoid this, the import of IFrame is placed within the method _fetch_scope_object. This ensures that the
        IFrame class is only imported when this method is executed and not during the initial module import. By
        doing so, we prevent the cyclic import issue, and the code remains functional without any import errors.

        Args:
            actual_locator (By | dict): The actual locator strategy used to identify the element.

        Returns:
            object: The scope object used for element search.
        """
        from .iframe import IFrame

        if actual_locator.search_scope == "document":
            return self.root.automation_adapter

        if isinstance(self.parent, IFrame):
            return self.root.automation_adapter

        if hasattr(self.parent, "automation_adapter"):
            return self.parent.automation_adapter

        return self.parent.element_adapter

    def _fetch_actual_locator(self, locator: Union[By, Dict[str, Any]]) -> By:
        """
        Fetches the actual locator for the element search, considering platform, operating system, and viewport specific
        locators.

        The actual locator is resolved based on a strict and recursive order, checking platform-specific locators
        first, followed by operating system-specific locators, and finally viewport-specific locators.

        Args:
            locator (By | dict, optional): The locator strategy to be resolved. Defaults to None, in which case the
                                           class's locator attribute is used.

        Returns:
            By: The resolved actual locator strategy.
        """
        # if the locator is a By object, it's the only locator
        if locator.__class__ is not dict:
            return cast(By, locator)

        # if the locator is a dictionary, it means that it has multiple By objects categorized by platform, operating
        # system, and viewport. We need to decide which one to use based on the current execution environment.

        # The resolve order is strict and recursive:

        # 1. Check if there's a platform-specific locator. If so, use it.
        # 2. If there's no platform-specific locator, check if there's an operating system-specific locator. If so, use
        # it.
        # 3. If there's no operating system-specific locator, check if there's a viewport-specific locator. If so, use
        # it.
        # 4. If none of the above match, try to use the 'default' locator.
        # 5. If no locator matches the current environment, raise an IncorrectLocatorException.
        locator = cast(Dict, locator)
        # Resolve platform-specific locator if any
        if self.platform in locator.keys():
            return self._fetch_actual_locator(locator[self.platform])
        # Resolve operating system-specific locator if any
        if self.os in locator.keys():
            return self._fetch_actual_locator(locator[self.os])
        # Resolve viewport-specific locator if any
        if self.viewport in locator.keys():
            return self._fetch_actual_locator(locator[self.viewport])
        # If none of the above match, try to use the 'default' locator if provided
        if "default" in locator.keys():
            return self._fetch_actual_locator(locator["default"])

        # If no locator matches the current environment, raise an IncorrectLocatorException
        raise IncorrectLocatorException(
            f"[{self.__full_name__}] Does not contain a locator for '{self.platform}' "
            f"platform, '{self.os}' operating system, and '{self.viewport}' viewport"
        )


import fileinput
import re

def main():
    regex = re.compile('([FB]{7})([LR]{3})$')
    trans = str.maketrans('FBLR', '0101')

    ids = [int(line.translate(trans), 2) for line in fileinput.input()]
    ids.sort()

    previous = ids[0]
    for id in ids[1:]:
        if id != previous + 1:
            print(previous + 1)
        previous = id

if __name__ == "__main__":
    main()


#!/usr/bin/python3
import sys
import os
import numpy as np
import csv
import pandas as pd
import tempfile
import subprocess

def get_avp_input(file):
	file=open(file)
	copy=False
	seq=[]
	for line in file:

		if "ATOM" in line:
			if "H  95" in line:
				copy = True
				seq.append(line)
				continue
			elif "H 103" in line:
				#seq.append(line)
				copy = False
				continue
			elif copy==True:
				seq.append(line)
	loop_pdb_tmp = open("./loop_pdb_tmp.txt", "w")
	for i in seq:
		loop_pdb_tmp.write(i)
	#print(loop_pdb_tmp.read())
	loop_pdb_tmp.close()
	return None

def run_avp():
	#make temporary files to store the avp output
	avp_voids_tmp = open("./avp_voids_tmp.txt", "a")
	near_voids_tmp = open("./near_voids_tmp.txt", "a")
	avp_voids_tmp.close()
	near_voids_tmp.close()

	#run the avp command - -n option so that additional file is outputted that contains the pdb data of all atoms near the voids
	#solvent size of 1.4 Angstroms (water-molecule) and probe size of 0.5 is suggested in the paper by A. CR Martin (as if probe too small,
	#separate voids join into one huge one)
	command="avp -q -R -p 0.5 -s 1.4 -n {} {} {}".format("./near_voids_tmp.txt", "./loop_pdb_tmp.txt", "./avp_voids_tmp.txt")
	avp_out_tmp=subprocess.call(command, shell=True)

	#read and parse the avp output file with the voids
	columns_voids=["largest_total", "buried_surface_void", "type", "overall_vols", "surface_buried_vols", "ind_vols", "midpoint", "x", "y", "z", "void_type"]
	voids=pd.read_csv("./avp_voids_tmp.txt", header=None, delim_whitespace=True, names=columns_voids)
	if len(voids.loc[(voids['largest_total'] == "Void:")].to_numpy())==0:
		return pd.DataFrame()
	surface_ind_voids= voids.loc[(voids['midpoint'] == "midpoint")&(voids['void_type']=="Buried-void")]
	surface_ind_voids=surface_ind_voids[["x", "y", "z", "ind_vols"]]
	buried_ind_voids= voids.loc[(voids['midpoint'] == "midpoint")&(voids['void_type']=="Buried-void")]
	buried_ind_voids=buried_ind_voids[["x", "y", "z", "ind_vols"]]
	summarized_voids=pd.DataFrame()

	largest_void=(voids.loc[(voids['largest_total'] == "Largest")&(voids['buried_surface_void']=="void"),"overall_vols"]).to_numpy()[0]
	summarized_voids["total_buried"]=voids.loc[(voids['largest_total'] == "Total")&(voids['buried_surface_void']=="buried"), "surface_buried_vols"]
	summarized_voids["total_surface"]=float(voids.loc[(voids['largest_total'] == "Total")&(voids['buried_surface_void']=="surface"),"surface_buried_vols"])
	summarized_voids["largest_buried"]=float(voids.loc[(voids['largest_total'] == "Largest")&(voids['buried_surface_void']=="buried"),"surface_buried_vols"])
	summarized_voids["largest_surface"]=float(voids.loc[(voids['largest_total'] == "Largest")&(voids['buried_surface_void']=="surface"),"surface_buried_vols"])
	summarized_voids["total_void"]=float(voids.loc[(voids['largest_total'] == "Total")&(voids['buried_surface_void']=="void"),"overall_vols"])
	summarized_voids["largest_void"]=float(voids.loc[(voids['largest_total'] == "Largest")&(voids['buried_surface_void']=="void"),"overall_vols"])
	largestx=voids.loc[(voids['largest_total'] == "Void:")&(voids['ind_vols']==max(voids["ind_vols"].to_numpy()))]
	summarized_voids["largest_x"]=largestx[["x"]].to_numpy()
	largesty=voids.loc[(voids['largest_total'] == "Void:")&(voids['ind_vols']==max(voids["ind_vols"].to_numpy()))]
	summarized_voids["largest_y"]=largesty[["y"]].to_numpy()
	largestz=voids.loc[(voids['largest_total'] == "Void:")&(voids['ind_vols']==max(voids["ind_vols"].to_numpy()))]
	summarized_voids["largest_z"]=largestx[["z"]].to_numpy()

	#read and parse the avp output file containing the atoms near the voids
	columns_near=["atom_nr", "atom_name", "res", "chain", "pos","x", "y", "z", "occ", "temp", "element"]
	near_voids=pd.read_csv("./near_voids_tmp.txt", header=None, delim_whitespace=True, names=columns_near)
	summarized_near=pd.DataFrame()
	summarized_near=near_voids[["res", "atom_name","chain", "pos", "x", "y", "z"]]
	dist_list=[]
	for i in range(summarized_near.shape[0]):
		nearest=(abs(float(summarized_near["x"].to_numpy()[i])-float(summarized_voids["largest_x"].to_numpy()[0]))
		+abs(float(summarized_near["y"].to_numpy()[i])-float(summarized_voids["largest_y"].to_numpy()[0]))
		+abs(float(summarized_near["z"].to_numpy()[i])-float(summarized_voids["largest_z"].to_numpy()[0])))

		dist_list.append(nearest)
	index_min=dist_list.index(min(dist_list))
	closest=summarized_near.iloc[index_min].to_numpy()
	closest = pd.DataFrame(closest.reshape(-1, len(closest)),columns=["res", "atom_name","chain", "pos", "x_closest", "y_closest", "z_closest"], index=None)
	print(closest)

	#summarized_near=summarized_near.drop_duplicates(subset=["res", "chain", "pos"])

	#join the datafram on the voids with the df on the atom closest to the largest void

	summarized_voids.reset_index(drop=True, inplace=True)
	closest.reset_index(drop=True, inplace=True)
	voids_df=pd.concat([summarized_voids, closest], axis=1)

	#remove the files, as no longer needed
	os.remove("./near_voids_tmp.txt")
	os.remove("./avp_voids_tmp.txt")
	os.remove("./loop_pdb_tmp.txt")

	return voids_df

def get_voids(file):
	get_avp_input(file)
	voids_df=run_avp()
	voids_df=voids_df[["pos", 'largest_void', "total_surface", "largest_surface", "total_void"]]
	print(voids_df)
	return voids_df

if __name__ == '__main__':
	print(get_voids(sys.argv[1]))

#sample output from avp for the voids
"""Void:    1 voxelCount:   3 volume:    2.218 midpoint:   21.825  40.522  31.265 Surface-void
Void:    2 voxelCount:   1 volume:    2.342 midpoint:   23.825  40.022  32.765 Surface-void
Void:    3 voxelCount:  12 volume:    9.664 midpoint:   26.325  40.022  30.765 Surface-void
Void:    4 voxelCount:   1 volume:    0.922 midpoint:   24.825  41.022  32.765 Surface-void

Total buried void volume: 0.000000
Largest buried void volume: 0.000000

Total surface void volume: 15.146000
Largest surface void volume: 9.664000

Total void volume: 15.146000
Largest void volume: 9.664000"""

#sample output from avp for the nearest atoms to voids
"""ATOM   1589  O   ARG H  96      23.300  40.621  30.807  1.00  0.00           O
ATOM   1597  O   ASP H  97      21.354  39.948  33.284  1.00  0.00           O
ATOM   1610  N   ARG H  99      24.506  38.512  33.129  1.00  0.00           N
ATOM   1620  O   ARG H  99      26.261  38.955  30.956  1.00  0.00           O
ATOM   1589  O   ARG H  96      23.300  40.621  30.807  1.00  0.00           O
ATOM   1632  CG  ASP H 101      26.201  39.441  26.525  1.00  0.00           C
ATOM   1571  CA  GLU H  95      28.096  41.866  31.323  1.00  0.00           C
ATOM   1610  N   ARG H  99      24.506  38.512  33.129  1.00  0.00           N
ATOM   1579  N   ARG H  96      25.932  41.924  30.352  1.00  0.00           N
ATOM   1598  N   TYR H  98      23.252  40.828  34.152  1.00  0.00           N"""


from . import paragraphs, syntax, terms, code_terms
from docutils import nodes
from sphinx.domains import ObjType
from sphinx.environment.collectors import EnvironmentCollector
from sphinx.roles import SphinxRole
from sphinx.transforms import SphinxTransform
import sphinx

KINDS = [
    paragraphs,
    syntax,
    terms,
    code_terms,
]

class DefIdNode(nodes.Element):
    def __init__(self, kind, text):
        text, id = parse_target_from_text(text)
        super().__init__(def_kind=kind, def_text=text, def_id=id_from_text(id))

    def astext(self):
        return self["def_text"]

class DefRefNode(nodes.Element):
    def __init__(self, kind, source_doc, text):
        text, target = parse_target_from_text(text)

        super().__init__(
            ref_kind=kind,
            ref_source_doc=source_doc,
            ref_text=text,
            ref_target=id_from_text(target),
        )

    def astext(self):
        return self["ref_text"]

def parse_target_from_text(text):
    if "<" in text and text.endswith(">"):
        target_start = text.rfind("<")
        target = text[target_start + 1 : len(text) - 1]
        text = text[:target_start].rstrip()
    elif "[" in text and "]" in text:
        target = text[text.find("[") + 1 : text.rfind("]")]
        text = text.replace("[", "").replace("]", "")
    else:
        target = text

    return text, target

class DefIdRole(SphinxRole):
    def __init__(self, kind):
        self.kind = kind

    def run(self):
        return [DefIdNode(self.kind, self.text)], []

class DefRefRole(SphinxRole):
    def __init__(self, kind):
        self.kind = kind

    def run(self):
        return [DefRefNode(self.kind, self.env.docname, self.text)], []

class DefinitionsCollector(EnvironmentCollector):
    def clear_doc(self, app, env, docname):
        """
        Remove all definitions and references contained into a document.

        This is called by Sphinx during incremental builds, either when a
        document was removed or when the document has been changed. In the
        latter case, process_doc is called after this method.
        """
        for kind in KINDS:
            storage = get_storage(env, kind)
            for item in list(storage.values()):
                if item.document == docname:
                    del storage[item.id]

    def merge_other(self, app, env, docnames, other):
        """
        Merge the collected informations from two environments into one.

        Sphinx supports parallel builds, with each process having its own
        environment instance, but once each document is processed those
        parallel environments need to be merged together. This method does it.
        """
        for kind in KINDS:
            storage = get_storage(env, kind)
            other_storage = get_storage(other, kind)

            for item in other_storage.values():
                if item.document in docnames:
                    storage[item.id] = item

    def process_doc(self, app, document):
        """
        Collect all the definitions and references present in the document.

        This method can expect no existing information about the same document
        being stored in the environment, as during incremental rebuilds the
        clear_doc method is called ahead of this one.
        """
        for kind in KINDS:
            storage = get_storage(app.env, kind)
            nodes = filter(
                lambda node: node["def_kind"] == kind.NAME,
                document.findall(DefIdNode),
            )
            for item in kind.collect_items_in_document(app, nodes):
                storage[item.id] = item

class DefinitionsTransform(SphinxTransform):
    default_priority = 500

    def apply(self):
        for kind in KINDS:
            storage = get_storage(self.env, kind)

            for node in self.document.findall(DefIdNode):
                if node["def_kind"] != kind.NAME:
                    continue
                item = storage[node["def_id"]]
                kind.replace_id_node(self.app, node, item)

            for node in self.document.findall(DefRefNode):
                if node["ref_kind"] != kind.NAME:
                    continue

                if node["ref_target"] in storage:
                    item = storage[node["ref_target"]]
                    node.replace_self(
                        sphinx.util.nodes.make_refnode(
                            self.app.builder,
                            node["ref_source_doc"],
                            item.document,
                            item.anchor(),
                            kind.create_ref_node(self.env, node["ref_text"], item),
                        )
                    )
                else:
                    new = nodes.inline(
                        "",
                        "",
                        kind.create_ref_node(self.env, node["ref_text"], None),
                    )
                    new["classes"].append("spec-missing-ref")
                    node.replace_self(new)

def get_objects(env):
    """
    Get objects that will be included in the objects.inv file and in search.

    The return type is a list of tuples of:
    * Fully qualified name
    * Display name in search
    * Object type
    * Document name
    * Anchor
    * Search priority (1: default, 0: important, 2: not important, -1: ignore)
    """
    result = []
    for kind in KINDS:
        storage = get_storage(env, kind)
        for item in storage.values():
            result.append(
                (
                    item.id,
                    item.display_name(env),
                    kind.NAME,
                    item.document,
                    item.anchor(),
                    1 if item.include_in_search() is not None else -1,
                )
            )

    return result

def get_storage(env, kind):
    key = f"spec_items_{kind.NAME}"
    if not hasattr(env, key):
        setattr(env, key, {})
    return getattr(env, key)

def get_roles():
    result = {}
    for kind in KINDS:
        result["d" + kind.ROLE[0]] = DefIdRole(kind.NAME)
        result[kind.ROLE[0]] = DefRefRole(kind.NAME)
    return result

def get_object_types():
    result = {}
    for kind in KINDS:
        result[kind.NAME] = ObjType(kind.PRETTY_NAME, kind.ROLE)
    return result

def id_from_text(text):
    return "".join(c if c.isalnum() else "_" for c in text.lower())

def setup(app):
    app.add_node(DefIdNode)
    app.add_env_collector(DefinitionsCollector)
    app.add_post_transform(DefinitionsTransform)


import csv

def parse_data_from_file(filename):

    temperatures = []

    with open(filename) as csvfile:

        reader = csv.reader(csvfile, delimiter=",")

        # Skip the first line
        next(reader)

        # Append to lists
        for row in reader:
            temperatures.append(float(row[1]))

    return temperatures


#https://jjluveeecom.tistory.com/13?category=370124
# 오우 이 징검다리는 좀 심각해 진짜 모르겠어
# 이분탐색문제에서 어떤것을 이분탐색의 기준으로 삼을지 판단해야한다.
# 이 문제에서는 돌과 돌사이의 거리가 그 기준이다
# 돌과 돌 사이의 거리는 rocks[1]-prev값이다.
#min_inter값은 리턴되는 값을 구하기위해 간격들의 최소값을 구하기위한 변수이다. math_inf는 무한데 값으로 최소값을 구하기위한 초기값을 넣어두는 것이다

distance=25
rocks=[2, 14, 11, 21, 17]
n=2
import math

answer=0
start,end=0,distance
rocks=sorted(rocks)
rocks.append(distance)
rnum=len(rocks)

while start<=end:
    remove=0
    prev=0
    min_inter=math.inf
    mid = (start+end)//2

    for i in range(rnum):
        if rocks[i]-prev < mid:  #rocks[i]-prev가 inter값
            remove+=1 #바위제거

        else : #바위 제거 안할경우
            min_inter=min(min_inter,rocks[i]-prev) #inter값 갱신.
            prev=rocks[i] #현재 바위위치를 prev로

    #너무 많이 제거되었을 경우, 기준을 높여서 제거를 줄여야함
    if remove > n:
        end=mid-1
    #적게 제거되었을 경우, 기준을 낮춰서 제거를 늘려야함
    else:
        answer=min_inter
        start=mid+1
print(answer)


import collections
import math
import os
import random
import torch
from d2l import torch as d2l

d2l.DATA_HUB['ptb'] = (d2l.DATA_URL + 'ptb.zip', '319d85e578af0cdc590547f26231e4e31cdf1e42')

#@save
def read_ptb():
    """Load the PTB dataset into a list of text lines."""
    data_dir = d2l.download_extract('ptb')
    # Read the training set
    with open(os.path.join(data_dir, 'ptb.train.txt')) as f:
        raw_text = f.read()
    return [line.split() for line in raw_text.split('\n')]

def subsample(sentences, vocab):
    """Subsample high-frequency words."""
    # Exclude unknown tokens ('<unk>')
    sentences = [[token for token in line if vocab[token] != vocab.unk]
                 for line in sentences]
    counter = collections.Counter([
        token for line in sentences for token in line])
    num_tokens = sum(counter.values())

    # Return True if `token` is kept during subsampling
    def keep(token):
        return random.uniform(0, 1) < math.sqrt(1e-4 / counter[token] * num_tokens)

    return ([[token for token in line if keep(token)] for line in sentences],
            counter)
# def compare_counts(token):
#     return (f'# of "{token}": '
#             f'before={sum([l.count(token) for l in sentences])}, '
#             f'after={sum([l.count(token) for l in subsampled])}')

def get_centers_and_contexts(corpus, max_window_size):
    """Return center words and context words in skip-gram."""
    centers, contexts = [], []
    for line in corpus:
        # To form a "center word--context word" pair, each sentence needs to
        # have at least 2 words
        if len(line) < 2:
            continue
        centers += line
        for i in range(len(line)):  # Context window centered at `i`
            window_size = random.randint(1, max_window_size)
            indices = list(range(max(0, i - window_size),
                                 min(len(line), i + 1 + window_size)))
            indices.remove(i)
            contexts.append([line[idx] for idx in indices])
    return centers, contexts

def get_negatives(all_contexts, vocab, counter, K):
    """Return noise words in negative sampling."""
    # Sampling weights for words with indices 1, 2, ... (index 0 is the
    # excluded unknown token) in the vocabulary
    sampling_weights = [counter[vocab.to_tokens(i)]**0.75
                        for i in range(1, len(vocab))]
    all_negatives, generator = [], RandomGenerator(sampling_weights)
    for contexts in all_contexts:
        negatives = []
        while len(negatives) < len(contexts) * K:
            neg = generator.draw()
            # Noise words cannot be context words
            if neg not in contexts:
                negatives.append(neg)
        all_negatives.append(negatives)
    return all_negatives

#@save
def batchify(data):
    """Return a minibatch of examples for skip-gram with negative sampling."""
    max_len = max(len(c) + len(n) for _, c, n in data)
    centers, contexts_negatives, masks, labels = [], [], [], []
    for center, context, negative in data:
        cur_len = len(context) + len(negative)
        centers += [center]
        contexts_negatives += [context + negative + [0] * (max_len - cur_len)]
        masks += [[1] * cur_len + [0] * (max_len - cur_len)]
        labels += [[1] * len(context) + [0] * (max_len - len(context))]
    return (torch.tensor(centers).reshape((-1, 1)), torch.tensor(
        contexts_negatives), torch.tensor(masks), torch.tensor(labels))

#@save
def load_data_ptb(batch_size, max_window_size, num_noise_words):
    """Download the PTB dataset and then load it into memory."""
    num_workers = d2l.get_dataloader_workers()
    sentences = read_ptb()
    vocab = d2l.Vocab(sentences, min_freq=10)
    subsampled, counter = subsample(sentences, vocab)
    corpus = [vocab[line] for line in subsampled]
    all_centers, all_contexts = get_centers_and_contexts(
        corpus, max_window_size)
    all_negatives = get_negatives(
        all_contexts, vocab, counter, num_noise_words)

    class PTBDataset(torch.utils.data.Dataset):
        def __init__(self, centers, contexts, negatives):
            assert len(centers) == len(contexts) == len(negatives)
            self.centers = centers
            self.contexts = contexts
            self.negatives = negatives

        def __getitem__(self, index):
            return (self.centers[index], self.contexts[index],
                    self.negatives[index])

        def __len__(self):
            return len(self.centers)

    dataset = PTBDataset(all_centers, all_contexts, all_negatives)

    data_iter = torch.utils.data.DataLoader(dataset, batch_size, shuffle=True,
                                      collate_fn=batchify,
                                      num_workers=num_workers)
    return data_iter, vocab

class RandomGenerator:
    """Randomly draw among {1, ..., n} according to n sampling weights."""
    def __init__(self, sampling_weights):
        # Exclude
        self.population = list(range(1, len(sampling_weights) + 1))
        self.sampling_weights = sampling_weights
        self.candidates = []
        self.i = 0

    def draw(self):
        if self.i == len(self.candidates):
            # Cache `k` random sampling results
            self.candidates = random.choices(
                self.population, self.sampling_weights, k=10000)
            self.i = 0
        self.i += 1
        return self.candidates[self.i - 1]

#
# sentences = read_ptb()
# f'# sentences: {len(sentences)}'
# vocab = d2l.Vocab(sentences, min_freq=10)
# subsampled, counter = subsample(sentences, vocab)
# # d2l.show_list_len_pair_hist(['origin', 'subsampled'], '# tokens per sentence', 'count', sentences, subsampled)
# print(compare_counts('the'))
# print(compare_counts('join'))
# corpus = [vocab[line] for line in subsampled]
# print(corpus[:3])
# tiny_dataset = [list(range(7)), list(range(7, 10))]
# print('dataset', tiny_dataset)
# for center, context in zip(*get_centers_and_contexts(tiny_dataset, 2)):
#     print('center', center, 'has contexts', context)
#
#
# all_centers, all_contexts = get_centers_and_contexts(corpus, 5)
# print(f'# center-context pairs: {sum([len(contexts) for contexts in all_contexts])}')
#
# generator = RandomGenerator([2, 3, 4])
# print([generator.draw() for _ in range(10)])
#
# #@save
#
#
# all_negatives = get_negatives(all_contexts, vocab, counter, 5)
#
#
# x_1 = (1, [2, 2], [3, 3, 3, 3])
# x_2 = (1, [2, 2, 2], [3, 3])
# batch = batchify((x_1, x_2))
#
# names = ['centers', 'contexts_negatives', 'masks', 'labels']
# for name, data in zip(names, batch):
#     print(name, '=', data)
#
#
# names = ['centers', 'contexts_negatives', 'masks', 'labels']
# data_iter, vocab = load_data_ptb(512, 5, 5)
# for batch in data_iter:
#     for name, data in zip(names, batch):
#         print(name, 'shape:', data.shape)
#     break


import os

from flask import current_app, Blueprint, flash, redirect, render_template, request, url_for, send_from_directory
from flask_login import current_user, login_required

from botnet.dao import file_dao, payload_dao
from web_gui.utils import get_sessions_serialized, get_tasks_serialized

# Blueprint
root = Blueprint('root', __name__)

# Routes
@root.route("/home")
@root.route("/")
def home():
	if current_user.is_authenticated:
		return redirect(url_for('root.sessions'))
	else:
		return redirect(url_for('users.login'))

@root.route("/dashboard")
@root.route("/sessions")
@login_required
def sessions():
	sessions = get_sessions_serialized(current_user.id)
	return render_template("sessions.html", sessions=sessions)

@root.route("/payloads")
@login_required
def payloads():
	payloads = payload_dao.get_user_payloads(current_user.id)
	return render_template("payloads.html",
							payloads=payloads[::-1],
							owner=current_user.username)

@root.route("/files")
@login_required
def files():
	user_files = file_dao.get_user_files(current_user.id)
	return render_template("files.html",
							files=user_files[::-1],
							owner=current_user.username)

@root.route("/tasks")
@login_required
def tasks():
	tasks = get_tasks_serialized(current_user.id)
	return render_template("tasks.html",
							tasks=tasks[::-1])

#####################
#
# DOWNLOADS
#
#####################

@root.route("/executables/<user>/exe/<filename>")
@login_required
def download_payload(user, filename):
	return send_from_directory(os.path.join(current_app.config['OUTPUT_DIR'], user, 'exe'), filename, as_attachment=True)

@root.route("/exfiltratedfiles/<user>/<session>/<filename>")
@login_required
def download_file(user, session, filename):
	return send_from_directory(os.path.join(os.path.abspath('botnet/exfiltratedfiles'), user, session), filename, as_attachment=True)


import numpy as np
import csv

path = '../data/YelpChi/labeled_reviews.tsv'
pathMetaNYC = '../data/YelpNYC/metadata'
pathReviewNYC = '../data/YelpNYC/reviewContent'
outpath = '../data/large_balanced.tsv'

def main():
    true_reviews = []
    fake_reviews = []
    print("Parsing through Chicago file")
    with open(path) as tsvfile:
        reader = csv.reader(tsvfile, dialect='excel-tab')
        for row in reader:
            if row[1] == '0':
                true_reviews.append(row)
            else:
                fake_reviews.append(row)
    print("Parsing through NYC file")
    with open(pathMetaNYC) as metaNYC, open(pathReviewNYC) as reviewNYC:
        meta_reader = csv.reader(metaNYC, dialect = 'excel-tab')
        review_reader = csv.reader(reviewNYC, dialect = 'excel-tab')
        for metarow, reviewrow in zip(meta_reader, review_reader):
            if metarow[3] == '1':
                true_reviews.append([reviewrow[3], '0'])
            else:
                fake_reviews.append([reviewrow[3], '1'])

    print("Creating new true review list")
    num_fake = len(fake_reviews)
    num_true = len(true_reviews)
    true_reviews_indices = np.random.choice(num_true, num_fake, replace=False)
    true_reviews_subset = []
    for index in true_reviews_indices:
        true_reviews_subset.append(true_reviews[index])
    under_sample_reviews = np.concatenate([true_reviews_subset, fake_reviews])

    print("Saving true reviews")
    with open(outpath, 'w', newline='\n') as outfile:
        writer = csv.writer(outfile, delimiter='\t')
        for row in under_sample_reviews:
            writer.writerow(row)

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
# @Author: MirrorAi
# @Date:   2020-01-08 20:53:54
# @Last Modified by:   MirroAi
# @Last Modified time: 2020-01-08 20:57:00

# 输出一个随机数

import random

def run():

    l = input("请输入随机数的上下边界，如 4-50：\n").split("-")

    a, b = int(l[0]), int(l[1])
    print("输出随机数为：%d" % random.randint(a, b))

if __name__ == "__main__":
    run()


def cursoMatriculadoEstudiante():
    diccionarioAlumnos ={'1':['-','ninja'],'2':['-','paolo'],'3':['-','andy'],'4':['-','piter']}
    alumnoMatriculado={'2': [[1, '-'], [4, '-']],'1':[[1,'-']],'3':[[2,'-']]}
    diccionarioCursos={'1':['Logica'],'2':['computacion'],'3':['Redes'],'4':['linux']}
    identificacionEstudiante =(input("Identificacion del Estudiante : "))
    print("CURSOS-POR-ALUMNO\n")
    print("El alumno ",diccionarioAlumnos[identificacionEstudiante][1]," esta matriculado en los cursos : ")

    for ide,value in alumnoMatriculado.items():
        iterando =0
        #print('cantidad de cursos : ',cantidad)
        if(identificacionEstudiante==ide):
            cantidad=len(alumnoMatriculado[ide])
            while(cantidad>0):
                x = ((alumnoMatriculado[ide][iterando][0])) #tengo el codigo de curso
                parseo = str(x)
                print(diccionarioCursos[parseo][0])
                iterando +=1
                cantidad -=1

def cursosDictaProfesor():
        #datos = [nombre, apellido,direccion,telefono]
    diccionarioCursos={'1':['Logica',23,'4','5',33,'1'],'2':['computacion',25,'4','5',33,'1'],'3':['Redes',23,'4','5',33,'2'],'4':['linux',23,'4','5',33,'2']}
    diccionarioProfesores={'1':['ANGEL','BONILLA','SAN MARTIN','98823231'],'2':['PEDRO','LOLO','PALMERAS','98852318'],'3':['PITER','CASTLE','LIBRE','9885248']}
    cedula= (input("Cedula del Profesor : "))
    print("El Profesor : ",diccionarioProfesores[cedula][0]," ",diccionarioProfesores[cedula][1],' dicta : ')
    #print(diccionarioCursos['1'][0])
    for ide,value in diccionarioCursos.items():
        x=diccionarioCursos[ide][5]
        if(cedula==x):
            print(diccionarioCursos[ide][0])

def actualizarNotas():
    alumnoMatriculado={'2': [[1, '-'], [4, '-']],'1':[[1,'-']],'3':[[2,'-'],[1,'-']]}
    #diccionarioCursos={'1':['Logica',23,'4','5',33,'1'],'2':['computacion',25,'4','5',33,'1'],'3':['Redes',23,'4','5',33,'2'],'4':['linux',23,'4','5',33,'2']}
    index =0
    for iden,value in alumnoMatriculado.items():
        identificacionEstudiante =(input("Identificacion del Estudiante : "))
        print('ESTUDIANTE CON IDENTIFICADOR ',identificacionEstudiante)
        cantidad=len(alumnoMatriculado[identificacionEstudiante])
        while(cantidad>0):
                print('CURSO ',alumnoMatriculado[identificacionEstudiante][index][0])
                nota =int(input("Nota del curso : "))
                alumnoMatriculado[identificacionEstudiante][index][1]=nota
                index +=1
                cantidad -=1
        print('Los Notas actualizadas son : ',alumnoMatriculado)
        index=0

def ListCursosEstCalificaciones():
    diccionarioAlumnos ={'1':['-','ninja'],'2':['-','paolo'],'3':['-','andy'],'4':['-','piter'],'5':['-','Alonso']}
    alumnoMatriculado={'2': [[1, 12], [4, 10]],'1':[[1,17]],'3':[[2,13],[1,17]]}
    diccionarioCursos={'1':['Logica',23,'4','5',33,'1'],'2':['computacion',25,'4','5',33,'1'],'3':['Redes',23,'4','5',33,'2'],'4':['linux',23,'4','5',33,'2']}

    identificacionEstudiante =(input("Identificacion del Estudiante : "))
    print("Alumno ",diccionarioAlumnos[identificacionEstudiante][1])
    print("\nCURSOS  -  NOTA")
    index =0
    for ide,value in alumnoMatriculado.items():
        if(identificacionEstudiante==ide):
            cantidad=len(alumnoMatriculado[identificacionEstudiante])
            while(cantidad>0):
                codigo= alumnoMatriculado[identificacionEstudiante][index][0]
                parse = str(codigo)
                print(diccionarioCursos[parse][0],' - ',alumnoMatriculado[identificacionEstudiante][index][1])
                index +=1
                cantidad -=1
            index=0

if __name__ == '__main__':
    #valor = 8
    #while valor > 0:
    ListCursosEstCalificaciones()
    #    valor = valor - 1


#!/usr/bin/env python
# coding: utf-8

# ### Data preprocessing

# In[1]:

import numpy as np
import matplotlib.pyplot as plt
from skimage import exposure
from LabelTrans_Regression import label_transform

images_npy = '../Datasets/images.npy'
labels_npy = '../Datasets/labels.npy'
images = np.load(images_npy)
labels = np.load(labels_npy)

# In[32]:

print(labels[0])

gamma_transfer = exposure.adjust_gamma(images[0], 3)

plt.imshow(gamma_transfer, cmap ='gray')

# In[30]:

plt.imshow(images[0], cmap ='gray')

# In[2]:

enhance_images = exposure.adjust_gamma(images, 3)

# ---

# ### Basic Model Shape

# In[8]:

from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPooling2D, Input, BatchNormalization
from keras import backend
import tensorflow as tf
import keras

model = Sequential()
model.add(Input(shape=(150, 150, 1)))
model.add(Conv2D(filters=32,
                    kernel_size=(3, 3),
                    activation='relu',
                    kernel_regularizer='l2'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(BatchNormalization())
model.add(Dropout(0.25))
model.add(Conv2D(filters=64,
                    kernel_size=(3, 3),
                    activation='relu',
                    kernel_regularizer='l2'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(BatchNormalization())
model.add(Dropout(0.25))
model.add(Conv2D(filters=128,
                    kernel_size=(3, 3),
                    activation='relu',
                    kernel_regularizer='l2'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(BatchNormalization())
model.add(Dropout(0.25))

model.add(Flatten())
model.add(Dense(units=256,
                activation='relu',
                kernel_regularizer='l2'))
model.add(Dense(units=128,
                activation='relu',
                kernel_regularizer='l2'))
model.add(Dense(units=1, activation='linear'))

model.summary()

# ---

# ### Regression

# In[3]:

# Enhance
import numpy as np
from sklearn.model_selection import train_test_split
from Regression import regression, preprocessing
import matplotlib.pyplot as plt

batch_size = 32
epochs = 100
lr = 0.001

images_npy = '../Datasets/images.npy'
labels_npy = '../Datasets/labels.npy'
# images = np.load(images_npy)
images = enhance_images
labels = np.load(labels_npy)

x_train, x_test, y_train, y_test = train_test_split(images, labels, train_size=0.8)
x_train, x_test, input_shape = preprocessing(x_train, x_test)
y_train = y_train[:, 0] + y_train[:, 1] / 60  # single output labels
y_test = y_test[:, 0] + y_test[:, 1] / 60

regression_history, regression_his = regression(x_train, y_train, x_test, y_test, batch_size, epochs, lr, input_shape)

# In[4]:

regression_x_test, regression_y_test = x_test, y_test

# In[5]:

# Training plot with enhance
train_loss = regression_his.history['loss']
train_mse = regression_his.history['custom_mae']
val_loss = regression_his.history['val_loss']
val_mse = regression_his.history['val_custom_mae']
epochs = range(1, len(train_loss) + 1)
plt.plot(epochs, train_loss, 'orange', label='train_loss')
plt.plot(epochs, train_mse, label='train_custom_mae')
plt.plot(epochs, val_loss, label='val_loss')
plt.plot(epochs, val_mse, label='val_custom_mae')
plt.legend()
plt.show()

# In[18]:

regression_pred = regression_history.predict(x_test)
print('min predict:', np.min(regression_pred), 'max predict:', np.max(regression_pred))
print("y_predict", '   ', 'y_test')
for i in range(5):
    print(regression_pred[i], ' ', y_test[i])

# In[26]:

regerss_sum = 0
for i in range(3600):
    regerss_sum += min(abs(regression_pred[i]%1 - regression_y_test[i]%1), abs(1 + regression_pred[i]%1 - regression_y_test[i]%1))
regerss_sum / 3600 * 60

# In[31]:

def sincos_mse(y_true, y_pred):
    return np.sqrt((np.sum(np.square(y_true - y_pred)* [3600, 3600, 1, 1]) ) / 3601)

regression_pred_minutes = regression_pred % 1
regression_pred_hours = regression_pred - regression_pred_minutes
regression_y_minutes = regression_y_test % 1
regression_Y_hours = regression_y_test - regression_y_minutes
trans_regression_pred = label_transform([regression_pred_hours, regression_pred_minutes])
trans_regression_y_test = label_transform([regression_Y_hours, regression_y_minutes])

sincos_mse(trans_regression_pred, trans_regression_y_test)

# In[32]:

# Without Enhance
import numpy as np
from sklearn.model_selection import train_test_split
from Regression import regression, preprocessing
import matplotlib.pyplot as plt

batch_size = 32
epochs = 100
lr = 0.001

images_npy = '../Datasets/images.npy'
labels_npy = '../Datasets/labels.npy'
images = np.load(images_npy)
# images = enhance_images
labels = np.load(labels_npy)

x_train, x_test, y_train, y_test = train_test_split(images, labels, train_size=0.8)
x_train, x_test, input_shape = preprocessing(x_train, x_test)
y_train = y_train[:, 0] + y_train[:, 1] / 60  # single output labels
y_test = y_test[:, 0] + y_test[:, 1] / 60

regression_w_history, regression_w_his = regression(x_train, y_train, x_test, y_test, batch_size, epochs, lr, input_shape)

# In[33]:

regression_x_w_test, regression_y_w_test = x_test, y_test

# In[58]:

# Training plot without enhance
w_eH = regression_w_his
w_eHis = regression_w_history
train_loss = w_eH.history['loss']
train_mse = w_eH.history['custom_mae']
val_loss = w_eH.history['val_loss']
val_mse = w_eH.history['val_custom_mae']
epochs = range(1, len(train_loss) + 1)
plt.plot(epochs, train_loss, 'orange', label='train_loss')
plt.plot(epochs, train_mse, label='train_custom_mae')
plt.plot(epochs, val_loss, label='val_loss')
plt.plot(epochs, val_mse, label='val_custom_mae')
plt.legend()
plt.show()

# ---

# ### Classfication

# In[4]:

from keras.utils.np_utils import to_categorical
from sklearn.model_selection import train_test_split
import numpy as np
import matplotlib.pyplot as plt
from Classfication import classfication, preprocessing

classes_num = 12 * 2
batch_size = 128
epochs = 100
lr = 0.001

images_npy = '../Datasets/images.npy'
labels_npy = '../Datasets/labels.npy'

# images = np.load(images_npy)
images = enhance_images
labels = np.load(labels_npy)
x_train, x_test, y_train, y_test = train_test_split(images, labels, train_size=0.8)

x_train, x_test, input_shape = preprocessing(x_train, x_test)
y_train = y_train[:, 0] * 2 + (y_train[:, 1] * 2 / 60).astype('int32')
y_test = y_test[:, 0] * 2 + (y_test[:, 1] * 2 / 60).astype('int32')

y_train_trans2class = to_categorical(y_train, classes_num)
y_test_trans2class = to_categorical(y_test, classes_num)

classfication_history, classfication_his = classfication(x_train, y_train_trans2class, x_test, y_test_trans2class, batch_size, epochs, lr, classes_num, input_shape)

# In[5]:

acc = classfication_his.history['categorical_accuracy']
loss = classfication_his.history['loss']
accVal = classfication_his.history['val_categorical_accuracy']
lossVal = classfication_his.history['val_loss']
epochs = range(1, len(loss) + 1)

# plt.title('Accuracy and Loss')
# fig = plt.figure()
ax1 = plt.subplot()
ax2 = ax1.twinx()
ax1.set_ylabel('Loss', fontsize=15)
ax2.set_ylabel('Accuracy', fontsize=15)
ax1.set_xlabel('epochs', fontsize=15)
L1, = ax2.plot(epochs, acc, 'blue', label='Training Accuracy')
L2, = ax1.plot(epochs, loss, 'green', label='Training Loss')
L3, = ax2.plot(epochs, accVal, 'red', label='Test Accuracy')
L4, = ax1.plot(epochs, lossVal, 'orange', label='Test Loss')
plt.legend(handles = [L1, L2, L3, L4], loc = 'upper left', fontsize=10)

# #### 720 categories

# In[25]:

from keras.utils.np_utils import to_categorical
from sklearn.model_selection import train_test_split
import numpy as np
import matplotlib.pyplot as plt
from Classfication import classfication, preprocessing

classes_num = 720
batch_size = 128
epochs = 100
lr = 0.001

images_npy = '../Datasets/images.npy'
labels_npy = '../Datasets/labels.npy'

# images = np.load(images_npy)
images = enhance_images
labels = np.load(labels_npy)
x_train, x_test, y_train, y_test = train_test_split(images, labels, train_size=0.8)

x_train, x_test, input_shape = preprocessing(x_train, x_test)
y_train = y_train[:, 0] * 60  + (y_train[:, 1]).astype('int32')
y_test = y_test[:, 0] * 60 + (y_test[:, 1]).astype('int32')

y_train_trans2class = to_categorical(y_train, classes_num)
y_test_trans2class = to_categorical(y_test, classes_num)

history, his = classfication(x_train, y_train_trans2class, x_test, y_test_trans2class, batch_size, epochs, lr, classes_num, input_shape)

# In[26]:

acc = his.history['categorical_accuracy']
loss = his.history['loss']
accVal = his.history['val_categorical_accuracy']
lossVal = his.history['val_loss']
epochs = range(1, len(loss) + 1)

# plt.title('Accuracy and Loss')
# fig = plt.figure()
ax1 = plt.subplot()
ax2 = ax1.twinx()
ax1.set_ylabel('Loss', fontsize=15)
ax2.set_ylabel('Accuracy', fontsize=15)
ax1.set_xlabel('epochs', fontsize=15)
L1, = ax2.plot(epochs, acc, 'blue', label='Training Accuracy')
L2, = ax1.plot(epochs, loss, 'green', label='Training Loss')
L3, = ax2.plot(epochs, accVal, 'red', label='Test Accuracy')
L4, = ax1.plot(epochs, lossVal, 'orange', label='Test Loss')
plt.legend(handles = [L1, L2, L3, L4], loc = 'upper left', fontsize=10)

# ### Multi head
#

# In[6]:

from keras.utils.np_utils import to_categorical
from sklearn.model_selection import train_test_split
import numpy as np
import matplotlib.pyplot as plt
from MultiheadModel import multi_classfication, preprocessing, minutes_transform

batch_size = 128
epochs = 100
lr = 0.001
classes_num = 12

images_npy = '../Datasets/images.npy'
labels_npy = '../Datasets/labels.npy'

# images = np.load(images_npy)
images = enhance_images
labels = np.load(labels_npy)
x_train, x_test, y_train, y_test = train_test_split(images, labels, train_size=0.8)
x_train, x_test, input_shape = preprocessing(x_train, x_test)

hours_train = y_train[:,0]
minutes_train = y_train[:,1]
hours_test = y_test[:,0]
minutes_test = y_test[:,1]

y_train_hours = to_categorical(hours_train, classes_num)
y_train_mintues = minutes_train / 60
y_test_hours = to_categorical(hours_test, classes_num)
y_test_mintues = minutes_test  / 60
# y_train_mintues = minutes_transform(minutes_train)
# y_test_mintues = minutes_transform(minutes_test)

y_train = [y_train_hours, y_train_mintues]
y_test = [y_test_hours, y_test_mintues]

history, his = multi_classfication(x_train, y_train, x_test, y_test, batch_size, epochs, lr, classes_num, input_shape)

# In[29]:

minutes_mse = his.history['minutes_mse'][1:]
minutes_loss = his.history['minutes_loss'][1:]
Vminutes_mse = his.history['val_minutes_mse'][1:]
Vminutes_loss = his.history['val_minutes_loss'][1:]
epochs = range(1, len(minutes_mse) + 1)
ax1 = plt.subplot()
ax1.set_ylabel('Loss', fontsize=15)
ax1.set_xlabel('epochs', fontsize=15)
ax1.plot(epochs, minutes_mse, 'orange', label='minutes_mse')
ax1.plot(epochs, minutes_loss, label='minutes_loss')
ax1.plot(epochs, Vminutes_mse, label='val_minutes_mse')
ax1.plot(epochs, Vminutes_loss, label='val_minutes_loss')
plt.legend()
plt.show()

# In[30]:

hours_acc = his.history['hours_categorical_accuracy']
hours_loss = his.history['hours_loss']

Vhours_acc = his.history['val_hours_categorical_accuracy']
Vhours_loss = his.history['val_hours_loss']
epochs = range(1, len(hours_loss) + 1)

# plt.title('Accuracy and Loss')
# fig = plt.figure()
ax1 = plt.subplot()
ax2 = ax1.twinx()
ax1.set_ylabel('Loss', fontsize=15)
ax2.set_ylabel('Accuracy', fontsize=15)
ax1.set_xlabel('epochs', fontsize=15)
L1, = ax2.plot(epochs, hours_acc, 'blue', label='hours_categorical_accuracy')
L2, = ax1.plot(epochs, hours_loss, 'green', label='hours_loss')
L3, = ax2.plot(epochs, Vhours_acc, 'red', label='val_hours_categorical_accuracy')
L4, = ax1.plot(epochs, Vhours_loss, 'orange', label='val_hours_loss')
plt.legend(handles = [L1, L2, L3, L4], loc = 'right', fontsize=10)

# In[9]:

pred = history.predict(x_test)

# In[10]:

print("y_predict", '   ', 'y_test')
for i in range(100):
    print(np.argmax(pred[0][i]), ' ', np.argmax(y_test[0][i]))

# In[11]:

# minutes difference and average minutes difference
sum = 0
for i in range(len(pred[1])):
    sum += abs(pred[1][i] - y_test[1][i])
    print(pred[1][i] , ' ', y_test[1][i] )

sum / len(pred[1]) * 60

# In[33]:

pred_hours = np.array([np.argmax(pred[0][i]) for i in range(3600)])
pred_minutes = np.array([pred[1][i] for i in range(3600)])
y_test_hours = np.array([np.argmax(y_test[0][i]) for i in range(3600)])
y_test_minutes = np.array([y_test[1][i] for i in range(3600)])

# In[34]:

def sincos_mse(y_true, y_pred):
    return np.sqrt((np.sum(np.square(y_true - y_pred)* [3600, 3600, 1, 1]) ) / 3601)

trans_pred = label_transform([pred_hours, pred_minutes])
trans_y_test = label_transform([y_test_hours, y_test_minutes])

sincos_mse(trans_pred, trans_y_test)

# ### Label trans regression

# In[35]:

from keras.utils.np_utils import to_categorical
from sklearn.model_selection import train_test_split
import numpy as np
import matplotlib.pyplot as plt
from LabelTrans_Regression import label_trans_regression, preprocessing, label_transform

batch_size = 32
epochs = 100
lr = 0.001

images_npy = '../Datasets/images.npy'
labels_npy = '../Datasets/labels.npy'

# images = np.load(images_npy)
images = enhance_images
labels = np.load(labels_npy)

# labels_trans = label_transform(labels)

x_train, x_test, y_train, y_test = train_test_split(images, labels, train_size=0.8)

y_o_train = y_train
y_o_test = y_test

y_train = label_transform(y_train)
y_test = label_transform(y_test)

x_train, x_test, input_shape = preprocessing(x_train, x_test)

label_trans_regression_history, label_trans_regression_his = label_trans_regression(x_train, y_train, x_test, y_test, batch_size, epochs, input_shape)

# In[42]:

train_loss = label_trans_regression_his.history['loss']
train_mse = label_trans_regression_his.history['mae']
val_loss = label_trans_regression_his.history['val_loss']
val_mse = label_trans_regression_his.history['val_mae']
epochs = range(1, len(train_loss) + 1)
plt.plot(epochs, train_loss, 'orange', label='train_loss')
plt.plot(epochs, train_mse, label='train_mse')
plt.plot(epochs, val_loss, label='val_loss')
plt.plot(epochs, val_mse, label='val_mse')
plt.legend()
plt.show()

# In[44]:

label_trans_pred = label_trans_regression_history.predict(x_test)

# In[56]:

def sincos_mse(y_true, y_pred):
    return np.sqrt((np.sum(np.square(y_true - y_pred)* [3600, 3600, 1, 1]) ) / 3601)

# In[57]:

sincos_mse(label_trans_pred, y_test)

# ### re-Label trans
# return real clock from sin&cos format

# In[47]:

def label_transform_map(h_list, m_list):
    hours_array = np.zeros(shape=(len(h_list), 2))
    minutes_array = np.zeros(shape=(len(m_list), 2))
    for i in range(len(h_list)):
        hour = h_list[i]
        hours_array[i] = (np.sin(hour*(1/6)*np.pi),np.cos(hour*(1/6)*np.pi))

    for i in range(len(m_list)):
        minute = m_list[i]
        minutes_array[i] = (np.sin(minute * (1/3) * np.pi), np.cos(minute * (1/3)*np.pi))

    return hours_array, minutes_array

# In[48]:

def distance(v1, v2):
    dis_array = np.zeros(shape=(len(v2)))
    for i in range(len(v2)):
        dis_array[i] = float(np.sqrt(sum((v1 - v2[i])**2)))
    return dis_array

# In[49]:

hours_list = np.arange(0, 12).astype('float32')
minutes_list = np.arange(0, 60).astype('float32')

h_map, m_map = label_transform_map(hours_list, minutes_list)

# In[51]:

pred = label_trans_pred
hours = pred[:, 0:2]
minutes = pred[:, 2:]

def real_clock(hours, minutes):
    length = len(hours)
    result_list = np.zeros(shape=(length, 2))
    for i in range(length):
        hours_dis = distance(hours[i], h_map)
        minutes_dis = distance(minutes[i], m_map)

        h_min = np.argsort(hours_dis)[0:2]
        m_min = np.argsort(minutes_dis)[0]
        if (h_min[0] != 0) | (h_min[1] != 11):
            if m_min > 30:
                result_list[i] = (h_min[0]-1, m_min)
            else:
                result_list[i] = (h_min[0], m_min)
        else:
            if h_min[0] == 0:
                if m_min > 30:
                    result_list[i] = (11, m_min)
                else:
                    result_list[i] = (0, m_min)
            if h_min[0] == 11:
                if m_min > 30:
                    result_list[i] = (10, m_min)
                else:
                    result_list[i] = (11, m_min)
    return result_list

# In[55]:

result_list = real_clock(hours, minutes)
true_result = real_clock(y_test[:, 0:2], y_test[:, 2:])

# In[131]:

sum = 0
for i in range(3600):
    sum += min(abs(result_list[i, 1] - y_o_test[i, 1]), abs(60 + result_list[i, 1] - y_o_test[i, 1]))

# In[132]:

sum/3600

# In[119]:

for i in range(100):
    print(result_list[i], ' ', y_o_test[i])

# ---

# ### Other attempts

# In[46]:

# two classification Multi-head

from keras.utils.np_utils import to_categorical
from sklearn.model_selection import train_test_split
import numpy as np
import matplotlib.pyplot as plt
from MultiheadModelAlex import multi_classficationAlex, preprocessing, minutes_transform
from skimage import exposure

batch_size = 128
epochs = 100
lr = 0.001
classes_num = 30

images_npy = '../Datasets/images.npy'
labels_npy = '../Datasets/labels.npy'

images = np.load(images_npy)
enhance_images = exposure.adjust_gamma(images, 3)
images = enhance_images
labels = np.load(labels_npy)
x_train, x_test, y_train, y_test = train_test_split(images, labels, train_size=0.8)
x_train, x_test, input_shape = preprocessing(x_train, x_test)

hours_train = y_train[:,0]
minutes_train = y_train[:,1]
hours_test = y_test[:,0]
minutes_test = y_test[:,1]

minutes_train = (minutes_train / 2).astype('int32')
minutes_test = (minutes_test / 2).astype('int32')

y_train_hours = to_categorical(hours_train, 12)
y_test_hours = to_categorical(hours_test, 12)
y_train_mintues = to_categorical(minutes_train, classes_num)
y_test_mintues = to_categorical(minutes_test, classes_num)

y_train = [y_train_mintues, y_train_hours]
y_test = [y_test_mintues, y_test_hours]

history, his = multi_classficationAlex(x_train, y_train, x_test, y_test, batch_size, epochs, lr, classes_num, input_shape)

# In[2]:

# different fully connected layer

from keras.utils.np_utils import to_categorical
from sklearn.model_selection import train_test_split
import numpy as np
import matplotlib.pyplot as plt
from MultiheadModelBeta import multi_classficationBeta, preprocessing, minutes_transform

batch_size = 128
epochs = 100
lr = 0.001
classes_num = 12

images_npy = '../Datasets/images.npy'
labels_npy = '../Datasets/labels.npy'

# images = np.load(images_npy)
images = enhance_images
labels = np.load(labels_npy)
x_train, x_test, y_train, y_test = train_test_split(images, labels, train_size=0.8)
x_train, x_test, input_shape = preprocessing(x_train, x_test)

hours_train = y_train[:,0]
minutes_train = y_train[:,1]
hours_test = y_test[:,0]
minutes_test = y_test[:,1]

y_train_hours = to_categorical(hours_train, classes_num)
y_train_mintues = minutes_train / 60
y_test_hours = to_categorical(hours_test, classes_num)
y_test_mintues = minutes_test  / 60
# y_train_mintues = minutes_transform(minutes_train)
# y_test_mintues = minutes_transform(minutes_test)

y_train = [y_train_hours, y_train_mintues]
y_test = [y_test_hours, y_test_mintues]

history, his = multi_classfication(x_train, y_train, x_test, y_test, batch_size, epochs, lr, classes_num, input_shape)

# ---

# ### Model Structure

# In[3]:

# Multi head model
from keras.models import Model
from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPooling2D, Input, BatchNormalization
from keras import backend
import tensorflow as tf
import keras

input = Input(shape=(150, 150, 1))

conv1 = Conv2D(32, (3, 3),
                activation='relu',
                kernel_regularizer='l2')(input)
pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
normal1 = BatchNormalization()(pool1)
drop1 = Dropout(0.25)(normal1)

conv3 = Conv2D(64, (3, 3),
                activation='relu',
                kernel_regularizer='l2')(drop1)
pool2 = MaxPooling2D(pool_size=(2, 2))(conv3)
normal2 = BatchNormalization()(pool2)
drop2 = Dropout(0.25)(normal2)

conv3 = Conv2D(128, (3, 3),
                activation='relu',
                kernel_regularizer='l2')(drop2)
pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)
normal3 = BatchNormalization()(pool3)
drop3 = Dropout(0.25)(normal3)

flatten = Flatten()(drop3)
Dense1 = Dense(256,
                activation='relu',
                kernel_regularizer='l2')(flatten)
Dense2 = Dense(128,
            activation='relu',
            kernel_regularizer='l2')(Dense1)
output_class = Dense(12, activation='softmax', name='hours')(Dense2)
# Regression
output_regress = Dense(1, activation='linear', name='minutes')(Dense2)
multi_model = Model(inputs = input, outputs = [output_class, output_regress])
multi_model.summary()


from django.db import models
from django.core.validators import MinLengthValidator, MinValueValidator

from main_app.managers import ProfileManager

class TimeStampModel(models.Model):
    class Meta:
        abstract = True

    creation_date = models.DateTimeField(auto_now_add=True, )

class Profile(TimeStampModel):
    full_name = models.CharField(
        max_length=100,
        validators=[
            MinLengthValidator(2),
        ]
    )
    email = models.EmailField()
    phone_number = models.CharField(max_length=15)
    address = models.TextField()
    is_active = models.BooleanField(default=True, )

    objects = ProfileManager()

    def __str__(self):
        return f'Profile name: {self.full_name}'

class Product(TimeStampModel):
    name = models.CharField(max_length=100, )
    description = models.TextField()
    price = models.DecimalField(
        max_digits=10,
        decimal_places=2,
        validators=[
            MinValueValidator(0.01)
        ]
    )
    in_stock = models.PositiveIntegerField()
    is_available = models.BooleanField(default=True, )

    def __str__(self):
        return f'Product name: {self.name}'

class Order(TimeStampModel):
    profile = models.ForeignKey(
        to=Profile,
        on_delete=models.CASCADE,
        related_name='orders'
    )
    products = models.ManyToManyField(
        to=Product,
        related_name='products_orders'
    )
    total_price = models.DecimalField(
        max_digits=10,
        decimal_places=2,
        validators=[
            MinValueValidator(0.01)
        ]
    )
    is_completed = models.BooleanField(default=False, )

    def __str__(self):
        return f'Order number: {self.id}'


from recipe_engine.recipe_api import Property

DEPS = [
    'depot_tools/git',
    'fuchsia/display_util',
    'fuchsia/upload',
    'recipe_engine/buildbucket',
    'recipe_engine/cipd',
    'recipe_engine/context',
    'recipe_engine/file',
    'recipe_engine/json',
    'recipe_engine/path',
    'recipe_engine/properties',
    'recipe_engine/step',
    'recipe_engine/swarming',
    'flutter/json_util',
    'repo_util',
    'yaml',
]

# This recipe builds the fuchsia_ctl CIPD package and tests it against the Linux
# Fuchsia builder.
def RunSteps(api):
  packages_dir = api.path['start_dir'].join('packages')
  packages_git_rev = api.repo_util.checkout(
      'packages',
      packages_dir,
      api.properties.get('git_url'),
      api.properties.get('git_ref'),
  )
  # Validates packages builders json format.
  api.json_util.validate_json(packages_dir)

  # Build and uploads a new version of the fuchsia_ctl CIPD package.
  fuchsia_ctl_path = packages_dir.join('packages', 'fuchsia_ctl')
  with api.context(cwd=fuchsia_ctl_path):
    api.step('run tool/build.sh', cmd=['tool/build.sh'])

  cipd_package_name = 'flutter/fuchsia_ctl/${platform}'
  cipd_zip_path = 'fuchsia_ctl.zip'
  api.cipd.build(
      fuchsia_ctl_path.join('build'), cipd_zip_path, cipd_package_name)
  pin = api.cipd.register(cipd_package_name, cipd_zip_path)

  # Test the new fuchsia_ctl CIPD package with a Linux Fuchsia build.
  props = {'fuchsia_ctl_version': pin.instance_id}
  builds = ScheduleBuilds(api, 'Linux Fuchsia', props)
  builds = CollectBuilds(api, builds)
  api.display_util.display_builds(
      step_name='display builds',
      builds=builds.values(),
      raise_on_failure=True,
  )

def ScheduleBuilds(api, builder_name, drone_props):
  req = api.buildbucket.schedule_request(
      swarming_parent_run_id=api.swarming.task_id,
      builder=builder_name,
      properties=drone_props)
  return api.buildbucket.schedule([req])

def CollectBuilds(api, builds):
  return api.buildbucket.collect_builds([build.id for build in builds],
                                        mirror_status=True)

def GenTests(api):
  yield api.test(
      'demo',
      api.properties(
          git_ref='refs/pull/123/head', git_url='https://abc.com/repo'),
  )


def productExceptSelf(nums):
    n = len(nums)
    lproducts = [1 for _ in range(n)]
    rproducts = [1 for _ in range(n)]
    for i in range(1, len(nums)):
        lproducts[i] = lproducts[i-1] * nums[i-1]
    for i in range(len(nums)-2, -1, -1):
        rproducts[i] = rproducts[i+1] * nums[i+1]
    ans = []
    for i in range(n):
        ans.append(lproducts[i] * rproducts[i])
    return ans


import math
import torch
import torch.nn as nn
import torch.nn.functional as F

class ResBlock(nn.Module):
    def __init__(self, nFin, nFout):
        super(ResBlock, self).__init__()
        self.conv_block = nn.Sequential()
        self.conv_block.add_module('BNorm1', nn.BatchNorm2d(nFin))
        self.conv_block.add_module('LRelu1', nn.LeakyReLU(0.1))
        self.conv_block.add_module('ConvL1',
            nn.Conv2d(nFin,  nFout, kernel_size=3, padding=1, bias=False))
        self.conv_block.add_module('BNorm2', nn.BatchNorm2d(nFout))
        self.conv_block.add_module('LRelu2', nn.LeakyReLU(0.1))
        self.conv_block.add_module('ConvL2',
            nn.Conv2d(nFout, nFout, kernel_size=3, padding=1, bias=False))
        self.conv_block.add_module('BNorm3', nn.BatchNorm2d(nFout))
        self.conv_block.add_module('LRelu3', nn.LeakyReLU(0.1))
        self.conv_block.add_module('ConvL3',
            nn.Conv2d(nFout, nFout, kernel_size=3, padding=1, bias=False))

        self.skip_layer = nn.Conv2d(nFin, nFout, kernel_size=1, stride=1)

    def forward(self, x):
        return self.skip_layer(x) + self.conv_block(x)

class ResNetLike(nn.Module):
    def __init__(self, opt):
        super(ResNetLike, self).__init__()
        self.in_planes = opt['in_planes']
        self.out_planes = [64, 96, 128, 256]
        self.num_stages = 4

        if type(self.out_planes) == int:
            self.out_planes = [self.out_planes for i in range(self.num_stages)]
        assert(type(self.out_planes)==list)
        assert(len(self.out_planes)==self.num_stages)
        num_planes = [self.out_planes[0],] + self.out_planes
        userelu = opt['userelu'] if ('userelu' in opt) else False
        dropout = opt['dropout'] if ('dropout' in opt) else 0

        self.feat_extractor = nn.Sequential()
        self.feat_extractor.add_module('ConvL0',
            nn.Conv2d(self.in_planes, num_planes[0], kernel_size=3, padding=1))
        for i in range(self.num_stages):
            self.feat_extractor.add_module('ResBlock'+str(i),
                ResBlock(num_planes[i], num_planes[i+1]))
            self.feat_extractor.add_module('MaxPool'+str(i),
                nn.MaxPool2d(kernel_size=2,stride=2,padding=0))

        self.feat_extractor.add_module('AvgPool', nn.AdaptiveAvgPool2d(1))
        self.feat_extractor.add_module('BNormF1',
            nn.BatchNorm2d(num_planes[-1]))
        self.feat_extractor.add_module('ReluF1', nn.ReLU(inplace=True))
        self.feat_extractor.add_module('ConvLF1',
            nn.Conv2d(num_planes[-1], 384, kernel_size=1))
        if dropout>0.0:
            self.feat_extractor.add_module('DropoutF1',
                nn.Dropout(p=dropout, inplace=False))

        self.feat_extractor.add_module('BNormF2', nn.BatchNorm2d(384))
        self.feat_extractor.add_module('ReluF2', nn.ReLU(inplace=True))
        self.feat_extractor.add_module('ConvLF2',
            nn.Conv2d(384, 512, kernel_size=1))
        self.feat_extractor.add_module('BNormF3', nn.BatchNorm2d(512))
        if dropout>0.0:
            self.feat_extractor.add_module('DropoutF2',
                nn.Dropout(p=dropout, inplace=False))

        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                m.weight.data.normal_(0, math.sqrt(2. / n))
            elif isinstance(m, nn.BatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()

    def forward(self, x):
        out = self.feat_extractor(x)
        return out.view(out.size(0),-1)

def create_model(opt):
    return ResNetLike(opt)


import webbrowser

from PyQt5.QtGui import *
from PyQt5.QtCore import *
from PyQt5.QtWidgets import *
from PyQt5.QtPrintSupport import *

from PyUi import Ui_MainWindow
from Utils.digital_image import DigitalImage
from PyUi import Ui_About_App, Ui_About_Authors
from PyForm import EdgeParam, GaussParam, ThresholdParam

class MainWindow(QMainWindow, Ui_MainWindow):

    def __init__(self):
        QMainWindow.__init__(self)
        self.setupUi(self)

        self.image = None
        self.imagesTree = dict()
        self.timecount = 0
        self.timer = QTimer(self)
        self.timer.timeout.connect(self.TimerAction)

        self.menuFile.triggered[QAction].connect(self.MenuFile)
        self.menuFilter.triggered[QAction].connect(self.MenuFilter)
        self.menuOptions.triggered[QAction].connect(self.MenuOptions)
        self.menuView.triggered[QAction].connect(self.MenuView)
        self.treeWidget.clicked.connect(self.SelectItemAction)

        self.pushButton_Plus.clicked.connect(self.EnlargeAction)
        self.pushButton_Minus.clicked.connect(self.ShrinkAction)
        self.pushButton_Minus.clicked.connect(self.ShrinkAction)
        self.pushButton_Plus.setHidden(True)
        self.pushButton_Minus.setHidden(True)

        self.scrollArea.setWidget(self.label)
        self.treeWidget.expanded.connect(self.ExpandAction)

    def ExpandAction(self):
        self.treeWidget.resizeColumnToContents(0)

    def MenuView(self, triggered):
        if triggered.text() == 'Image Tree':
            self.treeWidget.setHidden(not self.actionImagesTree.isChecked())
        if triggered.text() == 'Table':
            self.tableWidget.setHidden(not self.actionTable.isChecked())
        self.resizeEvent(None)

    def MenuOptions(self, triggered):
        if triggered.text() == 'Help':
            self.HelpAction()
        if triggered.text() == 'Orientation':
            self.OrientationAction()
        if triggered.text() == 'Report':
            self.ReportAction()
        if triggered.text() == 'About the Authors':
            self.AboutAuthorsAction()
        if triggered.text() == 'About the App':
            self.AboutAppAction()

    def HelpAction(self):
        pass

    def OrientationAction(self):
        path = '../doc/Bordes Imagenes Medicas.pdf'
        webbrowser.open_new(path)

    def ReportAction(self):
        pass

    def AboutAuthorsAction(self):
        dialog = QDialog()
        ui_dialog = Ui_About_Authors()
        ui_dialog.setupUi(dialog)
        dialog.exec()

    def AboutAppAction(self):
        dialog = QDialog()
        ui_dialog = Ui_About_App()
        ui_dialog.setupUi(dialog)
        dialog.exec()

    def MenuFilter(self, triggered):
        if triggered.text() == 'Edges':
            self.EdgesAction()
        if triggered.text() == 'Gaussian':
            self.GaussAction()
        if triggered.text() == 'Threshold':
            self.ThresholdAction()

    def ThresholdAction(self):
        thresh = ThresholdParam()
        thresh.exec_()

        if not thresh.type:
            return
        image = self.image.ThresholdFilter(thresh)

        self.imagesTree[image.GetName] = image
        image.widget = QTreeWidgetItem(self.image.widget, [image.GetName])
        self.image = image
        self.UpdateFrame

    def GaussAction(self):
        gauss = GaussParam()
        gauss.exec_()

        if not gauss.sigma:
            return
        image = self.image.GaussFilter(gauss)

        self.imagesTree[image.GetName] = image
        image.widget = QTreeWidgetItem(self.image.widget, [image.GetName])
        self.image = image
        self.UpdateFrame

    def EdgesAction(self):
        edges = EdgeParam()
        edges.exec_()

        if not edges.filter:
            return
        image = self.image.EdgesFilter(edges)

        self.imagesTree[image.GetName] = image
        image.widget = QTreeWidgetItem(self.image.widget, [image.GetName])
        self.image = image
        self.UpdateFrame

    def TimerAction(self):
        self.timecount += 1
        if self.timecount > 50000:
            self.pushButton_Plus.setHidden(True)
            self.pushButton_Minus.setHidden(True)
            self.timer.stop()
            self.timecount = 0

    def mouseMoveEvent(self, event):
        if self.image and not self.timecount :
            self.pushButton_Plus.setHidden(False)
            self.pushButton_Minus.setHidden(False)
            self.timer.start()
        elif self.image:
            self.timecount = 0

    def resizeEvent(self, event):
        table = (0,self.height()-44-80,self.width(),80 if self.actionTable.isChecked() else 0)
        tree = (0,0,self.width()/4 if self.actionImagesTree.isChecked() else 0, self.height()-44-table[3])
        label = (tree[2], 0, self.width()-tree[2], self.height()-44-table[3])

        self.treeWidget.setGeometry(*tree)
        self.label.setGeometry(*label)
        self.scrollArea.setGeometry(*label)
        self.pushButton_Plus.setGeometry(label[0] + 15, 15, 25, 25)
        self.pushButton_Minus.setGeometry(label[0] + 45, 15, 25, 25)
        self.tableWidget.setGeometry(*table)
        self.tableWidget.horizontalHeader().setDefaultSectionSize(self.width()/3-4)

    def MenuFile(self, triggered):
        if triggered.text() == 'Load Image':
            self.LoadImageAction()
        if triggered.text() == 'Remove Image':
            self.RemoveImageAction()
        if triggered.text() == 'Save Image':
            self.SaveAction()
        if triggered.text() == 'Save Images in Group':
            self.SaveGroupAction()
        if triggered.text() == 'Exit':
            self.ExitAction()

    def LoadImageAction(self):
        path = QFileDialog.getOpenFileName(self, directory='../img/', caption='Load Image', filter='Image File (*.jpg  *.png)')[0]

        if not path:
            return
        self.image = DigitalImage(path)

        try:
            self.imagesTree[self.image.GetName]
        except KeyError:
            self.imagesTree[self.image.GetName] = self.image
            self.image.widget = QTreeWidgetItem(self.treeWidget, [self.image.GetName])

        self.UpdateFrame
        self.SetEnable(True)

    def RemoveImageAction(self):
        MainWindow.RemoveChild(self.imagesTree, self.image)

        if self.image.parent:
            self.image.parent.widget.removeChild(self.image.widget)
            self.image = self.image.parent
        else:
            self.treeWidget.invisibleRootItem().removeChild(self.image.widget)
            self.SetEnable(False)
            self.image = None

        self.UpdateFrame

    def SelectItemAction(self):
        imagename =  self.treeWidget.currentItem().text(0)
        if imagename == self.image.GetName:
            return

        self.image = self.imagesTree[imagename]
        self.UpdateFrame

    def SaveAction(self):
        path = QFileDialog.getExistingDirectory(self, directory='../img/', caption='Save Image')

        if not path:
            return
        self.image.SaveImage(path)

    def SaveGroupAction(self):
        path = QFileDialog.getExistingDirectory(self, directory='../img/', caption='Save Images in Group')

        if not path:
            return
        self.image.SaveGroupImage(path)

    def EnlargeAction(self):
        bixmap = QPixmap(f'.temp/{self.image.filename}')

        self.image.width += 100
        self.image.height += 100

        self.timecount = 0
        self.label.setPixmap(bixmap.scaled(self.image.width, self.image.height, Qt.KeepAspectRatio))

    def ShrinkAction(self):
        bixmap = QPixmap(f'.temp/{self.image.filename}')

        self.image.width -= 100
        self.image.height -= 100

        self.timecount = 0
        self.label.setPixmap(bixmap.scaled(self.image.width, self.image.height, Qt.KeepAspectRatio))

    def ExitAction(self):
        self.close()

    def SetEnable(self, boolean):
        self.actionSave.setEnabled(boolean)
        self.actionSave_Group.setEnabled(boolean)
        self.actionRemove_Image.setEnabled(boolean)
        self.actionEdges.setEnabled(boolean)
        self.actionGaussian.setEnabled(boolean)
        self.actionThreshold.setEnabled(boolean)

    @property
    def UpdateFrame(self):
        if self.image:
            self.label.setPixmap(QPixmap(f'.temp/{self.image.filename}'))
            measure = self.image.QualityMeasures
            self.tableWidget.item(0, 0).setText(f'{measure[0]}')
            self.tableWidget.item(0, 1).setText(f'{measure[1]}')
            self.tableWidget.item(0, 2).setText(f'{measure[2]}')
            self.image.widget.setStatusTip(0, f'RMSE:{measure[0]}\t\t\tPSNR:{measure[1]}\t\t\tSNR:{measure[2]}\t')
            self.label.setStatusTip(f'RMSE:{measure[0]}\t\t\tPSNR:{measure[1]}\t\t\tSNR:{measure[2]}\t')
        else:
            self.label.setPixmap(QPixmap())

    @staticmethod
    def RemoveChild(imagesTree, image):
        imagesTree.pop(image.GetName)

        for item in image.child:
            MainWindow.RemoveChild(imagesTree, item)


from time import sleep

def printaLinha():
    print('=-'*20)

def apresenta():
    printaLinha()
    print('Contagem de 1 até 10 de 1 em 1: ')
    for c in range(1,11):
        sleep(0.4)
        print(c, end=' ',flush=True)
    print()
    printaLinha()
    print('Contagem de 10 até 0 de 2 em 2: ')
    for c in range(10,-1,-2):
        sleep(0.4)
        print(c, end=' ',flush=True)
    print()
    printaLinha()

def contador(i,f,p):
    if i > f:
        if p > 0:
            p = -p
        elif p == 0:
            p = -1
        for c in range(i,f-1,p):
            sleep(0.4)
            print(c, end=' ',flush=True)
    else:
        if p == 0:
            p = 1
        for c in range(i,f+1,p):
            sleep(0.4)
            print(c, end=' ',flush=True)

apresenta()

print('Agora é sua vez')
i = int(input('Digite onde começará a contagem: '))
f = int(input('Digite aonde terminará a contagem: '))
p = int(input('Digite de quanto em quanto a contagem será feita: '))
contador(i,f,p)