import sys
import traceback
import zipfile
from array import *
import re as regEx
import random
import socket
import struct
import ipaddress
import subprocess
import os
from datetime import datetime
import time
import logging

# Regex strings for all it should search for in the files
ipv4Pattern = regEx.compile(r'(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9][0-9]|[0-9])\.(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]'
                            r'[0-9]|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9][0-9]|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|'
                            r'1[0-9]{2}|[1-9][0-9]|[0-9])')

ipv6Pattern = regEx.compile(r'(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:)'
                            r'{1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]'
                            r'{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|'
                            r'([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4})'
                            r'{1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::'
                            r'(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|'
                            r'(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9])'
                            r'{0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))')

imsiPattern = regEx.compile(r'[0-9]{13,17}')

imsiHexPattern = regEx.compile(r'(IMSI :   { |IMSI : )(\d{2}\ |\d[A-Z]\ |[A-Z]\d\ )(\d{2}\ |\d[A-Z]\ |[A-Z]\d\ )'
                               r'(\d{2}\ |\d[A-Z]\ |[A-Z]\d\ )(\d{2}\ |\d[A-Z]\ |[A-Z]\d\ )(\d{2}\ |\d[A-Z]\ |'
                               r'[A-Z]\d\ )(\d{2}\ |\d[A-Z]\ |[A-Z]\d\ )(\d{2}\ |\d[A-Z]\ |[A-Z]\d\ )(\d{2}\ |'
                               r'\d[A-Z]\ |[A-Z]\d\ )(\d{2}|\d[A-Z]|[A-Z]\d)')

macPattern = regEx.compile(r'(?:[0-9A-Fa-f]{2}[:-]){5}(?:[0-9A-Fa-f]{2})')

hostnamePattern = regEx.compile(r'(?i)epcgw')

usernamePattern = regEx.compile(r'(?i)serviceuser')

urlPattern = regEx.compile(r'(?i)splunk')

hostnameMatches = {}

usernameMatches = {}

urlMatches = {}

# Dictionaries and lists for all matches found during the washing
ipv4AddressMatches = {}
ipv6AddressMatches = {}
imsiMatches = {}
imsiHexMatches = {}
macMatches = {}
urlFound = []
hostnameFound = []
usernameFound = []

# Different variables needed throughout the script
inboxLocation = '/nfs/data/inbox/'
folderLocation = '/nfs/data/'
tmpLocation = '/nfs/data/tmp/'
folder_dt = datetime.now()
dt_string = folder_dt.strftime('%y-%m-%d-%H.%M')
outboxLocation = folderLocation + '/outbox/'
outboxDirname = 'washed-' + dt_string
outboxDir = folderLocation + '/outbox/' + outboxDirname
tmpDir = tmpLocation + outboxDirname
script_log = '/nfs/data/cron-script.log'
washing_log = '/local/scramble/washing-script/log/washing-script.log'
zipEnd = '.zip'
zipLogEnd = '.log.zip'
gzEnd = '.gz'
gzLogEnd = '.log.gz'
tarEnd = '.tar'
tarLogEnd = '.log.tar'
targzEnd = '.tar.gz'
targzLogEnd = '.log.tar.gz'

def replaceCharsInTuple(tuple):
    try:
        # Clean up IP-addresses in tuple and return as string
        tupleToReturn = tuple.replace(',', '.').replace('(', '').replace(')', '').replace(' ', '').replace('\'', '')
        return ''.join(tupleToReturn)
    except Exception:
        logger.error(current_time() + ' - An error occurred:\n' + traceback.format_exc())

def replaceIpv6Tuple(tuple):
    try:
        ipv6TupleToReturn = tuple[0]
        return ipv6TupleToReturn
    except Exception:
        logger.error(current_time() + ' - An error occurred:\n' + traceback.format_exc())

def InsertSpaceInTupleImsiHex(tuple):
    try:
        # Clean up IMSI hex in tuple and return as string
        tupleToReturn = tuple.replace(',', '').replace('(', '').replace(')', '').replace('\'', '') \
            .replace('IMSI : ', '').replace('IMSI :   { ', '').replace(':', '').replace('{', '').replace('   ', '') \
            .replace('  ', ' ')
        return ''.join(tupleToReturn)
    except Exception:
        logger.error(current_time() + ' - An error occurred:\n' + traceback.format_exc())

def checkIfIpv4ExistsAndReplace(match):
    try:
        # If match not found in dictionary, generate a new IPv4 address with the first two octates as x
        if ipv4AddressMatches.get(match) == None:
            ipv4List = ['x', 'x']
            ipv4List.append(str(random.randint(1, 255)))
            ipv4List.append(str(random.randint(1, 255)))
            ipv4AddressMatches[match] = '.'.join(ipv4List)
        return ipv4AddressMatches.get(match)
    except Exception:
        logger.error(current_time() + ' - An error occurred:\n' + traceback.format_exc())

def checkIfIpv6ExistsAndReplace(match):
    try:
        # If match not found in dictionary, generate a new IPv6 address
        false_ipv6 = regEx.compile(r'[a-fA-F]{1,3}::|::[a-fA-F]{1,3}| ::|:: | :: ')
        false_match = regEx.findall(false_ipv6, match)
        socket_match = socket.inet_pton(socket.AF_INET6, match)
        if True:
            if match != '::':
                if len(false_match) == 0:
                    if ipv6AddressMatches.get(match) == None:
                        ipv6AddressMatches[match] = ipaddress.IPv6Address(
                            random.randint(0, 2 ** 128 - 1))  # Add random IPv6
                    return ipv6AddressMatches.get(match)
                if len(false_match) <= 1:
                    ipv6AddressMatches[match] = false_match[0]
                    return ipv6AddressMatches.get(match)
            if match == '::':
                ipv6AddressMatches[match] = '::'
                return ipv6AddressMatches.get(match)
        if False:
            ipv6AddressMatches[match] = false_match[0]
            return ipv6AddressMatches.get(match)

    except socket.error:
        ipv6AddressMatches[match] = false_match[0]
        return ipv6AddressMatches.get(match)

def checkIfMacExistsAndReplace(match):
    try:
        # If match not found in dictionary, generate a new mac address
        if macMatches.get(match) == None:
            macMatches[match] = '%02x:%02x:%02x:%02x:%02x:%02x' % (
                random.randint(0, 255),
                random.randint(0, 255),
                random.randint(0, 255),
                random.randint(0, 255),
                random.randint(0, 255),
                random.randint(0, 255))
        return macMatches.get(match)
    except Exception:
        logger.error(current_time() + ' - An error occurred:\n' + traceback.format_exc())

def checkIfImsiExistsAndReplace(match):
    try:
        # If match not found in dictionary, generate a new random number with xxxxxx in front
        if imsiMatches.get(match) == None:
            imsiList = ['xxxxx']
            imsiList.append(str(random.randint(1000000000, 9999999999)))
            imsiMatches[match] = ''.join(imsiList)
        return imsiMatches.get(match)
    except Exception:
        logger.error(current_time() + ' - An error occurred:\n' + traceback.format_exc())

def checkIfImsiHexExistsAndReplace(match):
    try:
        # If match not found in dictionary, generate a new random number with xx on part of the hex
        if imsiHexMatches.get(match) == None:
            imsiHexList = ['xx ', 'xx ', 'xx ', 'xx ']
            imsiHexList.append(str(random.randint(10, 99)) + ' ' + str(random.randint(10, 99)) + ' ' +
                               str(random.randint(10, 99)) + ' ' + str(random.randint(10, 99)) + ' ' +
                               str(random.randint(0, 9)) + 'F')
            imsiHexMatches[match] = str(''.join(imsiHexList))
        return imsiHexMatches.get(match)
    except Exception:
        logger.error(current_time() + ' - An error occurred:\n' + traceback.format_exc())

def checkIfHostnameExistsAndReplace(match):
    try:
        # If match not found in dictionary, replace it. If found replace item listed in the dictionary
        if hostnameMatches.get(match) == None:
            logger.warning(current_time() + ' - !!!!!!!!!!' + match + ' not found in hostname list!!!!!!!!!!\n'
                                                                       'Match removed from file')
            hostnameMatches[match] = str('xxxxxxx')
        return hostnameMatches.get(match)
    except Exception:
        logger.error(current_time() + ' - An error occurred:\n' + traceback.format_exc())

def checkIfUsernameExistsAndReplace(match):
    try:
        # If match not found in dictionary, replace it. If found replace item listed in the dictionary
        if usernameMatches.get(match) == None:
            usernameMatches[match] = str('xxxxxxxxx')
        return usernameMatches.get(match)
    except Exception:
        logger.error(current_time() + ' - An error occurred:\n' + traceback.format_exc())

def checkIfUrlExistsAndReplace(match):
    try:
        # If match not found in dictionary, replace it. If found replace item listed in the dictionary
        if urlMatches.get(match) == None:
            logger.warning(current_time() + ' - !!!!!!!!!!' + match + ' not found in URL list!!!!!!!!!!\n'
                                                                       'Match removed from file')
            urlMatches[match] = str('xxxxx')
        return urlMatches.get(match)
    except Exception:
        logger.error(current_time() + ' - An error occurred:\n' + traceback.format_exc())

def current_time():
    return time.strftime('%d-%m-%y %H:%M:%S', time.localtime())

logging.basicConfig(filename=washing_log,
                    format='%(message)s')
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)

def wash_filename(item):
    try:
        global washed_filename
        current_filename = str(item)
        filename_ipv4Match = regEx.findall(ipv4Pattern, item)
        filename_ipv6Match = regEx.findall(ipv6Pattern, item)
        filename_imsiMatch = regEx.findall(imsiPattern, item)
        filename_imsiHexMatch = regEx.findall(imsiHexPattern, item)
        filename_hostnameMatch = regEx.findall(hostnamePattern, item)
        filename_macMatch = regEx.findall(macPattern, item)
        filename_usernameMatch = regEx.findall(usernamePattern, item)
        filename_urlMatch = regEx.findall(urlPattern, item)
        filename_ipv4Array = []
        filename_ipv6Array = []
        filename_imsiArray = []
        filename_imsiHexArray = []
        filename_macArray = []
        filename_hostnameArray = []
        filename_usernameArray = []
        filename_urlArray = []
        for i in range(len(filename_ipv4Match)):
            filename_ipv4Array.append(replaceCharsInTuple(str(filename_ipv4Match[i])))
        for i in range(len(filename_ipv6Match)):
            filename_ipv6Array.append(replaceIpv6Tuple(filename_ipv6Match[i]))
        for i in range(len(filename_macMatch)):
            filename_macArray.append(filename_macMatch[i])
        for i in range(len(filename_imsiMatch)):
            filename_imsiArray.append(filename_imsiMatch[i])
        for i in range(len(filename_imsiHexMatch)):
            filename_imsiHexArray.append(InsertSpaceInTupleImsiHex(str(filename_imsiHexMatch[i])))
        for i in range(len(filename_hostnameMatch)):
            filename_hostnameArray.append(filename_hostnameMatch[i])
        for i in range(len(filename_usernameMatch)):
            filename_usernameArray.append(filename_usernameMatch[i])
        for i in range(len(filename_urlMatch)):
            filename_urlArray.append(filename_urlMatch[i])
        for ipv4 in filename_ipv4Array:
            replacedIpv4Address = checkIfIpv4ExistsAndReplace(ipv4)
            new_filename = item.replace(ipv4, replacedIpv4Address)
            washed_filename = new_filename  # Overwrite current filename with new filename
        for ipv6 in filename_ipv6Array:
            replacedIpv6Address = checkIfIpv6ExistsAndReplace(ipv6)
            new_filename = item.replace(ipv6, str(replacedIpv6Address))
            washed_filename = new_filename  # Overwrite current filename with new filename
        for mac in filename_macArray:
            replacedMacAddress = checkIfMacExistsAndReplace(mac.lower())
            new_filename = item.replace(mac, replacedMacAddress)
            washed_filename = new_filename  # Overwrite current filename with new filename
        for imsi in filename_imsiArray:
            replacedImsiAddress = checkIfImsiExistsAndReplace(imsi)
            new_filename = item.replace(imsi, replacedImsiAddress)
            washed_filename = new_filename  # Overwrite current filename with new filename
        for imsiHex in filename_imsiHexArray:
            replacedImsiHexAddress = checkIfImsiHexExistsAndReplace(imsiHex)
            new_filename = item.replace(imsiHex, replacedImsiHexAddress)
            washed_filename = new_filename  # Overwrite current filename with new filename
        for hostname in filename_hostnameArray:
            if hostname.lower() not in hostnameFound:
                hostnameFound.append(hostname.lower())
            replacedHostnameAddress = checkIfHostnameExistsAndReplace(hostname.lower())
            new_filename = item.replace(hostname, replacedHostnameAddress)
            washed_filename = new_filename  # Overwrite current filename with new filename
        for username in filename_usernameArray:
            if username.lower() not in usernameFound:
                usernameFound.append(username.lower())
            replacedUsernameAddress = checkIfUsernameExistsAndReplace(username.lower())
            new_filename = item.replace(username, str(replacedUsernameAddress))
            washed_filename = new_filename  # Overwrite current filename with new filename
        for url in filename_urlArray:
            if url.lower() not in urlFound:
                urlFound.append(url.lower())
            replacedurlAddress = checkIfUrlExistsAndReplace(url.lower())
            new_filename = item.replace(url, replacedurlAddress)
            washed_filename = new_filename  # Overwrite current filename with new filename
        if current_filename == str(washed_filename):
            return washed_filename
    except Exception:
        logger.error(current_time() + ' - An error occurred:\n' + traceback.format_exc())

def unzipFiles():
    try:
        os.chdir(tmpLocation)
        os.getcwd()
        # Walk down the tmp folder and find/unzip all zipped files
        # If there is a zipped file, it will start over to walk into the unzipped files aswell
        for root, dirs, files in os.walk(tmpLocation):
            for item in files:
                itemLocation = root + '/' + item
                # Unzip files ending with .gz, .zip or .tar, then delete the zipped version
                if os.path.exists(itemLocation):
                    if item.endswith(zipEnd):
                        logger.info(current_time() + ' - Unzipping: "' + root + '"/"' + item + '"')
                        newZipName = item.replace('.zip', '')
                        os.system('mkdir "' + str(root) + '"/"' + str(newZipName) + '"')
                        os.system('unzip -o -qq "' + str(root) + '"/"' + str(item) + '" -d "' + str(root) + '"/"' +
                                  str(newZipName) + '"')
                        os.system('rm -rf "' + str(root) + '"/"' + str(item) + '"')
                        unzipFiles()
                    if item.endswith(zipLogEnd):
                        logger.info(current_time() + ' - Unzipping: "' + root + '"/"' + item + '"')
                        newZipName = item.replace('.zip', '')
                        os.system('mkdir "' + str(root) + '"/"' + str(newZipName) + '"')
                        os.system('unzip -o -qq "' + str(root) + '"/"' + str(item) + '" -d "' + str(root) + '"/"' +
                                  str(newZipName) + '"')
                        os.system('rm -rf "' + str(root) + '"/"' + str(item) + '"')
                    if item.endswith(targzEnd) or item.endswith(tarEnd):
                        logger.info(current_time() + ' - Unzipping tar file: "' + root + '"/"' + str(item) + '"')
                        newTarName1 = item.replace('.gz', '')
                        newTarName2 = newTarName1.replace('.tar', '')
                        os.system('mkdir "' + str(root) + '"/"' + newTarName2 + '"')
                        os.system('tar -xf "' + str(root) + '"/"' + str(item) + '" -C "' + str(root) + '"/"' +
                                  str(newTarName2) + '"')
                        os.system('rm -rf "' + str(root) + '"/"' + str(item) + '"')
                        unzipFiles()
                    if item.endswith(targzLogEnd) or item.endswith(tarLogEnd):
                        logger.info(current_time() + ' - Unzipping tar file: "' + root + '"/"' + str(item) + '"')
                        newTarName1 = item.replace('.gz', '')
                        newTarName2 = newTarName1.replace('.tar', '')
                        os.system('mkdir "' + str(root) + '"/"' + newTarName2 + '"')
                        os.system('tar -xf "' + str(root) + '"/"' + str(item) + '" -C "' + str(root) + '"/"' +
                                  str(newTarName2) + '"')
                        os.system('rm -rf "' + str(root) + '"/"' + str(item) + '"')
                    if item.endswith(gzEnd):
                        logger.info(current_time() + ' - Unzipping gz file: "' + root + '"/"' + item + '"')
                        os.system('gzip -fd "' + str(root) + '"/"' + str(item) + '"')
                        unzipFiles()
                    if item.endswith(gzLogEnd):
                        logger.info(current_time() + ' - Unzipping gz file: "' + root + '"/"' + item + '"')
                        os.system('gzip -fd "' + str(root) + '"/"' + str(item) + '"')
        walking_file()
    except Exception as err:
        with open(script_log, 'a') as f:
            f.write(current_time() + ' - An error occurred\n' + traceback.format(err))
        logger.error(current_time() + ' - An error occurred\n' + traceback.format(err))
        quit()

def walking_file():
    try:
        global washed_filename
        tmp_dirs = os.listdir(tmpLocation)
        for folder in tmp_dirs:
            for root, dirs, files in os.walk(folder):
                for i in dirs:
                    washed_filename = ''
                    wash_filename(str(i))
                    if str(i) != washed_filename and washed_filename != '':
                        os.system('mv "' + root + '"/"' + str(i) + '" "' + root + '"/"' + washed_filename + '"')
                        walking_file()
            for root, dirs, files in os.walk(folder):
                for file in files:
                    washed_filename = ''
                    wash_filename(str(file))
                    if str(file) != washed_filename and washed_filename != '':
                        os.system('mv "' + root + '"/"' + file + '" "' + root + '"/"' + washed_filename + '"')
        washFiles()
    except Exception:
        logger.error(current_time() + ' - An error occurred:\n' + traceback.format_exc())

def washFiles():
    try:
        with open(script_log, 'a') as f:
            f.write(current_time() + ' - Done unzipping all folders\n')
        logger.info(current_time() + ' - Done unzipping all folders')
        tmp_dirs = os.listdir(tmpLocation)
        for directory in tmp_dirs:
            # Walking down the file structure in the inbox file and washing each file
            for root, dirs, files in os.walk(directory):
                for file in files:
                    logger.info(current_time() + ' - Now washing ' + file)
                    newFileContent = ''
                    with open(root + '/' + file, 'r+') as logFile:
                        for line in logFile:
                            # Going through each line in the file and finding regex mathes
                            currentLine = line.strip()
                            ipv4MatchesInLine = regEx.findall(ipv4Pattern, line)
                            ipv6MatchesInLine = regEx.findall(ipv6Pattern, line)
                            imsiMatchesInLine = regEx.findall(imsiPattern, line)
                            imsiHexMatchesInLine = regEx.findall(imsiHexPattern, line)
                            macMatchesInLine = regEx.findall(macPattern, line)
                            hostnameMatchesInLine = regEx.findall(hostnamePattern, line)
                            usernameMatchesInLine = regEx.findall(usernamePattern, line)
                            urlMatchesInLine = regEx.findall(urlPattern, line)
                            ipv4Array = []
                            ipv6Array = []
                            imsiArray = []
                            imsiHexArray = []
                            macArray = []
                            hostnameArray = []
                            usernameArray = []
                            urlArray = []
                            # Add all matches to their respective array
                            for i in range(len(ipv4MatchesInLine)):
                                ipv4Array.append(replaceCharsInTuple(str(ipv4MatchesInLine[i])))
                            for i in range(len(ipv6MatchesInLine)):
                                ipv6Array.append(replaceIpv6Tuple(ipv6MatchesInLine[i]))
                            for i in range(len(macMatchesInLine)):
                                macArray.append(macMatchesInLine[i])
                            for i in range(len(imsiMatchesInLine)):
                                imsiArray.append(imsiMatchesInLine[i])
                            for i in range(len(imsiHexMatchesInLine)):
                                imsiHexArray.append(InsertSpaceInTupleImsiHex(str(imsiHexMatchesInLine[i])))
                            for i in range(len(hostnameMatchesInLine)):
                                hostnameArray.append(hostnameMatchesInLine[i])
                            for i in range(len(usernameMatchesInLine)):
                                usernameArray.append(usernameMatchesInLine[i])
                            for i in range(len(urlMatchesInLine)):
                                urlArray.append(urlMatchesInLine[i])
                            # Replace all matches found with the other entry in the dictionary
                            for ipv4 in ipv4Array:
                                replacedIpv4Address = checkIfIpv4ExistsAndReplace(ipv4)
                                newLine = currentLine.replace(ipv4, replacedIpv4Address)
                                currentLine = newLine  # Overwrite current line with newLine to reflect changes made
                            for ipv6 in ipv6Array:
                                replacedIpv6Address = checkIfIpv6ExistsAndReplace(ipv6)
                                newLine = currentLine.replace(ipv6, str(replacedIpv6Address))
                                currentLine = newLine  # Overwrite current line with newLine to reflect changes made
                            for mac in macArray:
                                replacedMacAddress = checkIfMacExistsAndReplace(mac.lower())
                                newLine = currentLine.replace(mac, replacedMacAddress)
                                currentLine = newLine  # Overwrite current line with newLine to reflect changes made
                            for imsi in imsiArray:
                                replacedImsiAddress = checkIfImsiExistsAndReplace(imsi)
                                newLine = currentLine.replace(imsi, replacedImsiAddress)
                                currentLine = newLine  # Overwrite current line with newLine to reflect changes made
                            for imsiHex in imsiHexArray:
                                replacedImsiHexAddress = checkIfImsiHexExistsAndReplace(imsiHex)
                                newLine = currentLine.replace(str(imsiHex), str(replacedImsiHexAddress))
                                currentLine = newLine  # Overwrite current line with newLine to reflect changes made
                            for hostname in hostnameArray:
                                if hostname.lower() not in hostnameFound:
                                    hostnameFound.append(hostname.lower())
                                replacedHostnameAddress = checkIfHostnameExistsAndReplace(hostname.lower())
                                newLine = currentLine.replace(hostname, replacedHostnameAddress)
                                currentLine = newLine  # Overwrite current line with newLine to reflect changes made
                            for username in usernameArray:
                                if username.lower() not in usernameFound:
                                    usernameFound.append(username.lower())
                                replacedUsernameAddress = checkIfUsernameExistsAndReplace(username.lower())
                                newLine = currentLine.replace(username, str(replacedUsernameAddress))
                                currentLine = newLine  # Overwrite current line with newLine to reflect changes made
                            for url in urlArray:
                                if url.lower() not in urlFound:
                                    urlFound.append(url.lower())
                                replacedurlAddress = checkIfUrlExistsAndReplace(url.lower())
                                newLine = currentLine.replace(url, replacedurlAddress)
                                currentLine = newLine  # Overwrite current line with newLine to reflect changes made
                            newFileContent += currentLine + '\n'
                        logFile.truncate(0)  # Remove old content of file
                        logFile.seek(0)  # Start writing from index 0
                        logFile.write(newFileContent)
                    logger.info(current_time() + ' - ###Done washing ' + file + '###')
            os.system('touch ' + directory + '/washingreport.txt')
            # Making a file which contains all matches found and what they are changed to
            with open(directory + '/washingreport.txt', 'r+') as reportfile:
                stringToWrite = ('########################\nResult from washing\n########################\n\n' +
                                 'Ipv4 dictionary:\n' + str(ipv4AddressMatches) +
                                 '\n\nIpv6 dictionary:\n' + str(ipv6AddressMatches) +
                                 '\n\nImsi dictionary:\n' + str(imsiMatches) +
                                 '\n\nImsi hex dictionary:\n' + str(imsiHexMatches) +
                                 '\n\nMac address dictionary:\n' + str(macMatches) +
                                 '\n\nUsernames found:\n' + str(usernameFound) +
                                 '\n\nHostnames found:\n' + str(hostnameFound) +
                                 '\n\nUrl found:\n' + str(urlFound) + '\n')
                reportfile.write(stringToWrite)
            # Move all files washed in tmp folder to the respective outbox folder
            os.system('chmod 777 ' + directory)
            os.system('mv ' + directory + ' ' + outboxLocation)
            logger.info('\n\n' + current_time() + '\n########################\nMoved files to: ' + outboxLocation + directory +
                  '\n########################\n')
        with open(script_log, 'a') as f:
            f.write(str(current_time()) + ' - Washing is complete\n')
        logger.info(current_time() + ' - Washing is complete\n')
    except Exception:
        logger.error(current_time() + ' - An error occurred:\n' + traceback.format_exc())

def move_files():
    try:
        content_inbox = os.listdir(inboxLocation)
        content_tmp = os.listdir(tmpLocation)
        logger.info(current_time() + ' - Script starting')
        if len(content_inbox) == 0 and len(content_tmp) == 0:
            quit()
        if len(content_inbox) != 0 and len(content_tmp) == 0:
            while True:
                sftp_files = subprocess.check_output('du -H -d1 /nfs/data/inbox/', shell=True).decode()
                time.sleep(2)
                sftp_files_2 = subprocess.check_output('du -H -d1 /nfs/data/inbox/', shell=True).decode()
                if str(sftp_files) == str(sftp_files_2):
                    break
                if str(sftp_files) != str(sftp_files_2):
                    with open(script_log, 'a') as f:
                        f.write(str(current_time()) + ' - Files are still being transferred, will wait to start script\n')
                    logger.warning(current_time() + ' - Files are still being transferred, will wait to start script')
                    time.sleep(2)
            # Make a new directory in /tmp with date and time
            os.system('mkdir ' + tmpDir)
            os.system('cp -r ' + inboxLocation + '* ' + tmpDir)
            os.system('rm -rf ' + inboxLocation + '*')
            os.system('chmod -R 777 ' + tmpDir)
            tmp_folder_content = os.listdir(tmpDir)
            for folder in tmp_folder_content:
                if folder.endswith(zipEnd):
                    logger.info(current_time() + ' - Unzipping: ' + tmpDir + '/"' + folder + '"')
                    newZipName = folder.replace('.zip', '')
                    os.system('mkdir ' + str(tmpDir) + '/"' + str(newZipName) + '"')
                    os.system('unzip -o -qq ' + str(tmpDir) + '/"' + str(folder) + '" -d ' + str(tmpDir) + '/"' +
                              newZipName + '"')
                    os.system('rm -rf ' + str(tmpDir) + '/"' + str(folder) + '"')
                if folder.endswith(gzEnd):
                    logger.info(current_time() + ' - Unzipping gz file: ' + tmpDir + '/"' + folder + '"')
                    os.system('gzip -fd ' + str(tmpDir) + '/"' + str(folder) + '"')
                if folder.endswith(targzEnd) or folder.endswith(tarEnd):
                    logger.info(current_time() + ' - Unzipping tar file: ' + tmpDir + '/"' + folder + '"')
                    newTarName1 = folder.replace('.gz', '')
                    newTarName2 = newTarName1.replace('.tar', '')
                    os.system('mkdir ' + str(tmpDir) + '/"' + newTarName2 + '"')
                    os.system('tar -xf ' + str(tmpDir) + '/"' + str(folder) + '" -C ' + str(tmpDir) + '/"' +
                              newTarName2 + '"')
                    os.system('rm -rf ' + str(tmpDir) + '/"' + folder + '"')
            with open(script_log, 'a') as f:
                f.write(str(current_time()) + ' - Done unzipping root folder\n')
            logger.info(current_time() + ' - Done unzipping root folder')
            check_folder()
        if len(content_tmp) != 0 and len(content_inbox) == 0:
            check_folder()
    except Exception:
        logger.error(current_time() + ' - An error occurred:\n' + traceback.format_exc())

def check_folder():
    try:
        num_gz = subprocess.check_output('find ' + tmpLocation + ' -name "*gz"', shell=True).decode()
        num_gz = num_gz.split('\n')
        num_zipped = subprocess.check_output('find ' + tmpLocation + ' -name "*zip"', shell=True).decode()
        num_zipped = num_zipped.split('\n')
        num_tar = subprocess.check_output('find ' + tmpLocation + ' -name "*tar"', shell=True).decode()
        num_tar = num_tar.split('\n')
        total_zipped = len(num_gz) + len(num_zipped) + len(num_tar) - 3
        if total_zipped >= 500:
            new_limit = total_zipped * 3
            sys.setrecursionlimit(new_limit)
            limit = str(new_limit)
            with open(script_log, 'a') as f:
                f.write(str(current_time()) + ' - Amount of zipped files will exceed max amount of calls, will change '
                                              'recursion limit to: ' + limit + '\n')
            logger.warning(current_time() + ' - Amount of zipped files will exceed max amount of calls, will change '
                                             'recursion limit to: ' + limit)
            unzipFiles()
        if total_zipped <= 500:
            unzipFiles()
    except Exception:
        logger.error(current_time() + ' - An error occurred:\n' + traceback.format_exc())

if __name__ == '__main__':
    move_files()


import purchase
import utils.queries as queries

#Function that will take user input on how to order our clothing articles (for print-out)
#After it calls orderBy, this function will call purchase if input is valid. Input will be the clothing ID of the article of clothing you want to buy
def viewAndPlace(connection, custID):
	customerInteracting = True
	while customerInteracting == True:
		user_input = raw_input("Order by: ID No (0), Name (1), Type (2), Season (3), Price (4), Material (5)\n")
		myDict={
			"0":"ClothingID",
			"1":"Name",
			"2":"Type",
			"3":"Season",
			"4":"Price",
			"5":"Material"
		}
		val = myDict.get(user_input,"back")
		if val == "back":
			return
		else:
			if not queries.printClothesInOrder(connection,val):
				return

		isValid = False
		while isValid == False:
			user_input = raw_input("\nTo purchase an item, input its ID No. Type 'back' to go back to your action screen\n")
			if user_input == "back":
				return

			try:
				user_input = int(user_input)
				isValid = True
			except:
				print("That's an invalid id")

		article = queries.selectArticle(connection,user_input)
		if article == None:
			print("Could not select clothing item to purchase")
		else:
			purchase.purchase(connection, custID, article[0])


"""SQLAlchemy one-to-many relationship with multiple foreign keys.

https://avacariu.me/writing/2019/composite-foreign-keys-and-many-to-many-relationships-in-sqlalchemy
"""

from pathlib import Path
from typing import List

from sqlalchemy import (
    Column,
    ForeignKey,
    ForeignKeyConstraint,
    Integer,
    String,
    Table,
    create_engine,
    select,
)
from sqlalchemy.orm import Session, declarative_base, relationship

Base = declarative_base()

class Author(Base):
    """An author is identified by their userid and username."""

    __tablename__ = "author"

    userid = Column(Integer, primary_key=True)
    username = Column(String(30), primary_key=True)

    recipes = relationship(
        "Recipe",
        back_populates="author",
        # foreign_keys=[userid, username],
        # primaryjoin="and_(Recipe.author_userid==Author.userid, Recipe.author_username==Author.username)",
    )

class Recipe(Base):
    """A recipe is identified by its id and title."""

    __tablename__ = "recipe"

    id = Column(Integer, primary_key=True)
    title = Column(String(30))

    # author_userid = Column(Integer, ForeignKey("author.userid"), nullable=False)
    # author_username = Column(String(30), ForeignKey("author.username"), nullable=False)

    author_userid = Column(Integer, nullable=False)
    author_username = Column(String(30), nullable=False)

    author = relationship(
        "Author",
        back_populates="recipes",
        # primaryjoin="and_(Recipe.author_userid==Author.userid, Recipe.author_username==Author.username)",
        # foreign_keys=[author_userid, author_username],
    )

    # this produces
    # FOREIGN KEY(author_userid, author_username) REFERENCES author (userid, username)
    # which I kinda like more than the double references
    # FOREIGN KEY(author_userid) REFERENCES author (userid),
    # FOREIGN KEY(author_username) REFERENCES author (username)
    __table_args__ = (
        ForeignKeyConstraint(
            ["author_userid", "author_username"],
            ["author.userid", "author.username"],
        ),
    )

# Create the database and engine
sqlite_file_name = "local_o2m_sa_dup.db"
sqlite_fp = Path(sqlite_file_name)
if sqlite_fp.exists():
    sqlite_fp.unlink()
sqlite_url = f"sqlite:///{sqlite_file_name}"
engine = create_engine(sqlite_url, echo=True)

def create_db_and_tables() -> None:
    """Set up the database and tables."""
    Base.metadata.create_all(engine)

def create_and_select() -> None:
    with Session(engine) as session:
        author = Author(userid=1, username="spongebob")
        recipe = Recipe(id=1, title="Krabby Patty", author=author)
        session.add(recipe)
        session.commit()
        print(f"\nDONE\n")
        print(f"\nADDED RECIPE: {recipe.title}")

def main() -> None:
    """Main function."""
    create_db_and_tables()
    create_and_select()

if __name__ == "__main__":
    main()


import pytest
from django.test import RequestFactory
from mixer.backend.django import mixer

from apps.core.views import ResubscribeView

pytestmark = pytest.mark.django_db

class TestResubscribe:
    def test_auth_resubscribe_with_payments(self):
        profile = mixer.blend('core.Profile')
        mixer.blend('core.Payment', profile=profile)

        request = RequestFactory().get('/payment')
        request.user = profile.user

        response = ResubscribeView.as_view()(request)

        assert response.status_code == 200
        response.render()
        assert 'button_paypal' in response.content.decode('utf-8')


""" Perform quick baseline benchmarck based on bag of words
    for sentiment analysis

    Author: Pham Quang Nhat Minh (FTRI)
"""

import os
import sys
import pandas as pd
import numpy as np
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn import metrics
from sklearn import cross_validation
from sklearn.pipeline import Pipeline
from sklearn.ensemble import AdaBoostClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.svm import LinearSVC
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import BaggingClassifier
from sklearn.neighbors import KNeighborsClassifier

def get_w(w, tag):
    if tag:
        return w.split('/')[0]
    else:
        return w

if __name__ == '__main__':
    os.system('clear')
    tag = False

    # use raw data
    # datadir = './data/SA2016-training_data'

    # use data with word segmentation
    datadir = './data/SA2016-training-data-ws'

    # filenames = ['SA-training_positive.txt',
    #              'SA-training_negative.txt',
    #              'SA-training_neutral.txt',
    #             ]

    filenames = [
        'train_positive_tokenized.txt',
        'train_negative_tokenized.txt',
        'train_neutral_tokenized.txt',
    ]

    #label_codes = ['pos', 'neg']

    label_codes = ['pos', 'neg', 'neutral']

    print("******** Use binary features ********")
    sentences = []
    labels = []
    for i, filename in enumerate(filenames):
        path  = os.path.join(datadir, filename)
        label = label_codes[i]
        f = open(path, 'r')
        for line in f:
            line = line.rstrip()
            if line == '':
                continue
            words = [ get_w(w, tag) for w in line.split()]
            sentences.append( ' '.join( words ) )
            labels.append(label)

    y = np.array(labels)
    # count_vect = CountVectorizer( ngram_range = (1,3), binary=True )
    count_vect = CountVectorizer( binary=True )
    X_binary = count_vect.fit_transform( sentences )

    models = [
        LinearSVC(),
        RandomForestClassifier(n_estimators=100, max_depth=None,
                               min_samples_split=1, random_state=0),
    ]

    model_names = [
        'Linear SVM',
        'Random Forest',
    ]

    for clf, mdname in zip(models, model_names):
        print('== Use %s method ==' % mdname)
        X = X_binary
        if mdname == 'Gradient Boosting Trees':
            X = X_binary.toarray()
        predicted = cross_validation.cross_val_predict(clf, X, y, cv=10)
        print(metrics.classification_report(y, predicted))
        print

    print

    print("******** Use TF-IDF weighting **********")

    # count_vect = CountVectorizer(ngram_range = (1,3))

    count_vect = CountVectorizer()
    X_count = count_vect.fit_transform( sentences )
    tfidf_transformer = TfidfTransformer()
    X_tfidf = tfidf_transformer.fit_transform( X_count )

    for clf, mdname in zip(models, model_names):
        print('== Use %s method ==' % mdname)
        X = X_tfidf
        predicted = cross_validation.cross_val_predict(clf, X, y, cv=10)
        print(metrics.classification_report(y, predicted))
        print


# Services
import logging
from typing import List
# Own
from arq_server.base.ArqErrors import ArqError
from arq_server.services.CoreService import Configuration,Base
from arq_server.services.support.SecurityTools import Security

class NormalizeSelector:
    # Services TIPS
    __logger: logging.Logger
    __config: Configuration
    __security: Security

    def __init__(self, core, cross):
        self.__init_services(
            core.logger_service(),
            core.config_service(),
            cross.security_tools()
        )
        self.__logger.info("NormalizeSelector - Servicios core inicializados correctamente")
        self.__avaliableServicesWithInput={
            'configuration': core.config_service(),
            'security': cross.security_tools()
        }

        self.__logger.info("NormalizeSelector - lista de servicios que admiten instrucciones:\n"+str(self.__avaliableServicesWithInput.keys()))

    def addAvaliableService(self,singletonService:object):
        try:
            self.__avaliableServicesWithInput[singletonService.__class__.__name__]=singletonService
            self.__logger.info("Nuevo servicio incluído:" \
                + singletonService.__class__.__name__ \
                + "Ahora estan expuestos los siguientes servicios:\n" \
                + str(self.__avaliableServicesWithInput.keys())
            )
        except Exception as e:
            raise ArqError("¡Error añadiendo un servicio nuevo a los ya expuestos! -> "+str(e))

    def processInput(self,input:dict, headers:dict)->dict:
        """
        metadata:
        {
            'protocol' : '',
            'timeInit' : '',
            'user' : ''
        }
        """
        output = {}
        try:
            input = self.__validateInput(input) # Si la entrada no es correcta, salta excepción
            # Parametros de entrada
            context = input.pop('context')
            service = input.pop('service')
            metadata = input.pop('metadata')
            filtered_headers=self.__filterHeaders(headers,service) # Si faltan cabeceras, salta excepción
            if service not in self.__config.getProperty('logical','publicServices').split(','):
                self.__security.validate_token(filtered_headers['token']) # Valida token para los servicios protegidos
            self.__logger.info("Contexto: %s",context)
            if context == 'arq':
                output['response']=self.__arq_instructions(service,input,**filtered_headers)
            else:
                raise ArqError("contexto no válido")

            output['metadata']=metadata
        except ArqError as arqErr:
            output['error']=arqErr.normalize_exception()
        return output

    def __arq_instructions(self,service, input_instructions:dict,**kwargs):
        if service not in self.__avaliableServicesWithInput:
            raise ArqError("Servicio de arquitectura no existe o no admite instrucciones")
        return self.__avaliableServicesWithInput[service].read_input_instruccions(input_instructions,**kwargs)

    def __validateInput(self,raw_input:dict)->dict:
        """
        Comprueba que el input fuente contiene las claves configuradas. Descarta excesos
        ---
        Devuelve el diccionario de entrada filtrado
        """
        avaliableKeys= self.__config.getProperty('logical','avaliableInputKeys').split(',')
        try:
            filtered_input =  { av_key: raw_input[av_key] for av_key in avaliableKeys }
            if (not isinstance(filtered_input['args'],List)) or (not isinstance(filtered_input['kwargs'],List)):
                raise ArqError("Los argumentos no traen el formato correcto")
            self.__logger.info("La entrada es válida")
            return filtered_input
        except ArqError as arqe:
            raise arqe
        except Exception as e:
            raise ArqError("La entrada no cumple los requisitos, revisar:"+str(e))

    def __filterHeaders(self,raw_headers,service):
        """
        Comprueba que las cabeceras contiene las claves configuradas. Descarta excesos
        ---
        Devuelve el diccionario de cabeceras filtrado
        """
        service_headers= self.__config.getProperty('logical',service+'.avaliableHeaders')
        if service_headers is None:
            service_headers = self.__config.getProperty('logical','__default.avaliableHeaders')
        avaliableHeaders = service_headers.split(',')
        try:
            filtered_headers =  { av_key: raw_headers[av_key] for av_key in avaliableHeaders }
            return filtered_headers
        except ArqError as arqe:
            raise arqe
        except Exception as e:
            raise ArqError("Faltan cabeceras, revisar:"+str(e))

    def __init_services(self, logger, config, security):
        # Servicio de logging
        self.__logger = logger.arqLogger()
        self.__config = config
        self.__security = security


import pathlib
import sys
module_dir = pathlib.Path(__file__).parent.resolve()
root_dir = module_dir.parent
model_dir = root_dir.joinpath("models")
asvlite_wrapper_dir = root_dir.joinpath("dependency", "ASVLite", "wrapper", "cython")
sys.path.insert(0, str(asvlite_wrapper_dir))
import os.path
import math
import multiprocessing as mp
import pyproj
import epsg
import random
import numpy as np
import pandas as pd
from tqdm import tqdm # to show a progress bar
from sea_surface import py_Sea_surface
from asv import py_Asv, py_Asv_specification
from geometry import py_Coordinates_3D

class Rudder_PID_controller:
    def __init__(self, asv_spec, K=None):
        self.max_rudder_angle = math.pi/6 # 30 deg
        self.asv_spec = asv_spec
        self.error = 0.0 # error in each time step
        self.previous_error  = 0.0 # error in the previous time step
        self.cumulative_error = 0.0 # integral of errors
        self.delta_error = 0.0 # differential of error
        self.out_dir = root_dir.joinpath("results", "rudder_controller", "tuning")
        self.out_dir.mkdir(parents=True, exist_ok=True) # Create the out_dir if it does not exist.
        # P,I,D gain terms
        if K is not None:
            self.K = np.array(K)
        else:
            # K is None
            # Check if PID file exist in models
            rudder_PID_file = model_dir.joinpath("rudder_PID")
            if os.path.isfile(str(rudder_PID_file)):
                # PID values exist is file
                df_pid = pd.read_csv(rudder_PID_file, delim_whitespace=True)
                self.K = np.array(df_pid.iloc[0])
                print("Loading rudder PID from file - {}".format(rudder_PID_file))
                print("P,I,D = ", self.K)
            else:
                # PID values not provided and does not exist in file.
                initial_PIDs = []
                for P in [1,2]:
                    for I in [1,2]:
                        for D in [1,2]:
                            initial_PIDs.append([P,I,D])
                initial_PIDs = tqdm(initial_PIDs, leave=False)
                initial_PIDs.set_description("Tuning iterations")
                for initial_PID in initial_PIDs:
                    P,I,D = initial_PID
                    self.K = np.array([P,I,D])
                    self._tune_controller()

    def __relative_angle(self, asv, waypoint):
        theta = None
        p1 = asv.py_get_position_origin()
        p2 = asv.py_get_position_cog()
        p3 = waypoint
        # theta = math.atan2((p3.y-p1.y), (p3.x-p1.x)) - math.atan2((p2.y-p1.y), (p2.x-p1.x))
        long1, lat1 = epsg.PCS_to_GCS(p1.x, p1.y)
        long2, lat2 = epsg.PCS_to_GCS(p2.x, p2.y)
        long3, lat3 = epsg.PCS_to_GCS(p3.x, p3.y)
        geodesic = pyproj.Geod(ellps='WGS84')
        fwd_azimuth_1, back_azimuth_1, distance_1 = geodesic.inv(long1, lat1, long2, lat2)
        # fwd_azimuth_1 = fwd_azimuth_1 if fwd_azimuth_1 >= 0.0 else (360 + fwd_azimuth_1)
        fwd_azimuth_2, back_azimuth_2, distance_2 = geodesic.inv(long1, lat1, long3, lat3)
        # fwd_azimuth_2 = fwd_azimuth_2 if fwd_azimuth_2 >= 0.0 else (360 + fwd_azimuth_2)
        theta = (fwd_azimuth_2 - fwd_azimuth_1) * math.pi/180
        return theta

    def get_rudder_angle(self, asv, waypoint):
        # Compute the relative angle between the vehicle heading and the waypoint.
        theta = self.__relative_angle(asv, waypoint)
        # Set error as the difference of the current heading and the desired heading.
        self.previous_error = self.error
        self.error = theta
        gamma = 0.7 # Rate at which the past errors reduces.
        self.cumulative_error = self.error + gamma*self.cumulative_error
        self.delta_error = self.error - self.previous_error
        # Compute the rudder angle
        E = np.array([self.error, self.cumulative_error, self.delta_error]) # P, I, D errors.
        phi = np.dot(np.transpose(self.K), E) # radians because error is in radians.
        # Limit the rudder angle within the range (-PI/6, PI/6)
        if phi > self.max_rudder_angle:
            phi = self.max_rudder_angle
        elif phi < -self.max_rudder_angle:
            phi = -self.max_rudder_angle
        return phi # radians

    def _simulate_asv_in_sea_state(self, sea_state_and_PID):
        significant_wave_ht, asv_heading, P, I, D = sea_state_and_PID
        long, lat = (-150.0, 20)
        x, y = epsg.GCS_to_PCS(long, lat)
        start_point = py_Coordinates_3D(x, y, 0)
        long, lat = (-150.0, 20.01)
        x, y = epsg.GCS_to_PCS(long, lat)
        waypoint = py_Coordinates_3D(x, y, 0)
        # Init waves
        rand_seed = 1
        count_component_waves = 21
        wave = py_Sea_surface(significant_wave_ht, 0.0, rand_seed, count_component_waves)
        # Init ASV
        time_step_size = 40 # milli sec
        attitude = py_Coordinates_3D(0.0, 0.0, asv_heading)
        asv = py_Asv(self.asv_spec, wave, start_point, attitude)
        thrust_tuning_factor = 0.03 # The thrust tuning factor is an assumed and, for controller tuning, thrust tuning factor
                                    # is assumed as a constant for all sea states.
        asv.py_wg_set_thrust_tuning_factor(thrust_tuning_factor)
        # Init controller
        controller = Rudder_PID_controller(self.asv_spec, [P,I,D])
        # Simulate
        time = 0.0
        max_time = 60 # sec
        heading_error = 0
        while time < max_time:
            time += time_step_size/1000.0
            # Compute the dynamics
            rudder_angle = controller.get_rudder_angle(asv, waypoint)
            asv.py_wg_compute_dynamics(rudder_angle, time_step_size)
            # Compute the error in heading
            error = self.__relative_angle(asv, waypoint)
            heading_error += error*error
        # Root mean square error
        rms_error = math.sqrt(heading_error/(max_time * (1000/time_step_size)))
        return rms_error

    def _tune_controller(self):
        f = open("{}/{}_{}_{}.txt".format(self.out_dir, self.K[0], self.K[1], self.K[2]), "w")
        f.write("P I D error_avg error_std\n")
        pool = mp.Pool(mp.cpu_count()) # Create a pool of processes to run in parallel.
        delta = 0.25
        P_current, I_current, D_current = list(self.K)
        # Policy Gradient Reinforcement Learning
        iterations = tqdm(range(10), leave=False) # This is going to take some time, therefore show a progress bar.
        iterations.set_description("Policy iterations")
        for n in iterations:
            costs = []
            PIDs = []
            for P in [P_current-delta, P_current, P_current+delta]:
                for I in [I_current-delta, I_current, I_current+delta]:
                    for D in [D_current-delta, D_current, D_current+delta]:
                        PIDs.append([P,I,D])
            PIDs = tqdm(PIDs, leave=False)
            PIDs.set_description("Controller variants")
            for PID in PIDs:
                P,I,D = PID
                sea_states_and_PID = []
                for significant_wave_ht in np.arange(1.0, 10.0, 2.0):
                    for asv_heading in np.arange(0.0, 360.0, 45.0):
                        sea_states_and_PID.append([significant_wave_ht, asv_heading * math.pi/180, P, I, D])
                results = []
                for result in pool.imap_unordered(self._simulate_asv_in_sea_state, sea_states_and_PID): # Run multiple simulations in parallel
                    results.append(result) # append the return for each call to self._simulate_asv_in_sea_state to the list.
                # Compute cost for each combination of PID:
                costs.append([P, I, D, np.average(np.array(results))])
            # Compute the next set of PID terms
            costs = np.array(costs)
            f.write("{} {} {} {} {}\n".format(  P_current,
                                                I_current,
                                                D_current,
                                                np.average(costs, axis=0)[-1],
                                                np.std(costs, axis=0)[-1]))
            def compute_average_cost(index, K):
                mask = []
                for item in costs:
                    mask_value = True if item[index] == K else False
                    mask.append(mask_value)
                average_cost = costs[mask].mean(axis=0)[-1]
                return average_cost
            # Compute the average performance for all cases with P_current-delta
            avg_costs_P_minus = compute_average_cost(0, P_current-delta)
            # Compute the average performance for all cases with P_current
            avg_costs_P = compute_average_cost(0, P_current)
            # Compute the average performance for all cases with P_current+delta
            avg_costs_P_plus = compute_average_cost(0, P_current+delta)
            # Compute the average performance for all cases with I_current-delta
            avg_costs_I_minus = compute_average_cost(1, I_current-delta)
            # Compute the average performance for all cases with I_current
            avg_costs_I = compute_average_cost(1, I_current)
            # Compute the average performance for all cases with I_current+delta
            avg_costs_I_plus = compute_average_cost(1, I_current+delta)
            # Compute the average performance for all cases with D_current-delta
            avg_costs_D_minus = compute_average_cost(2, D_current-delta)
            # Compute the average performance for all cases with D_current
            avg_costs_D = compute_average_cost(2, D_current)
            # Compute the average performance for all cases with D_current+delta
            avg_costs_D_plus = compute_average_cost(2, D_current+delta)
            # Compute the Adjustment vector.
            A = [0,0,0]
            # Adjustment for P
            min_costs_P = min(avg_costs_P_minus, avg_costs_P, avg_costs_P_plus)
            if min_costs_P == avg_costs_P_minus:
                A[0] = -1
            elif min_costs_P == avg_costs_P:
                A[0] = 0
            else:
                A[0] = 1
            # Adjustment for I
            min_costs_I = min(avg_costs_I_minus, avg_costs_I, avg_costs_I_plus)
            if min_costs_I == avg_costs_I_minus:
                A[1] = -1
            elif min_costs_I == avg_costs_I:
                A[1] = 0
            else:
                A[1] = 1
            # Adjustment for D
            min_costs_D = min(avg_costs_D_minus, avg_costs_D, avg_costs_D_plus)
            if min_costs_D == avg_costs_D_minus:
                A[2] = -1
            elif min_costs_D == avg_costs_D:
                A[2] = 0
            else:
                A[2] = 1
            # Compute the new gain terms
            A = np.array(A)
            K_current = np.array([P_current, I_current, D_current])
            K_current = K_current + A*delta
            P_current, I_current, D_current = list(K_current)
            self.K = K_current
        f.close()

if __name__ == '__main__':
    import cProfile
    # Wave glider specs
    asv_spec = py_Asv_specification()
    asv_spec.L_wl       = 2.1 # m
    asv_spec.B_wl       = 0.6 # m
    asv_spec.D          = 0.25 # m
    asv_spec.T          = 0.09 # m
    asv_spec.max_speed  = 4.0 # m/s
    asv_spec.disp       = 0.09 # m3
    asv_spec.r_roll     = 0.2 # m
    asv_spec.r_pitch    = 0.6 # m
    asv_spec.r_yaw      = 0.6 # m
    asv_spec.cog        = py_Coordinates_3D(1.05, 0.0, -3.0) # m
    rudder_controller   = Rudder_PID_controller(asv_spec) # Will also tune the controller.


import cv2
import numpy as np
import matplotlib.pyplot as plt
import os
import csv
import pandas as pd
from PIL import Image

#root = '/home/miplab/data/Kaggle_Eyepacs/train/train_full'
#save_path = '/home/miplab/data/Kaggle_Eyepacs/train/train_full_CLAHE'
#annotations_path = '/home/miplab/data/Kaggle_Eyepacs/train/trainLabels.csv'

root = '/home/miplab/data/Kaggle_Eyepacs/EyeQ/EyeQ_dr/original/good_only'
save_path = '/home/miplab/data/Kaggle_Eyepacs/EyeQ/EyeQ_dr/CLAHE/good_only'
#annotations_path = '/home/miplab/data/Kaggle_Eyepacs/test/retinopathy_solution.csv'

#anno_df = pd.read_csv(annotations_path)
#print(anno_df)

#length = len(anno_df.index)

#for x,y,indx in zip(anno_df['image'], anno_df['level'], range(length)):
 #   x = x+".jpeg"

for path,dirs, files in os.walk(root):

    if path.endswith('original'): # skip the root directory

        continue
    elif path.endswith("filtered"):

        continue
    elif path.endswith("full"):

        continue
    elif path.endswith("good_only"):
        continue

    new_path = save_path + path[64:]

    if os.path.exists(new_path):
        indx =0
        length = len(files)
        for x in files:
            file_path = os.path.join(path, x)
            if os.path.isfile(file_path):

                image= cv2.imread(file_path)
                image = cv2.resize(image, (600,600))

                image_bw =  cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
#declatration of CLAHE

                clahe = cv2.createCLAHE(clipLimit = 10)

                final_img = clahe.apply(image_bw)

                cv2.imwrite(os.path.join(new_path, x), final_img)
                indx+=1
            print("{} out of {}".format(indx, length))

    else:
        os.mkdir(new_path)


from abc import ABCMeta

from nlpype.objects import cache
from nlpype.objects.core_token import CoreToken

class HasTokens(metaclass=ABCMeta):
    """
    Module for CoreNLP objects that have tokens
    """
    def __getitem__(self, index):
        """
        Accesses a token by index

        :param index: The index of the token
        :type index: int
        :return: The token at the specified index
        :rtype: CoreToken
        """
        return self.tokens()[index]

    def __setitem__(self, index, word):
        """
        Sets the text of the token(s) at the given index

        :param index: Either an index of a token or a slice of tokens to set
        :type index: Union[int, slice]
        :param word: The text to set the first token in the range to
        :type word: str
        """
        if isinstance(index, slice):
            start = index.start
            end = index.stop

            if start is None:
                start = 0
            if end is None:
                end = len(self)
        else:
            start = end = index

        self[start].text = word

        for token in self[start+1:end]:
            token.text = ''
        for token in self[start:end-1]:
            token.no_space()

    @cache
    def tokens(self):
        """
        Retrieves the tokens contained in this object

        :rtype: list of CoreToken
        """
        return [
            CoreToken(token, self._pipeline)
            for token in self._base.tokens()
        ]

    def __len__(self):
        return len(self.tokens())


import os
import time
import torch
from argparse import ArgumentParser
from MemSE.nas import DataloadersHolder, ResNetArchEncoder
from MemSE.training import RunManager, RunConfig
from ofa.model_zoo import ofa_net
from MemSE.nn import OFAxMemSE, FORWARD_MODE, MemSE
from MemSE import ROOT

parser = ArgumentParser()
parser.add_argument("--datapath", default=os.environ.get("DATASET_STORE", None))
args, _ = parser.parse_known_args()

device = torch.device("cuda:0") if torch.cuda.is_available() else torch.device("cpu")

ofa = ofa_net('ofa_resnet50', pretrained=True)
ofa.set_max_net()
default_gmax = MemSE(ofa.get_active_subnet()).quanter.Wmax
ofaxmemse = OFAxMemSE(ofa)
encoder = ResNetArchEncoder(default_gmax)

run_config = RunConfig(dataset_root=args.datapath, dataset='ImageNetHF')
run_manager = RunManager(run_config, mode=FORWARD_MODE.MONTECARLO)
datahld = DataloadersHolder(run_manager)

train_ld, eval_ld = datahld.get_image_size(128)

for i in range(10):
    print('Warming up ', i)
    a = encoder.random_sample_arch(ofaxmemse)
    a['image_size'] = 128
    ofaxmemse.set_active_subnet(a, train_ld)
torch.cuda.synchronize()

@torch.no_grad()
def eval(nb_batch, power=1):
    a = encoder.random_sample_arch(ofaxmemse)
    a['image_size'] = 128
    ofaxmemse.set_active_subnet(a, train_ld)
    ofaxmemse.quant(scaled=False)
    metrics = None
    if nb_batch > 0:
        _, metrics = run_manager.validate(
                            net=ofaxmemse,
                            data_loader=eval_ld,
                            no_logs=True,
                            nb_batchs=nb_batch,
                            nb_batchs_power=power
                        )
        metrics.display_summary()
    ofaxmemse.unquant()
    return metrics

res = {0: {}, 1: {}}
for n in range(150):
    for p in [0, 1]:
        print(n, p)
        times = []
        for _ in range(5):
            torch.cuda.synchronize()
            start_t = time.time()
            eval(n, power=p)
            torch.cuda.synchronize()
            elapsed = time.time() - start_t
            times.append(elapsed)
        res[p][n] = sum(times) / 5
        print(res[p][n])
torch.save(res, ROOT/ "experiments/conference_2/results/overhead.pth")


import sys

import requests
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.options import Options

def get_all_ids_ujz(category="most-popular", page_number="1"):
    url = "https://www.youjizz.com/" + category + "/" + page_number + ".html"

    headers = {
        'Cookie': 'commentPhrase=cllTWHVFd1ZzckYvVTRKd1ZZc1BWQ296clh5RCs0YjAyNTFjZGtQc3pLdz06OkgmihcPsRpzrB6ef53DIQU'
                  '=; RNLBSERVERID=ded6584 '
    }

    response = requests.request("GET", url, headers=headers)
    soup = BeautifulSoup(response.text, 'html.parser')
    a_elements = soup.find_all('a', class_="frame video")
    return [a_element['data-video-id'] for a_element in a_elements]

def get_single_movie_ujz(movie_id):
    url = "https://www.youjizz.com/videos/-" + str(movie_id) + ".html "
    chrome_options = Options()
    chrome_options.add_argument('--headless')
    chrome_options.add_argument('--no-sandbox')
    chrome_options.add_argument('--disable-dev-shm-usage')
    driver = webdriver.Chrome('/usr/bin/chromedriver', options=chrome_options)
    driver.get(url)
    # get the page source
    page_source = driver.page_source
    driver.close()
    # parse the HTML
    soup = BeautifulSoup(page_source, "html.parser")
    tags = soup.find('meta', {'name': 'keywords'})['content'].split(' , ')
    description = soup.find('meta', {'name': 'description'})['content']
    try:
        video_favorite = soup.find("input", {'id': "checkVideoFavorite"})
        views = video_favorite['data-views']
        rating = video_favorite['data-rating']
    except:
        views = 0
        rating = 0
    scripts = soup.find_all("script")
    image_url = "http:" + soup.find('meta', {'property': 'og:image'})['content']
    for script in scripts:
        if str(script).__contains__('var dataEncodings'):
            script = str(script)
            data_encodings = script.split('}];')[0]
            data_encodings = data_encodings.replace('<script>', '').replace(
                'var dataEncodings = ', '').replace('\n', '').replace(' ', '') + "}]"
            # data_encodings_str = data_encodings.split('dataEncodings =')[1] + "}]"
            # all_movie_data = list(eval(data_encodings_str))
            # download_links = [
            #     {'title': movie_data['name'],
            #      'link': ["https:" + movie_data['filename'].replace('\\', '')],
            #      'subtitle': '',
            #      'quality': movie_data['quality']}
            #     for movie_data in all_movie_data
            # ]
            return {'name': soup.find('title').text,
                    'farsi_name': '+18 کوس کوص کون کیر سکس سکسی پورن سوپر جنده لزبین ',
                    'description': description,
                    'views': views,
                    'url': url,
                    'rating': rating,
                    'movie_id': movie_id,
                    'tags': tags,  # type of this field is Array
                    'image': image_url,
                    'download_links': data_encodings
                    }

# if __name__ == '__main__':
#     for page_number in range(1, 100):
#         all_ids = get_all_ids_ujz(page_number=str(page_number))
#         print(all_ids)
#         for movie_id in all_ids:
#             print(get_single_movie_ujz(movie_id))


import os
import time
import cv2
import itertools
import numpy as np
from tqdm import tqdm

import matplotlib.pyplot as plt

from tensorflow.keras.optimizers import Adam
import tensorflow.keras.applications.inception_v3 as inception_v3
import tensorflow.keras.applications.inception_resnet_v2 as inception_resnet_v2
import tensorflow.keras.applications.densenet as densenet
import tensorflow.keras.applications.mobilenet_v2 as mobilenet_v2
import tensorflow.keras.applications.mobilenet as mobilenet
import tensorflow.keras.applications.resnet50 as resnet50
import tensorflow.keras.applications.vgg16 as vgg16
import tensorflow.keras.applications.vgg19 as vgg19
import tensorflow.keras.applications.xception as xception

from tensorflow.keras.models import Model
from tensorflow.keras.layers import Dense, GlobalAveragePooling2D, Input
from tensorflow.keras.layers import Lambda

from tensorflow.keras.constraints import NonNeg

from tensorflow.keras.layers import Activation

from tensorflow.keras import backend as K
from tensorflow.keras.preprocessing import image

def loadimgs(path, shape_img=(595, 842)):
    '''
    path => Path of train directory or test directory
    '''

    X = []
    y = []

    for cartorio in os.listdir(path):
        print("Carregando Cartorio: " + cartorio)
        cartorio_path = os.path.join(path, cartorio)
        for filename in os.listdir(cartorio_path):
            image_path = os.path.join(cartorio_path, filename)
            img = image.load_img(image_path, target_size=shape_img)
            img = image.img_to_array(img)
            img = img[:int(img.shape[0] * 1/3.), :, :]
            try:
                X.append(img)
                y.append(cartorio)

            except Exception as e:
                print(e)

    y = np.vstack(y)
    X = np.stack(X)

    return X, y

def gaussian(x):
    return K.exp(-K.pow(x, 2))

def get_siamese_model(name=None, input_shape=(224, 224, 3),
                      embedding_vec_size=512, not_freeze_last=2):
    """
        Model architecture
    """

    if name == "InceptionV3":

        base_model = inception_v3.InceptionV3(

            weights='imagenet', include_top=False)

        model_preprocess_input = inception_v3.preprocess_input

    if name == "InceptionResNetV2":

        base_model = inception_resnet_v2.InceptionResNetV2(

            weights='imagenet', include_top=False)

        model_preprocess_input = inception_resnet_v2.preprocess_input

    if name == "DenseNet121":

        base_model = densenet.DenseNet121(

            weights='imagenet', include_top=False)

        model_preprocess_input = densenet.preprocess_input

    if name == "DenseNet169":

        base_model = densenet.DenseNet169(

            weights='imagenet', include_top=False)

        model_preprocess_input = densenet.preprocess_input

    if name == "DenseNet201":

        base_model = densenet.DenseNet201(

            weights='imagenet', include_top=False)

        model_preprocess_input = densenet.preprocess_input

    if name == "MobileNetV2":

        base_model = mobilenet_v2.MobileNetV2(

            weights='imagenet', include_top=False)

        model_preprocess_input = mobilenet_v2.preprocess_input

    if name == "MobileNet":

        base_model = mobilenet.MobileNet(

            weights='imagenet', include_top=False)

        model_preprocess_input = mobilenet.preprocess_input

    if name == "ResNet50":

        base_model = resnet50.ResNet50(

            weights='imagenet', include_top=False)

        model_preprocess_input = resnet50.preprocess_input

    if name == "VGG16":

        base_model = vgg16.VGG16(

            weights='imagenet', include_top=False)

        model_preprocess_input = vgg16.preprocess_input

    if name == "VGG19":

        base_model = vgg19.VGG19(

            weights='imagenet', include_top=False)

        model_preprocess_input = vgg19.preprocess_input

    if name == "Xception":

        base_model = xception.Xception(

            weights='imagenet', include_top=False)

        model_preprocess_input = xception.preprocess_input

    # Verifica se existe base_model

    if 'base_model' not in locals():

        return ["InceptionV3", "InceptionResNetV2",

                "DenseNet121", "DenseNet169", "DenseNet201",

                "MobileNetV2", "MobileNet",

                "ResNet50",

                "VGG16", "VGG19",

                "Xception"

                ]

    # desativando treinamento

    for layer in base_model.layers[:-not_freeze_last]:

        layer.trainable = False

    x = base_model.layers[-1].output

    x = GlobalAveragePooling2D()(x)

    x = Dense(

        embedding_vec_size,

        activation='linear',  # sigmoid? relu?

        name='embedding',

        use_bias=False

    )(x)

    model = Model(

        inputs=base_model.input,

        outputs=x

    )

    left_input = Input(input_shape)

    right_input = Input(input_shape)

    # Generate the encodings (feature vectors) for the two images

    encoded_l = model(left_input)

    encoded_r = model(right_input)

    # Add a customized layer to compute the absolute difference between the encodings

    L1_layer = Lambda(lambda tensors: K.abs(tensors[0] - tensors[1]))

    L1_distance = L1_layer([encoded_l, encoded_r])

    # Add a dense layer with a sigmoid unit to generate the similarity score

    prediction = Dense(

        1,

        activation=Activation(gaussian),

        use_bias=False,

        kernel_constraint=NonNeg()

    )(L1_distance)

    # Connect the inputs with the outputs

    siamese_net = Model(

        inputs=[left_input, right_input],

        outputs=prediction

    )

    return {

        "model": siamese_net,

        "preprocess_input": model_preprocess_input

    }

# In[5]:

train_folder = "dataset/train/"

val_folder = 'dataset/test/'

save_path = 'model_data/'

# In[6]:

X, y = loadimgs(train_folder)

# In[7]:

Xval, yval = loadimgs(val_folder)

# In[8]:

def show_img(img):

    plt.figure(figsize=(20, 7))

    plt.imshow(img/255, aspect='auto', interpolation='nearest')

# In[9]:

show_img(X[10])

# In[10]:

model_name = "MobileNetV2"

# In[11]:

model_dict = get_siamese_model(model_name, X[0].shape)

model = model_dict["model"]

preprocess_input = model_dict["preprocess_input"]

# In[12]:

model.summary()

# In[13]:

X = preprocess_input(X)

# In[14]:

X.shape

# In[15]:

def get_index(list1, list2):

    comb = list(

        itertools.product(

            enumerate(list1),

            enumerate(list2)

        )

    )

    y = np.array([int(c[0][1][0] == c[1][1][0]) for c in comb])

    idx_left = np.array([c[0][0] for c in comb])

    idx_right = np.array([c[1][0] for c in comb])

    return y, idx_left, idx_right

# In[16]:

def get_batch(X, y, batch_size, proportion=0.5):

    n_examples, width, height, depth = X.shape

    y_, idx_left, idx_right = get_index(y, y)

    idx_one = np.random.choice(

        np.where(y_ == 1)[0],

        int(batch_size * proportion)

    ).tolist()

    idx_zero = np.random.choice(

        np.where(y_ == 0)[0],

        int(batch_size * (1-proportion))

    ).tolist()

    sel_idx = idx_one + idx_zero

    np.random.shuffle(sel_idx)

    y_batch = y_[sel_idx]

    X_batch_l = X[idx_left[sel_idx]]

    X_batch_r = X[idx_right[sel_idx]]

    return [X_batch_l, X_batch_r], y_batch

# In[17]:

optimizer = Adam(lr=0.001)

model.compile(loss="binary_crossentropy", optimizer=optimizer)

# In[18]:

batch_size = 64

n_epochs = 20

proportion = 0.3

# In[19]:

# train model on each dataset

for epoch in tqdm(range(n_epochs)):

    X_train, y_train = get_batch(X, y, batch_size, proportion)

    model.fit(X_train, y_train, batch_size=batch_size, epochs=5)

# In[ ]:

# In[20]:

model.save(model_name + "_siamese.h5")

# In[26]:

model_embedding = model.layers[2]

# In[21]:

model.predict([Xval[0:4], Xval[15:19]])

# In[22]:

show_img(Xval[3])

# In[23]:

show_img(Xval[18])

# In[24]:

model.predict([Xval[0:4], Xval[0:4]])

# In[32]:

model.summary()

# In[36]:

model2 = Model(inputs=model.input, outputs=model.layers[-1].output)

# In[39]:

model2.predict([Xval[0:4], Xval[0:4]]) + 0.5


MINUS_INFINITY = -10000000000
class Heap:

    def __init__(self, A):
        self.size = A[0]
        self.heap = [x for x in A]  # copy

    def swap(self, x, y):
        t = self.heap[x]
        self.heap[x] = self.heap[y]
        self.heap[y] = t

    def left(self, i):
        return i * 2

    def right(self, i):
        return i * 2 + 1

    def parent(self, i):
        return i / 2

    def heapify(self, i):
        largest = i
        l = self.left(i)
        r = self.right(i)

        if l <= self.heap[0] and self.heap[l] > self.heap[i]:
            largest = l

        if r <= self.heap[0] and self.heap[r] > self.heap[largest]:
            largest = r

        if largest != i:
            self.swap(i, largest)
            self.heapify(largest)
        else:
            return

    def buildMaxHeap(self):
        for i in range(int(self.heap[0] / 2), 0, -1):
            print(i)
            self.heapify(i)

    # increase heap[i] to key
    def increasedKey(self, i, key):
        if key < self.heap[i]:
            print("Error: increased key is less than the original one")
            return

        # i should > 1
        while i > 1 and self.heap[i] < self.heap[i / 2]:
            self.swap(i, i /2)
            i = i / 2

    def insert(self, x):
        self.heap[0] += 1 # size increased
        self.heap.append(MINUS_INFINITY)
        self.increasedKey(self.heap[0], x)  # increase key

A = [10, 2, 4, 1, 9, 0, 3, 7, 8, 6, 5]
h = Heap(A)
h.buildMaxHeap()
print(h.heap)


from typing import Any, Dict, Optional

import httpx

from ...client import Client
from ...models.mediation_grant import MediationGrant
from ...types import Response

def _get_kwargs(
    mediation_id: str,
    *,
    client: Client,
) -> Dict[str, Any]:
    url = "{}/mediation/requests/{mediation_id}/grant".format(client.base_url, mediation_id=mediation_id)

    headers: Dict[str, str] = client.get_headers()
    cookies: Dict[str, Any] = client.get_cookies()

    return {
        "method": "post",
        "url": url,
        "headers": headers,
        "cookies": cookies,
        "timeout": client.get_timeout(),
    }

def _parse_response(*, response: httpx.Response) -> Optional[MediationGrant]:
    if response.status_code == 201:
        response_201 = MediationGrant.from_dict(response.json())

        return response_201
    return None

def _build_response(*, response: httpx.Response) -> Response[MediationGrant]:
    return Response(
        status_code=response.status_code,
        content=response.content,
        headers=response.headers,
        parsed=_parse_response(response=response),
    )

def sync_detailed(
    mediation_id: str,
    *,
    client: Client,
) -> Response[MediationGrant]:
    """Grant received mediation

    Args:
        mediation_id (str):

    Returns:
        Response[MediationGrant]
    """

    kwargs = _get_kwargs(
        mediation_id=mediation_id,
        client=client,
    )

    response = httpx.request(
        verify=client.verify_ssl,
        **kwargs,
    )

    return _build_response(response=response)

def sync(
    mediation_id: str,
    *,
    client: Client,
) -> Optional[MediationGrant]:
    """Grant received mediation

    Args:
        mediation_id (str):

    Returns:
        Response[MediationGrant]
    """

    return sync_detailed(
        mediation_id=mediation_id,
        client=client,
    ).parsed

async def asyncio_detailed(
    mediation_id: str,
    *,
    client: Client,
) -> Response[MediationGrant]:
    """Grant received mediation

    Args:
        mediation_id (str):

    Returns:
        Response[MediationGrant]
    """

    kwargs = _get_kwargs(
        mediation_id=mediation_id,
        client=client,
    )

    async with httpx.AsyncClient(verify=client.verify_ssl) as _client:
        response = await _client.request(**kwargs)

    return _build_response(response=response)

async def asyncio(
    mediation_id: str,
    *,
    client: Client,
) -> Optional[MediationGrant]:
    """Grant received mediation

    Args:
        mediation_id (str):

    Returns:
        Response[MediationGrant]
    """

    return (
        await asyncio_detailed(
            mediation_id=mediation_id,
            client=client,
        )
    ).parsed


# -*- coding: utf-8 -*-
import scrapy
import sqlite3
from ..items import HvgarticleItem

class HvgarticlesSpider(scrapy.Spider):
    name = 'hvgarticles'
    allowed_domains = ['hvg.com']
    conn = sqlite3.connect(r'C:\Users\Athan\OneDrive\Documents\Dissertation\Python\webscraperorigo\url.db')
    curr = conn.cursor()
    urls=[]
    curr.execute("""SELECT url FROM 'hvgUrl_tb' WHERE
                 url LIKE "%201801%" OR
                 url LIKE "%201802%" OR
                 url LIKE "%201803%" OR
                 url LIKE "%201804%" OR
                 url LIKE "%201805%" OR
                 url LIKE "%2017%" OR
                 url LIKE "%2016%" OR
                 url LIKE "%2015%" OR
                 url LIKE "%201405%" or
                 url LIKE "%201406%" or
                 url LIKE "%201407%" or
                 url LIKE "%201408%" or
                 url LIKE "%201409%" or
                 url LIKE "%201410%" or
                 url LIKE "%201411%" or
                 url LIKE "%201412%" ORDER BY url;
                 """)
    for row in curr.fetchall():
        urlrow = str(row)
        urlrow = urlrow.replace('(',"")
        urlrow = urlrow.replace(')',"")
        urlrow = urlrow.replace("'","")
        urlrow = urlrow.replace(',',"")
        urls.append(urlrow)
    start_urls = urls

    def parse(self, response):
        items = HvgarticleItem()
        text = ['']
        connections = ['']
        tags = ['']
        start_url = ['']
        p = response.css(".entry-summary p::text,.entry-summary  p a::text, .entry-summary p a em::text,.entry-content p::text,.entry-content p a::text, .entry-content p a em::text").extract()
        connection = response.css(".entry-summary p a, .entry-summary p a em,.entry-content p,.entry-content p a, .entry-content p a em").xpath("@href").extract()
        tag = response.css(".article-tags .tag::text").extract()
        start_url[0] = response.request.url
        for paragaph in p:
            text[0] += " " + paragaph
        for c in connection:
            connections[0] += " " + c
        for t in tag:
            tags[0] += " " + t

        items['paragaph'] = text
        items['tags'] = tags
        items['connections'] = connections
        items['start_url'] = start_url

        yield items


__author__ = 'piyush'

k = int(input())
x = []
z = []
for i in range(k):
    x.append([int(i) for i in input().split()])
    z.append(int(input()))
for i in range(k):
    sum = 0
    a = list(str(z[i]))
    total = 0
    for j in range((x[i][0]-x[i][1])+1):
        temp = 1
        for l in range(j,j+x[i][1]):
            temp *= int(a[l])
        if total < temp:
            total = temp
    sum += total
    print(sum)


import torch
import torch.nn as nn

# Import the skrl components to build the RL system
from skrl.models.torch import Model, GaussianMixin, DeterministicMixin
from skrl.memories.torch import RandomMemory
from skrl.agents.torch.ppo import PPO, PPO_DEFAULT_CONFIG
from skrl.resources.schedulers.torch import KLAdaptiveRL
from skrl.resources.preprocessors.torch import RunningStandardScaler
from skrl.trainers.torch import SequentialTrainer
from skrl.envs.torch import wrap_env
from skrl.envs.torch import load_omniverse_isaacgym_env
from skrl.utils import set_seed
from learning.model import StochasticActorHeightmap, DeterministicHeightmap, ObserverationInfo, NetworkInfo
import hydra
from omegaconf import DictConfig
from hydra import compose, initialize
import wandb
import datetime
from omniisaacgymenvs.tasks.utils.camera.heightmap_distribution import Heightmap
from omniisaacgymenvs.tasks.utils.terrains.extract_terrain import extract_terrain
import os
#cfg_ppo = PPO_DEFAULT_CONFIG.copy()

# set the seed for reproducibility
set_seed(42)

class TrainerSKRL():
    def __init__(self):
        self._load_cfg()
        time_str = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
        #self.wandb_group =f"Improved_{time_str}"
        self.wandb_group ="New"
        #self.wandb_name = f"run_{time_str}"
        self.wandb_name = "Run1"
       # self.start_simulation()
        #self.start_training()

    def _load_cfg(self):
        initialize(config_path="cfg", job_name="test_app")
        cfg = compose(config_name="config")

        self.cfg_ppo = PPO_DEFAULT_CONFIG.copy()
        self.cfg_network = cfg.trainSKRL.network
        self.cfg_experiment = cfg.trainSKRL.experiment
        self.cfg_config = cfg.trainSKRL.config
        self.sim_params = cfg.task
        # Set all parameters according to cfg file
        for param, value in (cfg.trainSKRL.config).items():
            self.cfg_ppo[param] = value

        print(self.cfg_ppo)
        hydra.core.global_hydra.GlobalHydra.instance().clear()

    def start_simulation(self):
        env = load_omniverse_isaacgym_env(task_name="Rover")
        self.env = wrap_env(env)

    def log_parameters(self):
        config = {                    "mlp layers": self.cfg_network.mlp.layers,
                    "mlp_activation": self.cfg_network.mlp.activation,
                    "encoder_layers": self.cfg_network.encoder.layers,
                    "encoder_activation": self.cfg_network.encoder.activation,
                    "hyperparameters": self.cfg_config,
                    "rewards": self.sim_params.rewards,
                    "sim parameters": {"environment": self.sim_params.env,
                                        "simulation": self.sim_params.sim,},
                    }

        # for key,value in (self.cfg_config).items():
        #     config[key] = value
        return config
    def train(self,sweep=False):
        env = self.env
        device = env.device
        if not sweep:
            self.log_parameters()

        # Instantiate a RandomMemory as rollout buffer (any memory can be used for this)
        memory = RandomMemory(memory_size=60, num_envs=self.env.num_envs, device=device)

        # Get values from cfg
        mlp_layers = self.cfg_network.mlp.layers
        encoder_layers = self.cfg_network.encoder.layers
        activation_function = self.cfg_network.mlp.activation
        #print(env.num_exteroceptive)
        #TODO fix
        heightmap_distribution = Heightmap()
        num_sparse = heightmap_distribution.get_num_sparse_vector()
        num_dense = heightmap_distribution.get_num_dense_vector()
        num_beneath = heightmap_distribution.get_num_beneath_vector()
        networkInfo = NetworkInfo([256,160,128],[80,60],[80,60],[80,60],"leakyrelu")
        observerationInfo = ObserverationInfo(4,num_sparse,num_dense,num_beneath)

        # Instantiate the agent's models (function approximators).
        models_ppo = {  "policy": StochasticActorHeightmap(env.observation_space, env.action_space, networkInfo, observerationInfo ),
                    "value": DeterministicHeightmap(env.observation_space, env.action_space, networkInfo,observerationInfo)}

        # print()

        # Instantiate parameters of the model
        # for model in models_ppo.values():
        #     model.init_parameters(method_name="normal_", mean=0.0, std=0.05)

        self.cfg_ppo["experiment"]["write_interval"] = 100
        # Define agent
        agent = PPO(models=models_ppo,
                memory=memory,
                cfg=self.cfg_ppo,
                observation_space=env.observation_space,
                action_space=env.action_space,
                device=device)
        #agent.migrate("best.pt")
        #agent.load("agent_219000.pt")
        #agent.load("agent_13000.pt")
        #agent.load("agent_939000.pt")
        # Configure and instantiate the RL trainer
        cfg_trainer = {"timesteps": 1000000, "headless": True}
        trainer = SequentialTrainer(cfg=cfg_trainer, env=env, agents=agent)

        # start training
        #trainer.eval()
        trainer.train()

    def start_training_sweep(self,n_sweeps):
        self.start_simulation()
       # Define sweep config
        sweep_configuration = {
            'method': 'bayes',
            'name': 'sweep',
            'metric': {'goal': 'maximize', 'name': 'Reward / Total reward (mean)'},
            'parameters':
            {
                'mini_batches': {'values': [4, 8]},
                'lr': {'max': 0.003, 'min': 0.00003}

            }
        }

        # Initialize sweep by passing in config. (Optional) Provide a name of the project.
        sweep_id = wandb.sweep(sweep=sweep_configuration, project='isaac-rover-2.0')
        wandb.agent(sweep_id, function=self.sweep, count=n_sweeps)
        # Start sweep job.

    def sweep(self):
        time_str = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
        #self.wandb_name = f"test-Anton_{time_str}"
        self.wandb_name = "no_dense_encoder"
        run = wandb.init(project='isaac-rover-2.0', sync_tensorboard=True,name=self.wandb_name,group=self.wandb_group, entity="aalborg-university")
        self.cfg_ppo["learning_rate"] = wandb.config.lr
        self.cfg_ppo["mini_batches"] = wandb.config.mini_batches
        self.train()
       # wandb.finish()

    def start_training(self):
        self.start_simulation()
        config = self.log_parameters()
        log=True
        if log:
            wandb.init(project='isaac-rover-2.0', config=config, sync_tensorboard=True,name=self.wandb_name,group=self.wandb_group, entity="aalborg-university")
        self.train()
        if log:
            wandb.finish()

    def start_training_sequential(self):
        for i in range(3):
            print(i)
            self.train()
        pass
if __name__ == '__main__':
    # Get hyperparameter config
    terrainExist = os.path.exists("tasks/utils/terrain/")
    if not terrainExist:
        extract_terrain()
    trainer = TrainerSKRL()
   # trainer.start_training_sequential()
    #trainer.start_training_sweep(4)
    trainer.start_training()
    #parse_hydra_configs()


n = int(input())
matrix = [[int(i) for i in input().split()] for j in range(n)]
k = int(input())
top, bottom = 0, 0
for i in range(n):
  for j in range(n):
    if j > n - i - 1:
      bottom += matrix[i][j]
    elif j < n - i - 1:
      top += matrix[i][j]
balance = abs(top - bottom)
print('YES' if balance <= k else 'NO', balance, sep = '\n')


import matplotlib.pyplot as plt

def visualize_data(title,ylabel,xlabel):

    plt.scatter(x_train, y_train, marker='x', c='r')

    # Set the title
    plt.title(title)
    # Set the y-axis label
    plt.ylabel(ylabel)
    # Set the x-axis label
    plt.xlabel(xlabel)
    plt.show()


import errno
from tempfile import TemporaryDirectory
from unittest.mock import patch

import escapism
import pytest

import docker
from repo2docker.__main__ import make_r2d
from repo2docker.app import Repo2Docker
from repo2docker.utils import chdir

def test_find_image():
    images = [{"RepoTags": ["some-org/some-repo:latest"]}]

    with patch("repo2docker.docker.docker.APIClient") as FakeDockerClient:
        instance = FakeDockerClient.return_value
        instance.images.return_value = images

        r2d = Repo2Docker()
        r2d.output_image_spec = "some-org/some-repo"
        assert r2d.find_image()

        instance.images.assert_called_with()

def test_dont_find_image():
    images = [{"RepoTags": ["some-org/some-image-name:latest"]}]

    with patch("repo2docker.docker.docker.APIClient") as FakeDockerClient:
        instance = FakeDockerClient.return_value
        instance.images.return_value = images

        r2d = Repo2Docker()
        r2d.output_image_spec = "some-org/some-other-image-name"
        assert not r2d.find_image()

        instance.images.assert_called_with()

def test_image_name_remains_unchanged():
    # if we specify an image name, it should remain unmodified
    with TemporaryDirectory() as src:
        app = Repo2Docker()
        argv = ["--image-name", "a-special-name", "--no-build", src]
        app = make_r2d(argv)

        app.start()

        assert app.output_image_spec == "a-special-name"

def test_image_name_contains_sha1(repo_with_content):
    upstream, sha1 = repo_with_content
    app = Repo2Docker()
    # force selection of the git content provider by prefixing path with
    # file://. This is important as the Local content provider does not
    # store the SHA1 in the repo spec
    argv = ["--no-build", "file://" + upstream]
    app = make_r2d(argv)

    app.start()

    assert app.output_image_spec.endswith(sha1[:7])

def test_local_dir_image_name(repo_with_content):
    upstream, sha1 = repo_with_content
    app = Repo2Docker()
    argv = ["--no-build", upstream]
    app = make_r2d(argv)

    app.start()

    assert app.output_image_spec.startswith(
        "r2d" + escapism.escape(upstream, escape_char="-").lower()
    )

def test_build_kwargs(repo_with_content):
    upstream, sha1 = repo_with_content
    argv = [upstream]
    app = make_r2d(argv)
    app.extra_build_kwargs = {"somekey": "somevalue"}

    with patch.object(docker.APIClient, "build") as builds:
        builds.return_value = []
        app.build()
    builds.assert_called_once()
    args, kwargs = builds.call_args
    assert "somekey" in kwargs
    assert kwargs["somekey"] == "somevalue"

def test_run_kwargs(repo_with_content):
    upstream, sha1 = repo_with_content
    argv = [upstream]
    app = make_r2d(argv)
    app.extra_run_kwargs = {"somekey": "somevalue"}

    with patch.object(docker.DockerClient, "containers") as containers:
        app.start_container()
    containers.run.assert_called_once()
    args, kwargs = containers.run.call_args
    assert "somekey" in kwargs
    assert kwargs["somekey"] == "somevalue"

def test_root_not_allowed():
    with TemporaryDirectory() as src, patch("os.geteuid") as geteuid:
        geteuid.return_value = 0
        argv = [src]
        with pytest.raises(SystemExit) as exc:
            app = make_r2d(argv)
            assert exc.code == 1

        with pytest.raises(ValueError):
            app = Repo2Docker(repo=src, run=False)
            app.build()

        app = Repo2Docker(repo=src, user_id=1000, user_name="jovyan", run=False)
        app.initialize()
        with patch.object(docker.APIClient, "build") as builds:
            builds.return_value = []
            app.build()
        builds.assert_called_once()

def test_dryrun_works_without_docker(tmpdir, capsys):
    with chdir(tmpdir):
        with patch.object(docker, "APIClient") as client:
            client.side_effect = docker.errors.DockerException("Error: no Docker")
            app = Repo2Docker(dry_run=True)
            app.build()
            captured = capsys.readouterr()
            assert "Error: no Docker" not in captured.err

def test_error_log_without_docker(tmpdir, capsys):
    with chdir(tmpdir):
        with patch.object(docker, "APIClient") as client:
            client.side_effect = docker.errors.DockerException("Error: no Docker")
            app = Repo2Docker()

            with pytest.raises(SystemExit):
                app.build()
                captured = capsys.readouterr()
                assert "Error: no Docker" in captured.err


import math

from django.shortcuts import render

# Create your views here.
from django.views import View
from goods.models import *
from django.core.paginator import Paginator
from django.http.response import HttpResponseBase

# 主页显示
class IndexView(View):
    def get(self, request, cid=1, num=1):
        # 所有通过url位置传参传入进来的都需要进行强转，以免抛出异常
        cid = int(cid)
        num = int(num)

        # step1: 查询所有类别信息
        categorys = Category.objects.all().order_by('id')
        # step2: 查询当前类别下的所有信息(给个默认值女装：id=2)
        goodsList = Goods.objects.filter(category_id=cid).order_by('id')
        # step3: 分页，每页显示8条记录
        pager = Paginator(goodsList, 8)
        # step4: 获取当前页的数据
        page_goodList = pager.page(num)
        # step5: 处理首页和尾页(假设一共显示10页，选择页在list正中间)
        begin = (num - int(math.ceil(10.0 / 2)))
        # 首页禁止越界
        if begin < 1:
            begin = 1
        # 尾页禁止越界
        end = begin + 9
        if end > pager.num_pages:
            end = pager.num_pages
        if end <= 10:
            begin = 1
        else:
            begin = end - 9
        pagelist = range(begin, end + 1)
        return render(request, 'index.html',
                      {'categorys': categorys, 'goodList': page_goodList, 'currentCid': cid, 'pagelist': pagelist,
                       'currentNum': num})

# 使用二阶装饰器来构建缓存，实现“猜你喜欢”功能(保存你浏览过的网页的数据) 使用的是LCU
def recommend_view(func):
    def wrapper(detailView, request, goodsid, *args, **kwargs):
        # 将存放在cookie中的goodsId获取
        cookie_str = request.COOKIES.get('recommend', '')
        # 存放所有goodsid的列表
        goodsIdList = [gid for gid in cookie_str.split() if gid.strip()]

        # 思考1：最终需要获取的推荐商品
        goodsObjList = [Goods.objects.get(id=gsid) for gsid in goodsIdList if
                        gsid != goodsid and Goods.objects.get(id=gsid).category_id == Goods.objects.get(
                            id=goodsid).category_id][:4]
        # 将goodsObjList传递给get方法
        response = func(detailView, request, goodsid, goodsObjList, *args, **kwargs)
        # 判断goodsid是否存在goodsIdList中
        if goodsid in goodsIdList:
            goodsIdList.remove(goodsid)
            goodsIdList.insert(0, goodsid)
        else:
            goodsIdList.insert(0, goodsid)
        # goodsIdList中的int转化成str
        goodsIdList = [str(x) for x in goodsIdList]
        # 将goodsIdList中的数据保存到Cookie中
        response.set_cookie('recommend', str(" ".join(goodsIdList)))
        return response
    return wrapper

class DetailView(View):
    @recommend_view
    def get(self, request, goodsid, recommendList=[]):
        goodsid = int(goodsid)
        # 根据goodsid查询商品详情信息（goods对象）
        goods = Goods.objects.get(id=goodsid)
        return render(request, 'detail.html', {'goods': goods, 'recommendList': recommendList})


import threading
import time
import logging
import serial
import pynmea2

log = logging.getLogger('gnss')

class GnssThread(threading.Thread):
    def __init__(self, q, NMEAPort):
        threading.Thread.__init__(self)
        self.q = q
        self.NMEAPort = NMEAPort
        self.live = True
        self.nmea = None

    def run(self):
        log.debug(f"Listening for NMEA on {self.NMEAPort}...")
        self.nmea = serial.Serial(self.NMEAPort)

        while self.live:
            line = self.nmea.readline().decode('ASCII')
            if line.startswith("$GPGGA"):
                # Little silly to use PyNMEA2 for just this one thing, but the NMEA sentence format
                # is oddly complicated and this saves doing our own lat/lon format conversions.
                sentence = pynmea2.parse(line)
                # Only send on valid fixes, before gps_qual changes the results are either null or
                # have very high error.
                if sentence.gps_qual != 0:
                    self.q.put(['LocationFix', {'lat': sentence.latitude, 'lon': sentence.longitude, 'alt': sentence.altitude}])

    def stop(self):
        self.live = False


"""Adds versioning `User.authorized`

Revision ID: ee3c6c0702a6
Revises: 0fbbcf5eb614
Create Date: 2021-10-04 03:41:16.571947

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'ee3c6c0702a6'
down_revision = '0fbbcf5eb614'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('user', sa.Column('authorized', sa.Boolean(), nullable=True))
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_column('user', 'authorized')
    # ### end Alembic commands ###