from veneer.manage import start
import subprocess
import shutil
import os

parent_dir = os.getcwd()
job_name = 'work'
pst_file = '126001A.pst'
catchment_project= parent_dir + '\\pest_source\\MW_BASE_RC10.rsproj'
pest_path= parent_dir + '\\pest_source'
print('pest path ',pest_path)

python_path = 'C:\\UserData\\Qian\\anaconda'
os.environ['PATH'] = os.environ['PATH'] + ';' + pest_path
os.environ['PATH'] = os.environ['PATH'] + ';' + python_path
print(os.environ['PATH'])

# Setup Veneer
# define paths to veneer command and the catchment project
veneer_path = 'pest_source\\vcmd45\\FlowMatters.Source.VeneerCmd.exe'

# Number of instances to open
num_copies=4     # Important - set this to be a number ~ the number of CPU cores in your system!
first_port=15000

#Now, go ahead and start source
processes,ports = start(catchment_project,
                        n_instances=num_copies,
                        ports=first_port,
                        debug=True,
                        veneer_exe=veneer_path,
                        remote=False,
                        overwrite_plugins=True)

pest_master_version = 'ipestpp-ies.exe'
pest_slave_version = 'ipestpp-ies.exe'

# Set up job directories
instruction_file = 'output.ins'
template_file = 'parameters.tpl'

supp_files = ['126001A.pst','126001A.csv', 'observation_ensemble_0918.csv' , 'parameter_ensemble.csv',
			  'run_source.py', 'Plugins.xml']

# parent_dir = os.getcwd()
os.makedirs(job_name,exist_ok=True)
shutil.copyfile(instruction_file, job_name + '/' + instruction_file)
shutil.copyfile(template_file, job_name + '/' + template_file)
shutil.copyfile(pst_file, job_name + '/' + pst_file)
for file in supp_files:
    shutil.copyfile(file, job_name + '/' + file)

model_func_file = 'modeling_funcs.py'
shutil.copyfile('funcs/' + model_func_file, job_name + '/' + model_func_file)

os.chdir(job_name)
for port in ports :
    dir_name = 'Slave_' + str(port)
    os.makedirs(dir_name,exist_ok=True)
    shutil.copyfile(instruction_file, dir_name + '/' + instruction_file)
    shutil.copyfile(template_file, dir_name + '/' + template_file)
    shutil.copyfile(pst_file, dir_name + '/' + pst_file)
    shutil.copyfile(model_func_file, dir_name + '/' + model_func_file)
    for file in supp_files:
        shutil.copyfile(file, dir_name + '/' + file)
    veneer_connection = dir_name + '/veneer_connection.txt'
    with open(veneer_connection, 'w') as f:
        f.write(str(port))

os.chdir(parent_dir)

# Start PEST
os.chdir(job_name)
slave_processes = {}
pest_master_string = pest_master_version + '  ' +pst_file+ ' /h :4001 '
pest_slave_string = pest_slave_version + '  ' +pst_file+ ' /h localhost:4001'
current_dir = os.getcwd()
for port in ports:
    dir_name = 'Slave_' + str(port)
    os.chdir(dir_name)
    slave_processes[port] = subprocess.Popen(["start", "cmd", "/k",pest_slave_string], shell=True  )
    # slave_processes[port] = Popen(pest_slave_string,shell=True, stdout=PIPE, stderr=PIPE)
    os.chdir(current_dir)
#subprocess.Popen(["start", "cmd", "/k",pest_master_string], shell=True  )
os.system(pest_master_string )
os.chdir(parent_dir)
# subprocess.run(pest_master_string)


import re
import datetime
import time
import os
import shutil
import codecs
import mmap
import json
import logging
import MySQL_interactions
import spider_toolkit
import zipfile
import spider_exceptions
import utilities

def unzip_archive(path, target, ignore_list, only_single_files=True):
    """Extract a zip archive from a supplied path to a supplied target, ignoring
    files in ignore_list. Extracts all nested archives, flattening structure.
    Defaults to only unpacking if there is only one non-ignored file in the archive.
    Removes archives once the contents have been extracted.
    Returns path to last unzipped file
    """
    final_path = ''
    with zipfile.ZipFile(path, 'r') as myzip:
        useful_files = [i for i in myzip.infolist() if i.filename not in ignore_list]
        if only_single_files and len(useful_files) > 1:
            logging.info("Too many files in zip archive - one required file expected")
            raise spider_exceptions.TooManyFilesInArchiveError
        for info in useful_files:
            myzip.extract(info, target)
            if info.filename.lower().endswith('.zip'):
                final_path = unzip_archive(os.path.join(target, info.filename), target, ignore_list)
            else:
                final_path = os.path.join(target, info.filename)
    os.remove(path)
    return final_path

def clean_singleton_unzip(path, target, ignore_list):
    """Extract a single file from a zip archive, doesn't unpack anything if there are
    multiple files in the zip. On error, cleans up any files or folders left over from intermediate
    unzip stages, leaving the originating directory as it was previously. NB: this is possible because
    the inner unzip calls itself recursively, thus doesn't remove any archives until the innermost
    one has been successfully extracted. Returns path to new file if successful, path to original file otherwise
    """
    parent_dir = os.path.dirname(path)
    start_list = os.listdir(parent_dir)
    try:
        return unzip_archive(path, target, ignore_list, True)
    except:
        for f in os.listdir(parent_dir):
            if f not in start_list:
                os.remove(os.path.join(parent_dir, f))
        return path

def clean_folder(sel_path):
    """Cleanly and completely clear out a folder with the path provided,
    removing all files and subdirectories
    """
    for root, dirs, files in os.walk(sel_path):
        for f in files:
            os.unlink(os.path.join(root, f))

        for d in dirs:
            shutil.rmtree(os.path.join(root, d))

def make_test_file_path(file_path):
    """Take a production file path and convert it to something analogous but local, for testing purposes"""
    paths_to_replace = [u'H:', u'W:', u'U:', u'\\nyccelerra.fitch.corp\sfna$', u'\\nyccelerra\sfna$', u'\\Frafp03',
                        u'nyccelerra.fitch.corp\sfna$', u'nyccelerra\sfna$']
    test_path_base = 'C:\\temp\\spiderTest\\Targets'
    for path in paths_to_replace:
        file_path = file_path.replace(path, test_path_base)
        if file_path.startswith('\\'):
            file_path = file_path[1:]
    return file_path

class FilePathDefiner():
    """Define all widely used file paths with reference to base paths loaded
    from the local config file
    """

    def __init__(self):
        config = utilities.load_local_config()
        self.local_folder_base = config['localstorage']
        self.local_download_folder_base = os.path.join(self.local_folder_base, 'DownloadStaging')
        self.tidied_deal_folder_base = os.path.join(self.local_folder_base, 'CompletedDealFolders')
        self.data_folder_base = config['datalocation']
        self.file_dump_base = os.path.join(self.data_folder_base, 'UnmappedDownloads')

class SpiderFileHandler(FilePathDefiner):

    def __init__(self, trusteeName):
        FilePathDefiner.__init__(self)
        self.trustee = trusteeName
        #initialised here but accessed through the property as may be modified during a spider run
        self.download_folder_increment = 1
        #check if there are multiple download folders from unfinished runs
        if os.path.exists(self.local_download_folder_base):
            folders = [x for x in os.listdir(self.local_download_folder_base) if x.startswith(trusteeName)]
            for f in folders:
                inc = f.split('_')
                self.download_folder_increment = int(inc[1]) if int(inc[1]) > self.download_folder_increment else self.download_folder_increment
        #create folder if it doesn't exist already
        if not os.path.exists(self.local_download_folder):
            os.makedirs(self.local_download_folder)
        #these are fine as attributes because they do not need to be modified at runtime
        self.download_log = os.path.join(self.data_folder_base, trusteeName + '_DownloadLog.txt')
        self.screenshot_folder = os.path.join(self.data_folder_base, 'Screenshots', trusteeName)
        if not os.path.exists(self.screenshot_folder):
            os.makedirs(self.screenshot_folder)
        self.completed_deal_download_folder = os.path.join(self.tidied_deal_folder_base, self.trustee)
        if not os.path.exists(self.completed_deal_download_folder):
            os.makedirs(self.completed_deal_download_folder)
        self.unmapped_file_dump = os.path.join(self.file_dump_base, self.trustee)
        if not os.path.exists(self.unmapped_file_dump):
            os.makedirs(self.unmapped_file_dump)

        self.orphaned_report_folder_base = os.path.join(self.file_dump_base, self.trustee)
        self.state_file_folder = os.path.join(self.data_folder_base, 'JSONStateSnapshots', self.trustee)
        if not os.path.exists(self.state_file_folder):
            os.makedirs(self.state_file_folder)
        self.state_file = self.state_file_folder + '\\currentstateinfo.txt'
        self.temp_dl_list = []

    #essentially a readonly property as only this class should handle runtime
    #modifications of the local download folder path
    @property
    def local_download_folder(self):
        """Return fully qualified local download folder created from base plus spider-specific extension"""
        trustee_folder = self.trustee + '_' + str(self.download_folder_increment)
        return os.path.join(self.local_download_folder_base, trustee_folder)

    def increment_download_folder(self):
        """In the case of a download failing, make a new download staging folder for future downloads.
        Also clear the temporary download list so further fileinfolists in bundles do not contain erroneous data.
        """
        self.download_folder_increment += 1
        os.mkdir(self.local_download_folder)
        self.temp_dl_list = []

    def clean_download_folder(self):
        """Attempts to completely clear out the local download folder in case of interrupted previous attempts.
           If unable to clear, creates new local download folder path for subsequent downloads"""
        try:
            clean_folder(self.local_download_folder)
        except:
            logging.warning("Possible partial download, unable to clear temp folder")
            self.increment_download_folder()

    def clean_alternate_download_folders(self):
        '''Attempts to remove any extra temp download folders created when partial downloads
           prevent move/delete during spider run'''
        tempDirs = os.listdir(self.local_download_folder_base)
        for item in tempDirs:
            try:
                #delete anything called '<Trustee>_<X>'
                if self.trustee in item:
                    os.rmdir(self.local_download_folder_base + '\\' + item)
                #reset the download folder increment
                self.download_folder_increment = 1
                os.mkdir(self.local_download_folder)
            except:
                logging.warning(r"Unable to delete extra download folder - please check C:\Spider\DownloadStaging and clear manually")
                #self.writeSpiderLog("","Warning","", r"Unable to delete extra download folder - please check C:\Temp and clear manually")

    """
    def make_download_target_list(self, dealMap, dealName, repDetails):
        #TODO: improve detail in 'failure to map' error logging so we can see ~why~ it failed
        #TODO: improve... this method -_- Or at least understand and comment the blighter so
        #future folks can understand it without screaming

        #horrible: ext check not too bad, but reportName one is checked again repeatedly in this routine
        if not repDetails.has_key("ext"):
            repDetails["ext"] = ""
        if not repDetails.has_key("reportName"):
            repDetails["reportName"] = "unnamed"

        #this can stay
        copy_list = [];

        for location in dealMap[dealName]['locations']:
            #ok, so here we are looking for a date to build a folder structure
            #only even looks for a date if the website has reported one
            if repDetails.has_key("date"):
                aMatch = re.search(location['dtRegex'] , repDetails["date"])
                #did we find a match? if we did, let's make it a proper date
                if aMatch:
                    c = time.strptime(aMatch.group(0),location['fmtFileIn'])
                    #we chould now do anything that definitely requires a date here, where we know we have a date for certain
                    monthDir = str(c.tm_mon)
                    yearDir = time.strftime('%Y',c)
                    if location['fmtFolderMonth'] == "Q%q": #quarters not supported by time lib
                        monthDir = "Q"+ str(int((c.tm_mon+2)/3))
                    else:
                        monthDir = time.strftime(location['fmtFolderMonth'], c)
                else:
                    #no match but report thinks it has a date! eek
                    monthDir = 'DateMatchFailed'
                    yearDir = 'DateMatchFailed'
            else:
                #report doesn't even have a clue about dates
                monthDir = 'NoDate'
                yearDir = 'NoDate'

            #so that gives us a folder with some sort of year/month structure if at all possible
            folder = os.path.join(os.pardir, 'Data', location['folder'], yearDir, monthDir)

            #now off to get a filename:
            foundType = ""
            typeInDB = ""
            #OZ TWEAK members of reportMappings are now dicts so can ref by named keys instead of indeces
            for mapping in location['reportMappings']:
                if repDetails.has_key("url"):
                    urlMatch = re.search(mapping['linkURLRegex'], repDetails["url"])
                else:
                    urlMatch = False
                if repDetails.has_key("reportName"):
                    textMatch = re.search(mapping['linkTextRegex'], repDetails["reportName"])
                else:
                    textMatch = False
                #DANGER: why is the next bit 'both' when they surely mean 'either'? The success message 'and/or' implies an 'or'
                #ah no, success message is just trying to cover all bases and db field should be called 'requiredmatch' or something
                #hang on, this is a dict. I can call the keys whatever I want. Do I have to nearly match the db, really?
                if ((mapping['requiredMatch'] == 'Both') and urlMatch and textMatch) or \
                    ((mapping['requiredMatch'] == 'URL') and urlMatch) or \
                    ((mapping['requiredMatch'] == 'Text') and textMatch): # urlMatch replaces not(urlMatch is None) to avoid empty strings being truthy
                    #what is going on in this next bit? seems to be 'if no ext from web OR no ext from DB OR we have both and they match'
                    #so... only if we have both and they don't match, it falls through?
                    #does 'not mapping['extension'] work if ext is null?
                    if (repDetails["ext"] == '') or (not mapping['extension']) or (repDetails["ext"].lower() == mapping['extension'].lower()):
                        if mapping['linkTextRegex'] and repDetails.has_key("reportName"): #dont sub "" and have a target
                            foundType = re.sub(mapping['linkTextRegex'], mapping['reportOutput'], repDetails["reportName"])
                        else:
                            foundType = repDetails["reportName"]
                        extension = repDetails["ext"]
                        typeInDB = mapping['reportOutput']
                        if mapping['extension']:
                            extension = mapping['extension']
                            #OZ NOTE: if there is no extension found at all, this still passes. Is that right?
                        # Only take first match - we shouldn't map to multiple names for the same
                        break
            if foundType <> "":
                #foundType = re.sub(r"[\\|\/|\:|\*|\?|\"|\<|\>|\|]", "", foundType) # windows reserved chars
                foundType = sanitised_string(foundType)
                #NB: mapping at this point references the last value it took before the break
                logging.info('Matched on URL: ' + mapping['linkURLRegex'] + ' and/or Text: ' + mapping['linkTextRegex'])
                local_name = location['fmtToken']
                local_name = local_name.replace('?a',dealMap[dealName]['abbr'])
                local_name = local_name.replace('?t',foundType)
                local_name = local_name.replace('?d',time.strftime(location['fmtFileOut'],c))
                local_name = local_name.replace('?e',extension)
                copy_list.append({"localName": local_name, "checkPath":folder, "reportType":typeInDB})
        if len(copy_list) == 0:
            downloadDetails = ""
            repType = "unmatched"
            #DODGY: this next will never fail but could just be 'unnamed'
            if repDetails.has_key("reportName"):
                repType = repDetails["reportName"]
                downloadDetails = downloadDetails +"(RPT: "+ repDetails["reportName"] +")"
            if repDetails.has_key("url"):
                if repType == "unmatched": #unreachable
                    repType = repDetails["url"]
                downloadDetails = downloadDetails +"(URL: "+ repDetails["url"] +")"
            #DODGY: again, this has been set above so key certainly present but could just be ""
            if repDetails.has_key("ext"):
                downloadDetails = downloadDetails +"(EXT: "+ repDetails["ext"] +")"
            #deal_id = dealMap[dealName]['dealId']
            #DateLogFMT = ""
            #if repDetails.has_key("date"):
                #DateLogFMT = repDetails["date"]
            #self.writeSpiderLog(self._dealData[deal]['busGrp'],"Warning", deal, "Could not map filename", source_location=downloadDetails, deal_id=deal_id)
            logging.info('Could not map filename')
            for opt in repDetails:
                logging.info(opt +'\t'+ repDetails[opt])
        return copy_list

    def old_make_download_target_list(self, dealMap, dealName, repDetails):
        #here for reference as I start to hack it apart OZ 20151026
        if not repDetails.has_key("ext"):
            repDetails["ext"] = ""
        if not repDetails.has_key("reportName"):
            repDetails["reportName"] = "unnamed"
        copy_list = [];
        for location in dealMap[dealName]['locations']:
            c = "DateDotParsed"
            if repDetails.has_key("date"):
                aMatch = re.search(location['dtRegex'] , repDetails["date"])
                if aMatch:
                    c = time.strptime(aMatch.group(0),location['fmtFileIn'])
            #DANGER: the rest of the date stuff depends on the previous 'if' executing or the whole thing fails
            monthDir = str(c.tm_mon)
            if location['fmtFolderMonth'] == "Q%q": #quarters not supported by time lib
                monthDir = "Q"+ str(int((c.tm_mon+2)/3))
            else:
                monthDir = time.strftime(location['fmtFolderMonth'],c)
            folder = os.path.join(os.pardir, 'Data',location['folder'], time.strftime('%Y',c), monthDir)
            foundType = ""
            typeInDB = ""
            #OZ TWEAK members of reportMappings are now dicts so can ref by readable keys instead of indeces
            for mapping in location['reportMappings']:
                if repDetails.has_key("url"):
                    urlMatch = re.search(mapping['linkURLRegex'], repDetails["url"])
                else:
                    urlMatch = False
                if repDetails.has_key("reportName"):
                    textMatch = re.search(mapping['linkTextRegex'], repDetails["reportName"])
                else:
                    textMatch = False
                #DANGER: why is the next bit 'both' when they surely mean 'either'? The success message 'and/or' implies an 'or'
                if ((mapping['useRegex'] == 'Both') and urlMatch and textMatch) or \
                    ((mapping['useRegex'] == 'URL') and urlMatch) or \
                    ((mapping['useRegex'] == 'Text') and textMatch): # urlMatch replaces not(urlMatch is None) to avoid empty strings being truthy
                    if (repDetails["ext"] == '') or (not mapping['extension']) or (repDetails["ext"].lower() == mapping['extension'].lower()):
                        if mapping['linkTextURL'] and repDetails.has_key("reportName"): #dont sub "" and have a target
                            foundType = re.sub(mapping['linkTextURL'], mapping['reportOutput'], repDetails["reportName"])
                        else:
                            foundType = repDetails["reportName"]
                        extension = repDetails["ext"]
                        typeInDB = mapping['reportOutput']
                        if mapping['extension']:
                            extension = mapping['extension']
                        # Only take first match - we shouldn't map to multiple names for the same
                        break
            if foundType <> "":
                foundType = re.sub(r"[\\|\/|\:|\*|\?|\"|\<|\>|\|]", "", foundType) # windows reserved chars
                #WTF: surely mapping var is out of scope now?! How is this reliable?!
                logging.info('Matched on URL: ' + mapping[1] + ' and/or Text: ' + mapping[2])
                local_name = location['fmtToken']
                local_name = local_name.replace('?a',dealMap[dealName]['abbr'])
                local_name = local_name.replace('?t',foundType)
                local_name = local_name.replace('?d',time.strftime(location['fmtFileOut'],c))
                local_name = local_name.replace('?e',extension)
                copy_list.append({"localName": local_name, "checkPath":folder, "reportType":typeInDB})
        if len(copy_list) == 0:
            downloadDetails = ""
            repType = "unmatched"
            #DODGY: this next will never fail but could just be 'unnamed'
            if repDetails.has_key("reportName"):
                repType = repDetails["reportName"]
                downloadDetails = downloadDetails +"(RPT: "+ repDetails["reportName"] +")"
            if repDetails.has_key("url"):
                if repType == "unmatched":
                    repType = repDetails["url"]
                downloadDetails = downloadDetails +"(URL: "+ repDetails["url"] +")"
            #DODGY: again, this has been set above so key certainly present but could just be ""
            if repDetails.has_key("ext"):
                downloadDetails = downloadDetails +"(EXT: "+ repDetails["ext"] +")"
            #deal_id = dealMap[dealName]['dealId']
            #DateLogFMT = ""
            #if repDetails.has_key("date"):
                #DateLogFMT = repDetails["date"]
            #self.writeSpiderLog(self._dealData[deal]['busGrp'],"Warning", deal, "Could not map filename", source_location=downloadDetails, deal_id=deal_id)
            logging.info('Could not map filename')
            for opt in repDetails:
                logging.info(opt +'\t'+ repDetails[opt])
        return copy_list
    """
    def report_in_download_log(self, textToCheck):
        """Compare supplied report text to saved download log, returning True if text present"""
        #very first thing - is there even a download log at all? I really hope so, but just in case:
        if os.path.isfile(self.download_log) == False:
            self.write_download_log("Created at local time " + str(datetime.datetime.now()))

        try:
            with codecs.open(self.download_log, 'r', 'utf-8') as log:
                checker = mmap.mmap(log.fileno(), 0, access=mmap.ACCESS_READ)
                #return checker.find(textToCheck) != -1

                #ok you forced me to make this case-insensitive for STUPID REASONS
                #leave the faster (more than 5x faster in fact) version here for when all the spiders are replaced
                #and I can do a one-time replace to nuke all the uppercase in the files
                cleaned = textToCheck.replace('(', r'\(').replace(')', r'\)').replace('?',r'\?').replace('|', r'\|')
                #non-regex won't need the sanitising crud either
                return re.search(cleaned, checker, re.IGNORECASE) is not None
        except IOError:
            logging.error("Problem reading download log")
            raise

    def write_download_log(self, loginfo):
        """Write supplied logging information (usually report identifier) to the text
        download log file so future runs can check if this file has already been
        downloaded
        """
        try:
            with codecs.open(self.download_log, 'a', 'utf-8') as log:
                log.write('\n' + loginfo.strip() + '\r\n')
        except IOError:
            logging.error('Problem writing to download log file')
            raise

    def note_latest_downloaded_file(self, file_dict):
        """Get name of most recent file in dl directory and add it to a supplied dict"""
        #don't bother with any clever order-files-by-timestamp stuff, it's too flakey
        logging.info('Attempting to get latest download name')
        #give it a few tries with an increasing wait just in case
        ignorelist = self.temp_dl_list + ['currentstateinfo.txt']
        for i in range(1,6):
            files = os.listdir(self.local_download_folder)
            for f in files:
                #check for a new, non-partial file
                if f not in ignorelist and f.lower().endswith('.part') is False:
                    self.temp_dl_list.append(f)
                    file_dict["DownloadedName"] = f
                    logging.info('Latest file: ' + f)
                    #found one - add it to the list and let calling func continue
                    return
                else:
                    #nothing new found, wait a while and have another go
                    time.sleep((i/2))
        else:
            self.temp_dl_list.append('No new file found')
            logging.warning('No new file detected')

    def stage_latest_download(self, filesList, logCheckName, targetList, deal_name, deal_map, db_interaction, rep_date=''):
        """Write download log and file target list for the latest downloaded file. Date is for DB log and if supplied
        should be a string in '%Y-%m-%d' format.
        """
        #build dict of file target info etc. to map to download order in ordered list for later file handling
        file_dict = {'NamePreDL':logCheckName, 'FileTargetList':targetList}
        self.note_latest_downloaded_file(file_dict)
        #20160215 - trying out a new version moving this to the bundling routine so incomplete deal bundles
        #get completely re-downloaded
        #self.write_download_log(logCheckName)
        filesList.append(file_dict)
        for target in targetList:
            #write database download log here - while not 'correct' in that the file may not actually be in the specified
            #location yet, it's how the old spiders did it and we have to match their behaviour to maintain compatibility
            #(at least for the moment)
            if target['localName'] == '' and file_dict.has_key("DownloadedName"):
                target['localName'] = file_dict["DownloadedName"]

            db_interaction.write_db_log(self.trustee, deal_map[deal_name]['busGrp'],
                                        'File downloaded', deal_name, filename=target['localName'],
                                        directory=target['checkPath'], dealID=deal_map[deal_name]['dealId'],
                                        repDate=rep_date, repType=target['reportType'], use_ntp=True)

    def package_deal_downloads(self, dealName, dealID, filesList, expectedCount, check_name_list):
        """Make a nice human-readable-named folder to put the downloaded files in, and move them there with the
        ordered list of useful info for a later process to use in order to rename and move to final resting places
        """
        sanitised_dealname = utilities.sanitised_string(dealName, True, True)
        dealFolder = os.path.join(self.completed_deal_download_folder, (str(dealID) + ' ' + sanitised_dealname + ' ' + time.strftime('%Y%m%d')))
        if not os.path.exists(dealFolder):
            os.makedirs(dealFolder)
        #check for .part files, empty files, and whether the count matches the number expected
        filenames = os.listdir(self.local_download_folder)
        for f in filenames:
            if f.lower().endswith('.part') or os.path.getsize(os.path.join(self.local_download_folder, f)) == 0:
                #we have a situation - don't move these ones, make a note and move on
                filesList.append({'Incomplete':f})
                continue
            elif os.path.isfile(os.path.join(dealFolder, f)):
                #who knows how this happened, but if the file is already there we can ignore and delete
                logging.warning('File already exists in staging destination!')
                os.remove(os.path.join(self.local_download_folder, f))
            else:
                os.rename(os.path.join(self.local_download_folder, f), os.path.join(dealFolder, f))
        #all done? write the check_names to the download log
        for check in check_name_list:
            self.write_download_log(check)
        #compare the counts
        successfiles = os.listdir(dealFolder)
        #try and make a note of the download order, which should correspond to the file details supplied
        filesList.append({'DownloadOrder':self.temp_dl_list})
        filesList.append({'DownloadsRequested':expectedCount, 'SuccessfulDownloads':len(successfiles)})
        #chuck the useful info in after the whole bang-shoot
        with open(dealFolder + '\\FileInfoList.txt', 'w') as t:
            json.dump(filesList, t)
        #any fragments left? Don't care. Do the cleanup/increment so everything is peachy for the next deal
        self.clean_download_folder()
        self.temp_dl_list = []

    def handle_deal_file_bundle(self, dealDownloadFolderPath, delete_original=True, local_test=False):
        """Take a folder of files downloaded for a deal and move them to folders specified
        in an attached info list
        """
        #NOTE: called from polling process in different thread to Spider instances
        #if this bit fails we are stuffed and would prefer to bomb out completely
        fullpath = os.path.join(dealDownloadFolderPath,'FileInfoList.txt')
        with open(fullpath, 'r') as fileinfo:
            fileinfolist = json.load(fileinfo)

        #slightly annoying to have to iterate to extract the download order list, but such is life
        for info in fileinfolist:
            if 'DownloadOrder' in info:
                filedownloadorder = info['DownloadOrder']
                break

        #go through non-fileinfolist files and move according to supplied targets
        dealfiles = os.listdir(dealDownloadFolderPath)
        for df in [x for x in dealfiles if x != 'FileInfoList.txt']:
            if df in filedownloadorder:
                downloadindex = filedownloadorder.index(df)
                fileinfodict = fileinfolist[downloadindex]
                self.handle_downloaded_file(os.path.join(dealDownloadFolderPath, df), fileinfodict, delete_original, local_test)
            else:
                continue

        #At the end, if there are any files left we can try to put them in a folder for manual handling
        remaining = os.listdir(dealDownloadFolderPath)
        if len(remaining)>1:
            #more than one file left? copy all that's left of the bundle to a dump folder for safety pls
            pathparts = dealDownloadFolderPath.split("\\")
            deal_dirname = pathparts[-1]
            trustee = pathparts[-2]
            target_dump = os.path.join(self.unmapped_file_dump, deal_dirname)
            shutil.copytree(dealDownloadFolderPath, target_dump)
            logging.warning("Download copy failure - moved to dump folder")
            #let's also shove a warning in the database if necessary
            if delete_original:
                dbint = MySQL_interactions.MySQLInteraction(True)
                dbint.write_db_log(trustee, "", "Warning", deal_dirname.split(' ',1)[1],
                                   directory="Trapdoor: could not move files, bundle moved to dump",
                                   dealID=deal_dirname.split(' ')[0], use_ntp=True)

        logging.info('Deal bundle file handling complete')
        #then attempt to remove the parent deal folder once it's copied if necessary
        if delete_original:
            clean_folder(dealDownloadFolderPath)
            os.rmdir(dealDownloadFolderPath)

    def handle_downloaded_file(self, fullDownloadFilePath, fileInfoDict, delete_orig=True, local_test=False):
        """Rename (if required) and move a downloaded file within a download bundle
        to its target folder according to supplied dict values
        """
        targetlist = fileInfoDict['FileTargetList']
        overrideable_delete_orig = delete_orig

        #quick check: if there's no targets, just throw it in the heap and move on
        if len(targetlist) == 0:
            self.handle_unmapped_file(fullDownloadFilePath, delete_orig)
            logging.warning("Download with no target detected: " + fullDownloadFilePath + " moved to dump folder")
            return

        for target in targetlist:
            #NEW 20160224 - targets may come in with blank localName if they both had no mapping AND missed the
            #latest-download checker in stage_latest_download. Not sure if this is even possible?
            #no actually, I'm pretty sure it isn't because if they miss latestdownload then they aren't in
            #the downloadlist that is being iterated on to call this routine. Thus if localname blank here,
            #we are in trouble and do need to bomb out pls

            #there's a routine in the existing system that compares the actual file extension to
            #what the db expected it to be, and alters the latter to match to former if necessary
            #I think this is worth doing - only a tiny bit better
            db_extension = target['localName'].split('.')[-1]
            actual_extension = fullDownloadFilePath.split('.')[-1]
            if actual_extension.lower() == 'zip' and db_extension.lower() != actual_extension.lower():
                #we have a zip and we weren't expecting it. Let's have a quick go at unzipping:
                fullDownloadFilePath = clean_singleton_unzip(fullDownloadFilePath, os.path.dirname(fullDownloadFilePath), ['MadDisclaimer.pdf', 'Mad_Disclaimer.pdf'])
                #unzip to same place, and ignore the USBank-specific disclaimer file
                #(add db-to-fileinfolist ignorelist injection if we ever require this for other trustees!)
                #so now we either have the single desired unzipped file or a multi-file zip
            #need to re-check extension matching now
            actual_extension = fullDownloadFilePath.split('.')[-1]
            if db_extension != actual_extension:
                target['localName'] = target['localName'].replace(db_extension, actual_extension)
            #are we in test mode? change the target to a local one pls
            if local_test == True:
                target['checkPath'] = make_test_file_path(target['checkPath'])
            #now we're into bits that can fail on broken permissions, network glitches or incorrect paths
            try:
                fullTargetPath = os.path.join(target['checkPath'], target['localName'])
                #see if the required directory exists and if not, make it
                if not os.path.exists(target['checkPath']):
                    os.makedirs(target['checkPath'])
                #if the file isn't already there, copy it across to the target path
                if not os.path.exists(fullTargetPath):
                    shutil.copy(fullDownloadFilePath, fullTargetPath)
                    logging.info(target['localName'] + ' File copied to ' + fullTargetPath)
                else:
                    logging.warning("File already exists in target location: " + fullTargetPath)
                    continue
            except:
                #I know, too general - but dammit it really could be anything
                overrideable_delete_orig = False
                logging.warning("Error copying file to: " + fullTargetPath)
                dbint = MySQL_interactions.MySQLInteraction()
                dbint.write_db_log(self.trustee, '',
                                        'Warning', 'File copy failed', filename=target['localName'],
                                        directory=target['checkPath'], use_ntp=True)

        #and if all successful, remove the old one.
        if overrideable_delete_orig:
            os.remove(fullDownloadFilePath)

    def handle_unmapped_file(self, download_file_path, delete_orig):
        """If a report has somehow been bundled without a matching report mapping, move or copy it
        into a special folder for manual handling. Returns the final path of the moved/copied file.
        """
        #get useful info from the full download path - reliable as this was built by package_deal_downloads above
        dl_path_parts = download_file_path.replace("\\", "/").split("/")
        trustee, deal_name, file_name = dl_path_parts[-3], dl_path_parts[-2], dl_path_parts[-1]
        orphan_target = os.path.join(self.orphaned_report_folder_base, deal_name)
        if not os.path.exists(orphan_target):
            os.makedirs(orphan_target)
        full_target = os.path.join(orphan_target, file_name)
        shutil.copy2(download_file_path, full_target)
        dbint = MySQL_interactions.MySQLInteraction(True)
        dbint.write_db_log(trustee, "", "Warning", deal_name.split(' ',1)[1],
                           directory="Trapdoor: could not map filename, file moved to dump",
                           dealID=deal_name.split(' ')[0], use_ntp=True)
        if delete_orig:
            os.remove(download_file_path)
        return full_target

    def initialise_logger(self):
        """Simple call to spider_toolkit logging initialiser, passing in the name
        of the trustee, the target data folder, and setting outputs to file and tkinter
        """
        spider_toolkit.init_logger(self.trustee, True, False, True, self.data_folder_base, 'w')
        #spider_toolkit.init_logger(self.trustee, True, False, True, 'C:\\temp\\spiderTest', 'w')

    def write_state_to_disk(self, statedict):
        """Write dict containing useful spider state info (e.g. current deal, last report clicked)
        to a human-readable file on disk
        """
        with open(self.state_file, 'w') as t:
            json.dump(statedict, t)

    def read_state_from_disk(self):
        """Read saved spider state file back into dict for spider to use
        Return empty dict if no state file found
        """
        statedict = {}
        if os.path.isfile(self.state_file):
            with open(self.state_file, 'r') as f:
                statedict = json.load(f)
        return statedict

    def clear_saved_state(self):
        if os.path.isfile(self.state_file):
            os.remove(self.state_file)

class DownloadTargetBuilder():
    """Method object building the list of target locations to which a downloaded file
    should be copied, as per the Locations/Report Mappings in the database
    """

    def __init__(self, dealMap, dealName, repDetails):
        self.deal_map = dealMap
        self.deal_name = dealName
        self.report_details = repDetails

        self.matched_date = 'unknown_date'
        self.month_dir = 'DateMatchFailed'
        self.year_dir = 'DateMatchFailed'
        self.rep_type_name = ''
        self.type_name_in_db = ''

        #set some safe vals here to make subsequent tests and substitutions easier
        self.safe_ext = 'unknown' if not self.report_details.has_key('ext') else self.report_details['ext']
        self.safe_name = 'unnamed' if not self.report_details.has_key('reportName') else self.report_details['reportName']

    def constructed_report_file_name(self, location, mapping):
        """Build target file name from db results and matched text"""
        repTypeName = utilities.sanitised_string(self.rep_type_name)
        #NB: mapping at this point references the last value it took before the break
        logging.info('Matched on URL: ' + mapping['linkURLRegex'] + ' and/or Text: ' + mapping['linkTextRegex'])
        constructed_name = location['fmtToken']
        constructed_name = constructed_name.replace('?a', self.deal_map[self.deal_name]['abbr'])
        constructed_name = constructed_name.replace('?t', repTypeName)
        date_string = self.matched_date if self.matched_date == 'unknown_date' else time.strftime(location['fmtFileOut'], self.matched_date)
        constructed_name = constructed_name.replace('?d', date_string)
        constructed_name = constructed_name.replace('?e', self.safe_ext)
        return constructed_name

    def report_date_match(self, location):
        """Get any date search matches using regex strings from db to match against values from website"""
        if self.report_details.has_key("date"): #with date regex from db, check for match first in repDetails['date']
            dateMatch = re.search(location['dtRegex'], self.report_details["date"])
        elif self.report_details.has_key("url"): #and failing that in repDetails['URL']
            dateMatch = re.search(location['dtRegex'], self.report_details["url"])
        else:
            logging.warning('Date match failed in make target folder name') #if they both fail or aren't present, log it and move on

        return dateMatch

    def set_date_folder_names(self, location):
        """Set names of year and month/quarter folders for target location"""
        self.year_dir = time.strftime('%Y', self.matched_date)
        if location['fmtFolderMonth'] == "Q%q": #quarters not supported by time lib
            self.month_dir = "Q" + str(int((self.matched_date.tm_mon + 2) / 3))
        else:
            self.month_dir = time.strftime(location['fmtFolderMonth'], self.matched_date)

    def mapping_match_found(self, mapping):
        """Check for match in report details to url or link text regex from db"""
        #use db regex strings to search URL and/or reportname from repDetails as appropriate
        urlMatch = re.search(mapping['linkURLRegex'], self.report_details["url"]) if self.report_details.has_key("url") else False
        textMatch = re.search(mapping['linkTextRegex'], self.report_details["reportName"]) if self.report_details.has_key("reportName") else False

        #lots of stuff to test here so sod efficient, I want readable - let's break out the conditions:
        bothMatched = (mapping['requiredMatch'] == 'Both') and urlMatch and textMatch
        urlMatched = (mapping['requiredMatch'] == 'URL') and urlMatch
        textMatched = (mapping['requiredMatch'] == 'Text') and textMatch

        match_found = bothMatched or urlMatched or textMatched
        return match_found

    def log_unmatched_file_details(self):
        """Make any notes possible about a downloaded file which doesn't match any db vals
        Save these notes to log file and database log
        """
        downloadDetails = ""
        repType = "unmatched"
        if self.report_details.has_key("reportName"):
            repType = self.report_details["reportName"]
            downloadDetails = downloadDetails + "(RPT: " + self.report_details["reportName"] + ")"
        if self.report_details.has_key("url"):
            if repType == "unmatched":
                repType = self.report_details["url"]
            downloadDetails = downloadDetails + "(URL: " + self.report_details["url"] + ")"
        if self.report_details.has_key("ext"):
            downloadDetails = downloadDetails + "(EXT: " + self.report_details["ext"] + ")"
        #deal_id = self.deal_map[self.deal_name]['dealId']
        #self.writeSpiderLog(self.trustee, self._dealData[self.deal_name]['busGrp'],"Warning", self.deal_name, "Could not map filename", source_location=downloadDetails, deal_id=deal_id)
        logging.info('Could not map filename')
        logging.info('Report details for unmatched download: ' + str(self.report_details))

    def make_report_type_name(self, mapping):
        """A name for the given report: from mapping text regex if present, supplied
        report details if not
        """
        if mapping['linkTextRegex']:
            repTypeName = re.sub(mapping['linkTextRegex'], mapping['reportOutput'], self.safe_name)
        else:
            if self.safe_name == "unnamed":
                #link text useless, url not parsed by calling spider page into a useful name
                repTypeName = mapping['reportOutput']
            else:
                repTypeName = self.safe_name
        return repTypeName

    def make_download_target_list(self):
        """Make list of target filenames and folders to which a downloaded file will be copied"""
        target_list = []

        #go through all target file locations as found in database
        for location in self.deal_map[self.deal_name]['locations']:
            #reset type name so we can use it to check that a mapping has been found for this location
            self.rep_type_name = ''
            dateMatch = self.report_date_match(location)
            if dateMatch:
                #we should now do anything that definitely requires a date here, where we know we have a date for certain
                self.matched_date = time.strptime(dateMatch.group(0),location['fmtFileIn'])
                self.set_date_folder_names(location)

            #construct target folder name with db location folder + year/month structure
            folder = os.path.join(os.pardir, 'Data', location['folder'], self.year_dir, self.month_dir)

            #filenames: iterate over the reportmappings for the current location
            for mapping in location['reportMappings']:
                if not (self.mapping_match_found(mapping)): #no match, try the next mapping
                    continue
                #got initial match, now check that either we only have one version of the file extension or that they match
                if not ((self.safe_ext == 'unknown') or (not mapping.has_key('extension')) or (self.safe_ext.lower() == mapping['extension'].lower())):
                    logging.warning('Inconsistent file extensions')
                    continue
                #so we have a match and either a consistent or unknown file extension - carry on
                self.rep_type_name = self.make_report_type_name(mapping)
                self.safe_ext = self.safe_ext if not mapping.has_key('extension') else mapping['extension']
                self.type_name_in_db = mapping['reportOutput']
                break #and dive out now we have a match - only grab first match (as per current production code)

            #if something matched, we have a report type name (even if it is 'unnamed')
            #build a constructed_name and add to copy_list with the folder name and a 'type' from the db
            if self.rep_type_name != '':
                constructed_name = self.constructed_report_file_name(location, mapping)
                target_list.append({"localName": constructed_name, "checkPath":folder, "reportType":self.type_name_in_db})

        #iff target_list is empty, log a load of stuff about it not matching anything
        if len(target_list) == 0:
            self.log_unmatched_file_details()
            #we now have to add a blank target for each location
            for location in self.deal_map[self.deal_name]['locations']:
                if location['folder'] != r'H:\03. ESF - CMBS\09. Workflow\Special Notices':
                    unmapped_folder = os.path.join(os.pardir, 'Data', location['folder'], self.year_dir, self.month_dir, 'Unmapped')
                    target_list.append({"localName": "", "checkPath":unmapped_folder, "reportType":""})

        return target_list

class file_download_started():

    def __init__(self, downloadHandler, prevCount):
        self.download_handler = downloadHandler
        self.prev_count = prevCount

    def __call__(self, driver):
        """Returns True if there is one more file than previously in the specified download location"""
        return len(os.listdir(self.download_handler.local_download_folder)) > self.prev_count

class file_download_finished():

    def __init__(self, downloadHandler):
        self.download_handler = downloadHandler

    def __call__(self, driver):
        """Returns False if there is a .PART or empty file in the specified download location"""
        for f in os.listdir(self.download_handler.local_download_folder):
            if f.lower().endswith('.part'):
                return False
            elif os.path.getsize(self.download_handler.local_download_folder + '\\' + f) == 0:
                return False
        return True

class ReportDefinition():
    """Data structure storing metadata about reports in cleaned formats.
    Provides the USBank-specific 'name-pre-download' which matches old records
    in the download log.
    """

    #TODO: not sure if this is too USBank specific, keep an eye on it
    def __init__(self, trusteeDealName, rawName = '', rawDate = '', fileExt = '', portfolioGroup = '', reportGroup = '', url = ''):
        """A collection of attributes used to create a target list for file downloads.
        'rawDate', if used, should contain date in format to match regex in database Report Mapping Set
        'url' can also contain the Report Mapping Set date regex and is used to search for URL regex match as per Report Mapping Data
        'rawName' is used to search for Text regex match as per Report Mapping Data
        """
        self.trustee_deal_name = trusteeDealName
        self.raw_name = rawName
        self.raw_date = rawDate
        self.file_extension = fileExt
        #set up dictionary of optional report params to use for download target list
        self.report_opts = {"namePreDL": self.name_pre_download.replace(u"\u2026","").decode('unicode_escape').encode('ascii','ignore')}
        if rawName != '':
            self.report_opts['reportName'] = rawName.replace(u"\u2026","").decode('unicode_escape').encode('ascii','ignore')
        if rawDate != '':
            self.report_opts['date'] = rawDate
        if fileExt != '':
            self.report_opts['ext'] = '.' + fileExt.replace(u"\u2026","").decode('unicode_escape').encode('ascii','ignore')
        if portfolioGroup != '':
            self.report_opts['tabGroup'] = portfolioGroup.replace(u"\u2026","").decode('unicode_escape').encode('ascii','ignore')
        if reportGroup != '':
            self.report_opts['reportTab'] = reportGroup.replace(u"\u2026","").decode('unicode_escape').encode('ascii','ignore')
        if url != '':
            self.report_opts['url'] = url.replace(u"\u2026","").decode('unicode_escape').encode('ascii','ignore')

    @property
    def name_pre_download(self):
        constructed_name = (self.trustee_deal_name.lower().replace(' ', '-').replace('.', '') +
                            "-" + self.raw_name.lower().replace(' ', '-').replace('.', '').replace('---', ' - ') +
                            "-" + self.raw_date.replace("/", '-') +"."+ self.file_extension.lower())
        return constructed_name


from __future__ import print_function, division, absolute_import
import sys, os, copy
from libtbx.phil import parse
from dials.array_family import flex
from dxtbx.model.experiment_list import ExperimentListFactory, ExperimentList
from dials.algorithms.spot_prediction import StillsReflectionPredictor
from scitbx.matrix import col
from libtbx.easy_pickle import dump

phil_scope = parse('''
  grid_search_jungfrau {
    spotfinding_directory = None
      .type = str
      .help = Provide directory with results from spotfinding i.e strong.refl and imported.expt
    rayonix_expt_directory = None
      .type = str
      .help = Provide directory with results from rayonix indexing. Will need integrated.expt
    jungfrau_geometry_file = None
      .type = str
      .help = path to jungfrau geometry file
    minimum_number_of_spots = 3
      .type = int
      .help = minimum number of spots found on jungfrau to pursue further processing
    mosaicity_Deff = 1093.0
      .type = float
      .help = Set crystal Deff value if provided
    mosaicity_eta_half = 0.0001
      .type = float
      .help = Set crystal eta half value if provided
    grid_search_bins_x = 10
      .type = int
      .help = number of increments in x-direction on either side of mean value i.e +/- grid_search_bins_x*dx_mm
    grid_search_bins_y = 10
      .type = int
      .help = number of increments in y-direction on either side of mean value i.e +/- grid_search_bins_y*dy_mm
    grid_search_bins_z = 10
      .type = int
      .help = number of increments in z-direction on either side of mean value i.e +/- grid_search_bins_z*dz_mm
    dx_mm = 0.177
      .type = float
      .help = increment length in grid search x-direction
    dy_mm = 0.177
      .type = float
      .help = increment length in grid search y-direction
    dz_mm = 0.177
      .type = float
      .help = increment length in grid search z-direction
    use_mpi = False
      .type = bool
      .help = Whether to use MPI or not (striping mode)
    outdir = .
      .type = str
      .help = specify directory to dump output
    dump_intermediate_files = False
      .type = bool
      .help = flag to indicate whether you want to dump intermediate grid search files aka \
              refl/expt files from each prediction of each perturbed detector distance
    critical_pixel_dist = 10
      .type = float
      .help = critical pixel distance used to evaluate whether prediction is close enough to a strong spot
    critical_number_of_spots_within_cutoff = 2
      .type = int
      .help = Prints out the dx,dy,dz if atleast these many spots are within the critical_pixel_dist
}
''')

def params_from_phil(args):
  user_phil = []
  for arg in args:
    if os.path.isfile(arg):
      user_phil.append(parse(file_name=arg))
    else:
      try:
        user_phil.append(parse(arg))
      except Exception as e:
        raise Sorry("Unrecognized argument: %s"%arg)
  params = phil_scope.fetch(sources=user_phil).extract()
  return params

class GridSearch_Jungfrau(object):
  ''' Class to do a grid search of the jungfrau detector model
       Reads in a bunch of imported.expt and strong.refl files
       Filters if enough spots are there to do the grid search
       Then predicts spots based on rayonix crystal model for each point on grid'''

  def __init__(self):
    # Read phil stuff
    self.params = params_from_phil(sys.argv[1:])

  def setup_grid(self):
    dx_mm = self.params.grid_search_jungfrau.dx_mm
    dy_mm = self.params.grid_search_jungfrau.dy_mm
    dz_mm = self.params.grid_search_jungfrau.dz_mm
    self.dx = [0.0]
    self.dy = [0.0]
    self.dz = [0.0]
    for ix in range(1, self.params.grid_search_jungfrau.grid_search_bins_x+1):
      self.dx.append(ix*dx_mm)
      self.dx.append(-ix*dx_mm)
    for iy in range(1, self.params.grid_search_jungfrau.grid_search_bins_y+1):
      self.dy.append(iy*dy_mm)
      self.dy.append(-iy*dy_mm)
    for iz in range(1, self.params.grid_search_jungfrau.grid_search_bins_z+1):
      self.dz.append(iz*dz_mm)
      self.dz.append(-iz*dz_mm)

  def do_work(self, rank, iterable):
    for item in iterable:
      self.trial_result = [] # a list to store trial result evaluation
      ts, strong_refl_f, imported_expt_f, integrated_rayonix_expt_f = item

      strong_refl = flex.reflection_table.from_file(strong_refl_f)
      if len(strong_refl) <self.params.grid_search_jungfrau.minimum_number_of_spots: continue
      imported_expt = ExperimentListFactory.from_json_file(imported_expt_f, check_format=False)
      integrated_rayonix_expt = ExperimentListFactory.from_json_file(integrated_rayonix_expt_f, check_format=False)

      imported_expt[0].detector = self.jungfrau_geom[0].detector
      imported_expt[0].crystal = integrated_rayonix_expt[0].crystal
      imported_expt[0].beam = integrated_rayonix_expt[0].beam # Absolutely must use rayonix beam because ebeam is wrong here
      imported_expt[0].crystal.set_domain_size_ang(self.params.grid_search_jungfrau.mosaicity_Deff)
      imported_expt[0].crystal.set_half_mosaicity_deg(self.params.grid_search_jungfrau.mosaicity_eta_half)

      # This is the fun part aka grid search
      original_detector = copy.deepcopy(imported_expt[0].detector)
      d0 = original_detector.hierarchy()
      f0 = col(d0.get_local_fast_axis())
      s0 = col(d0.get_local_slow_axis())
      r0 = col(d0.get_local_origin())
      for dx in self.dx:
        print ('Stage dx', ts)
        for dy in self.dy:
          for dz in self.dz:
            imported_expt[0].detector=original_detector
            displacement = col((dx,dy,dz))
            det0 = imported_expt[0].detector.hierarchy()
            df = col((f0[0]*displacement[0], f0[1]*displacement[1], f0[2]*displacement[2]))
            ds = col((s0[0]*displacement[0], s0[1]*displacement[1], s0[2]*displacement[2]))
            dorigin = col((0,0,dz))
            r_new = r0+df+ds+dorigin
            det0.set_local_frame(det0.get_fast_axis(), det0.get_slow_axis(), r_new)

            predictor=StillsReflectionPredictor(imported_expt[0])
            ubx=predictor.for_ub(imported_expt[0].crystal.get_A())
            ubx['id']=flex.int(len(ubx),0)
            #print ('NUMBER OF PREDS=', len(ubx), ts, dx,dy,dz)
            #if len(ubx) == len(strong_refl):
            #  print ('NUMBER_OF_EXACT_PREDS=', len(ubx), ts, dx,dy,dz)
            self.evaluate_trial(strong_refl, ubx, imported_expt[0].detector, [dx,dy,dz])
            if self.params.grid_search_jungfrau.dump_intermediate_files:
              imported_expt.as_file(os.path.join(self.params.grid_search_jungfrau.outdir, 'prediction_%s_%.2f_%.2f_%.2f.expt'%(ts,dx,dy,dz)))
              ubx.as_file(os.path.join(self.params.grid_search_jungfrau.outdir, 'prediction_%s_%.2f_%.2f_%.2f.refl'%(ts,dx,dy,dz)))
      print ('Final_result_max: ', max(self.trial_result), self.trial_result.index(max(self.trial_result)), ts)
      dump(os.path.join(self.params.grid_search_jungfrau.outdir, 'trial_result_%s.pickle'%ts), self.trial_result)
      #from IPython import embed; embed(); exit()

  def evaluate_grid_search(self):
    pass

  def evaluate_trial(self, strong_refl, predicted_refl, detector, displacement_array):
    pixel_size = detector[0].get_pixel_size()[0]
    count = 0
    for ii, strong in enumerate(strong_refl.rows()):
      for jj, predicted in enumerate(predicted_refl.rows()):
        pstrong = detector[strong['panel']]
        ppredicted = detector[predicted['panel']]
        xyobs_px=strong['xyzobs.px.value'][0:2]
        xypred_px = predicted['xyzcal.px'][0:2]
        r_strong = col(pstrong.get_pixel_lab_coord(xyobs_px))
        r_predicted = col(ppredicted.get_pixel_lab_coord(xypred_px))
        dpx = (r_strong - r_predicted).length()/pixel_size
        if dpx < self.params.grid_search_jungfrau.critical_pixel_dist:
          count +=1
    if count >= self.params.grid_search_jungfrau.critical_number_of_spots_within_cutoff:
      print ('Spots within critical pixel RMSD cutoff = %d %.3f %.3f %.3f'%(count, displacement_array[0], displacement_array[1], displacement_array[2]))
    self.trial_result.append(count)

  def run(self):
    # Figure out what data to process
    # First read in the spotfinding directory and create a list of timestamps and folders to read in
    # Set up the iterable variable which will be distributed to all ranks in case of MPI
    iterable = []
    self.jungfrau_geom = ExperimentListFactory.from_json_file(self.params.grid_search_jungfrau.jungfrau_geometry_file, check_format=False)
    for f in os.listdir(self.params.grid_search_jungfrau.spotfinding_directory):
      if 'strong.refl' in f:
        strong_refl_f = os.path.join(self.params.grid_search_jungfrau.spotfinding_directory, f)
        ts = f[16:16+17]

        imported_expt_f = os.path.join(self.params.grid_search_jungfrau.spotfinding_directory, 'idx-jungfrauhit_%s_imported.expt'%ts)
        # Load up the rayonix integrated model
        integrated_rayonix_expt_f = os.path.join(self.params.grid_search_jungfrau.rayonix_expt_directory,'idx-%s_integrated.expt'%ts)
        #print (ts, strong_refl, imported_expt, integrated_rayonix_expt)
        # Make sure these files exist before proceeding
        if not os.path.exists(imported_expt_f) or not os.path.exists(integrated_rayonix_expt_f): continue
        iterable.append((ts, strong_refl_f, imported_expt_f, integrated_rayonix_expt_f))

    # Set up the grid
    self.setup_grid()

    # Process the data
    if self.params.grid_search_jungfrau.use_mpi:
      from mpi4py import MPI
      comm = MPI.COMM_WORLD
      rank = comm.Get_rank()  # each process in MPI has a unique id, 0-indexed
      size = comm.Get_size()  # size: number of processes running in this job

        # Configure the logging
      params=self.params
      if params.grid_search_jungfrau.outdir is None:
        logfile = None
      else:
        log_path = os.path.join(
            params.grid_search_jungfrau.outdir, "log_rank%04d.out" % rank)
        error_path = os.path.join(
            params.grid_search_jungfrau.outdir, "error_rank%04d.out" % rank)
        print("Redirecting stdout to %s" % log_path)
        print("Redirecting stderr to %s" % error_path)
        sys.stdout = open(log_path, "a", buffering=0)
        sys.stderr = open(error_path, "a", buffering=0)
        print("Should be redirected now")

        if True: # Just using the striping mode for MPI
          subset = [item for i, item in enumerate(iterable) if (i + rank) % size == 0]
          self.do_work(rank, subset)
    else:
      self.do_work(0, iterable)

if __name__ == '__main__':
  try:
    script = GridSearch_Jungfrau()
    script.run()
  except Exception as e:
    print (str(e))


from upemtk import *
from random import *

#---------------------------------------------- Variables globales ----------------------------------------------
vitesse = 0.1

largeurFenetre, hauteurFenetre = 400, 400
r = 6
nb_bonus = 5
longueur_raquette = 60
largeur_raquette = 10
nb_vies = 3
#---------------------------------------------- Mouvements de balle et de raquette ----------------------------------------------

#Fonction contrôlant tous les mouvements de la balle et les comportements adoptés suivant les situations
def balle(x, y, xDir, yDir, ax, bx, by):
    cercle(x, y, r, 'white', remplissage='white')

    #------------------------- Partie gérant les rebonds et angles sur la raquette -------------------------

    if x >= ax and x <= bx and y+r >= by:  #si la balle touche la raquette, on l'envoie dans l'autre sens
        yDir = -yDir
        if x >= bx - ((bx - ax) // 2):  #si la balle touche la raquette à droite, on la renvoie vers la droite
            xDir = 1 - ((bx - x) / 100) #plus la balle est à droite sur la raquette, plus elle va aller vers la droite
        if x <= bx - ((bx - ax) // 2):  #si la balle touche la raquette à gauche, on la renvoie vers la gauche
            xDir = -1 + ((x - ax) / 100) #plus la balle est à gauche sur la raquette, plus elle va aller vers la gauche

    #------------------------- Partie gérant les rebons sur les parois de la fenêtre -------------------------

    if y < r:  #si la balle touche la paroi en haut de la fenêtre, elle change de sens
        yDir = -yDir
    if x < r or x > 400 - r: #si la balle arrive à un des bords horizontaux, elle change de sens
        xDir = -xDir

    #------------------------- Déplacement vertical et horizontal de la raquette d'après les directions précédemment obtenues -------------------------
    y += round(vitesse, 1)*2 * yDir   #si yDir > 0, la balle va vers le bas, si yDir < 0, elle va vers le haut
    x += round(vitesse, 1) * xDir     #même principe
    return (round(x, 1), round(y, 1), xDir, yDir) #on renvoie dans un tuple les coordonnées de la balle ainsi que les directions

#Fonction contrôlant le déplacement de la raquette par l'utilisateur et empêchant celle-ci de sortir de la fenêtre
def raquette(ax, ay, bx, by):
    rectangle(ax, ay, bx, by, remplissage='black') #on affiche la raquette à partir de nos coordonnées
    evenement = donne_evenement()      #on capte un évènement sur le clavier
    typeEv = type_evenement(evenement)
    if typeEv == 'Touche':
        nomTouche=touche(evenement)
        if bx < largeurFenetre and ax > 0:
            if nomTouche == 'Right':      #on se déplace à droite ou à gauche selon la touche entrée par l'utilisateur
                bx += 10
                ax += 10
            elif nomTouche == 'Left':
                bx -= 10
                ax -= 10
        elif bx == largeurFenetre:   #si la raquette a atteint le bord droit, la seule possibilité est d'aller vers la gauche
            if nomTouche == 'Left':
                bx -= 10
                ax -= 10
        elif ax == 0: #même chose pour le bord gauche
            if nomTouche == 'Right':
                bx += 10
                ax += 10
    return (ax, ay, bx, by) #on renvoie les coordonnées de la raquette dans un tuple

#---------------------------------------------- Briques et Bonus ----------------------------------------------

#Fonction qui va crée une liste de briques, chaque élément de la liste correspondant à un tuple contenant les divers propriétés de la brique
def coord_briques():
    lst = []
    ay, by, bonus = 0, 0, 0
    for i in range(1, 5):   #on crée 4 lignes de briques
        ax, bx = 0, 0
        ay, by = by, ((hauteurFenetre / 10) - 10) * i
        for j in range(1, 9):  #on crée 10 colonnes de briques (10 briques par ligne)
            ax, bx = bx, ((largeurFenetre / 10) + 10) * j
            resistance = randint(1, 3)  #on attribue une valeur résistance aléatoire entre 1 et 3, qui correspondra au nombre de fois où la brique pourra être touchée avant de se casser
            color = couleur(resistance) #on attribue la couleur correspondante à la résistance
            lst.append((ax, ay, bx, by, color, resistance, bonus))  #on crée notre liste de tuples qui contiennent les coordonnées et tous les attributs de chaque brique
    lst = creer_bonus(lst)[0] #on applique à cette liste notre fonction creer_bonus qui va ajouter aléatoirement des bonus
    dico_bonus = creer_bonus(lst)[1]
    return lst, dico_bonus

#Fonction définissant la couleur d'une brique en fonction de sa propriété résistance
def couleur(resistance):
    if resistance > 0:
        if resistance == 3:
            color = 'DeepPink2'
        if resistance == 2:
            color = 'PaleVioletRed1'
        if resistance == 1:
            color = 'pink1'
        return color

#Fonction qui va ajouter aléatoirement des bonus dans notre liste en changeant la propriété bonus de la brique de 0 vers 1
def creer_bonus(lst):
    dico_bonus = dict()
    for i in range(nb_bonus):
        index_random = randint(0,len(lst)-1) #on crée notre indice aléatoire pour avoir une brique aléatoire
        ax, ay, bx, by = lst[index_random][0], lst[index_random][1], lst[index_random][2], lst[index_random][3] #on conserve les coordonnées de la brique dans laquelle le bonus est contenu
        x_bonus = ax+((bx-ax)/2)
        y_bonus = ay+((by-ay)/2)
        if lst[index_random][6] == 0: #on vérifie que la brique ne contient pas déjà un bonus
            transtyp = list(lst[index_random]) #on va transformer notre tuple en liste pour pouvoir le modifier
            transtyp[6] = 1
            lst[index_random] = tuple(transtyp) #on repasse notre liste en tuple
            dico_bonus[index_random] = (x_bonus, y_bonus)
    return lst, dico_bonus

def affiche_bonus(dico_bonus):
    for i in dico_bonus:
        x_bonus = dico_bonus[i][0]
        y_bonus = dico_bonus[i][1]
        mvt_bonus(x_bonus, y_bonus)
        cercle(x_bonus, y_bonus, r, 'DodgerBlue3', remplissage = 'DodgerBlue3')
    return (x_bonus,y_bonus)

def mvt_bonus(x_bonus, y_bonus):
    if y_bonus < hauteurFenetre  - r:
        y_bonus += 5
    cercle(x_bonus, y_bonus, r, 'DodgerBlue3', remplissage = 'DodgerBlue3')

def briques(x, y, lst, xDir, yDir):
    for coord in range(len(lst) - 1, -1, -1):
        ax, ay, bx, by, color, resistance, bonus = lst[coord][0], lst[coord][1], lst[coord][2], lst[coord][3], lst[coord][4], lst[coord][5], lst[coord][6]
        rectangle(ax, ay, bx, by, remplissage=color)   #on affiche les briques en parcourant la liste et selon les positions dans les tuples
        if (x >= ax and x <= bx and y-r == by) or (x >= ax and x <= bx and y+r == ay):
            yDir = -yDir
            lst[coord] = (ax, ay, bx, by, couleur(resistance - 1), resistance - 1, bonus)

        if (y <= by and y >= ay and x+r >= ax and x-r<= bx) or (y<= by and y>= ay and x-r <= bx and x+r >= ax) :
            xDir = -xDir
            lst[coord] = (ax, ay, bx, by, couleur(resistance - 1), resistance - 1, bonus)

        if resistance == 0:

            lst.remove(lst[coord])

    return (xDir, yDir, lst, resistance)

#---------------------------------------------- Fonctions de vérifications ----------------------------------------------

#Fonction qui va vérifier si la balle atteint le bord bas de la fenêtre, et qui affiche perdu si c'est le cas
def verif_perdu(x, y, r, ax, bx, nb_vies):
    if (x <= ax or x>= bx) and y > 400 - r: #si la balle ne peut plus toucher la raquette, c'est perdu
        if nb_vies > 0:
            texte(largeurFenetre//2, hauteurFenetre//2,"Perdu!","red","center")
            attente_touche()
            efface_tout()
            texte(largeurFenetre//2, hauteurFenetre//2,"Vies : " + str(nb_vies-1),"blue","center")
            attente_touche()
            efface_tout()
        else:
            ferme_fenetre()
        return True

#Fonction qui va vérifier si notre liste de brique est vide, et afficher gagné si c'est le cas
def verif_gagne(lst):
    if len(lst) == 0:
        texte(largeurFenetre//2, hauteurFenetre//2,"Gagné!","green","center")
        attente_touche()
        ferme_fenetre()

def initialise():
    ax = (largeurFenetre // 2) - (longueur_raquette / 2)
    bx = (largeurFenetre // 2) + (longueur_raquette / 2)
    ay, by = hauteurFenetre, hauteurFenetre - 10
    x, y = 200, 230
    xDir, yDir = 0, 1
    lst = coord_briques()[0]
    texte(200, 200,"Commencer?","green","center")
    attente_touche()
    return (ax, bx, ay, by, x, y, xDir, yDir, lst)

if __name__ == "__main__":
    cree_fenetre(largeurFenetre, hauteurFenetre)

#---------------------------------------------- Initialisation des coordonnées de la ballee, de la raquette et des briques ----------------------------------------------
    while nb_vies > 0:
        new = initialise()
        ax, bx, ay, by, x, y, xDir, yDir, lst = new[0], new[1], new[2], new[3], new[4], new[5], new[6], new[7], new[8]
        while True:
            rectangle(0, 0, 400, 400, remplissage='light sky blue') #on a choisi de définir un fond, pour une meilleure visibilité
            raq = raquette(ax, ay, bx, by)
            ball = balle(x, y, xDir, yDir, ax, bx, by)
            ax, ay, bx, by = raq[0], raq[1], raq[2], raq[3] #on redéfinit les coordonnées de la raquette à partir des coordonnées dans le tuple renvoyé par la fonction raquette
            x, y, xDir, yDir = ball[0], ball[1], ball[2], ball[3] #même principe pour la balle
            coord_brique = briques(x, y, lst, xDir, yDir)
            xDir, yDir, lst, resistance = coord_brique[0], coord_brique[1], coord_brique[2], coord_brique[3]
            verif_gagne(lst)
            if verif_perdu(x, y, r, ax, bx, nb_vies):
                nb_vies -= 1
                break
            mise_a_jour()
            efface_tout()


# -*- coding: utf-8 -*-
"""
Created on Tue Mar 20 16:48:04 2018

@author: so041e
"""

# XB boost + Neural network model
#
#
#

import numpy as np
import pandas as pd
import xgboost as xgb
import zipfile,os,sys

from datetime import datetime
from sklearn.preprocessing import LabelEncoder
from sklearn import preprocessing
from sklearn.preprocessing import OneHotEncoder

from xgboost.sklearn import XGBClassifier
#from xgboost.sklearn import LGBMClassifier
#from sklearn.grid_search import GridSearchCV
from sklearn.model_selection import GridSearchCV
from sklearn.preprocessing import StandardScaler

if sys.platform=='win32':
	Data_path="d:/tmp"
elif sys.platform=='linux2':
	Data_path='.'
elif sys.platform=='darwin':
	Data_path="~/tmp"
else:
	print (os.name)

#with zipfile.ZipFile(Data_path+"/zillow/properties_2016.csv.zip") as z:
#	properties=pd.read_csv(z.open('Properties_2016.csv'))
otrain = pd.read_csv('train_data.csv',skiprows=1,names=['imsi','et','byte','man','model','pro','rt','enb2','freq','bw','rsrp','rsrq','lt','uen','thr'])
otest = pd.read_csv('test_data_without_throughput.csv',skiprows=1,names=['imsi','et','byte','man','model','pro','rt','enb2','freq','bw','rsrp','rsrq','lt','uen'])
train=otrain.copy()
test=otest.copy()
cols=train.columns
#all_cols.sort()

#[(0, 'imsi'),             too many 	 not sure
# (1, 'event_time'),       all 0,      no
# (2, 'bytes'),            int         y
# (3, 'manufacturer'),     cat         y
# (4, 'model'),            cat         not sure
# (5, 'protocol'),         cat         y
# (6, 'rrc_time'),         all 0,      no
# (7, 'EUTRANCELL_NAME'),  just cell   y
# (8, 'FREQUENCY'),        cat         y
# (9, 'BANDWIDTH'),        int         y
# (10, 'rsrp'),            int         y
# (11, 'rsrq'),            int         y
# (12, 'load_time'),                   no
# (13, 'num_ue'),          int         y
# (14, 'throughput')]      float       TARGET
EXP=False

if EXP:
	train[cols[11]].value_counts().sort_index()
	train[cols[13]].value_counts().sort_index().plot()

train['enb']=train.enb2.apply(lambda x: x.split('_')[0])
test['enb']=test.enb2.apply(lambda x: x.split('_')[0])

# Exclude columns
#nouse_cols=['latitude','longitude','rawcensustractandblock','propertycountylandusecode','unitcnt','roomcnt']
#use_cols=[x for x in all_cols if x not in nouse_cols]
#ndf=properties[use_cols]

#train['et']=train.event_time.apply(lambda x: x.split()[1].split(':')[0])
#train_tar=np.log(train['thr'])  # LOG
train_tar=train['thr']
del train['thr']

#%% Handle nan values
#ndf[col]=properties[col].astype(float)
#ndf[col]=properties[col].astype(float).fillna(0)

#['imsi', 'et', 'byte', 'man', 'model', 'pro', 'rt', 'enb2', 'freq', 'bw','rsrp', 'rsrq', 'lt', 'uen', 'thr']
num_col=['byte','rsrp','rsrq','uen']
#lab_col=['man','model','enb','freq','bw']
lab_col=['imsi','freq','bw'] #'enb','model','man',

for col in lab_col:
	lbl = LabelEncoder()
	vals=train[col].astype('str')
	lbl.fit(vals)
	train[col] = lbl.transform(vals)

	test_vals=test[col].astype('str')
	test[col] = lbl.transform(test_vals)

#OneHotEncoder(categorical_features='all', dtype=<'numpy.float64'>, handle_unknown='error', n_values='auto', sparse=True)
lab_train=[]
lab_test=[]
for col in lab_col:
	ncol=pd.get_dummies(train[col],col)
	lab_train.append(ncol)
	ncol=pd.get_dummies(test[col],col)
	lab_test.append(ncol)
train_lab=pd.concat(lab_train,1)
test_lab=pd.concat(lab_test,1)

SCALER=True
if SCALER:
	scalerX = StandardScaler().fit(train[num_col])  # fit y scaler
	train_num=scalerX.transform(train[num_col])  # fit your pipeline to scaled Y
	test_num=scalerX.transform(test[num_col])  # fit your pipeline to scaled Y
else:
	train_num=train[num_col]
	test_num=test[num_col]

train_data=pd.concat([pd.DataFrame(train_num),train_lab],1)
test_data=pd.concat([pd.DataFrame(test_num),test_lab],1)

dtrain = xgb.DMatrix(train_data, train_tar)
dtest = xgb.DMatrix(test_data)

#%% Merge data and trim outliers
OUTLIER=False
if OUTLIER:
	train_df = train.merge(nnndf, how='left', on='parcelid')
	x_train=train_df.drop(['parcelid', 'logerror','transactiondate'], axis=1)
	y_train = train_df["logerror"].values.astype(np.float32)

	# Trim outliers for training
	train_df_trim=train_df[ (train_df.logerror > -0.4) & (train_df.logerror < 0.42) ]
	x_train_trim=train_df_trim.drop(['parcelid', 'logerror','transactiondate'], axis=1)
	y_train_trim = train_df_trim["logerror"].values.astype(np.float32)
	x_test = nnndf.drop(['parcelid'], axis=1)

	print('After removing outliers:')
	print('Shape train: {}\nShape test: {}'.format(x_train.shape, x_test.shape))

#%% Parameter tuning
xgb_params = {
		'eta': 0.11,
		'max_depth': 4,
		'subsample': 0.95,
		'objective': 'reg:linear',
		'eval_metric': 'mae',  #mae
		#'feval':evalerror,
		'base_score': train_data.mean(),
		#'nthread':8,
		'min_child_weight':2, #default 1
		'alpha':3.0, #default 0 - L1 regularize
		'lambda':1.0,
		#'gamma':0,
		#'max_leaves':0,
		#'grow_policy':'lossguide',
		#'nthread':8,
		'silent': 0
}
#%%
param = {'alpha':4.9,'max_depth':15, 'eta':0.8, 'silent':0, 'objective':'reg:linear'} #'min_child_weight':10,
#num_round = 5
#bst = xgb.train(param, dtrain, num_round)
CrossVal=True
if CrossVal:
	cv_result = xgb.cv(param, dtrain, nfold=10, num_boost_round=40)
	cv_result[['test-rmse-mean','train-rmse-mean']].plot()
	print (cv_result.tail(1))

#%%
TRAIN_WHOLE=True
if TRAIN_WHOLE:
	param = {'alpha':4.9, 'max_depth':25, 'eta':0.8, 'silent':0, 'objective':'reg:linear'} #'min_child_weight':10,
	model = xgb.train(param, dtrain, num_boost_round=10)
	pred = model.predict(dtest)

otest['Throughput (in Kbps)']=pred #np.exp(pred) #LOG
otest[['imsi','Throughput (in Kbps)']].to_csv('gv1.csv',index=False)

#%%
sys.exit()

xgb_params[parameter]=vv
cv_result = xgb.cv(xgb_params, dtrain, nfold=5, num_boost_round=500, early_stopping_rounds=250,  verbose_eval=10, show_stdv=False)
cv_results[(parameter,vv)]=cv_result.tail(1)
cv_result[['test-mae-mean','train-mae-mean']].plot()

CV=False
if CV:
	cv_results={}
	parameter='alpha'
	values=[3]
	for vv in values:
		xgb_params[parameter]=vv
		cv_result = xgb.cv(xgb_params, dtrain, nfold=5, num_boost_round=500, early_stopping_rounds=250,  verbose_eval=10, show_stdv=False)
		cv_results[(parameter,vv)]=cv_result.tail(1)
		cv_result[['test-mae-mean','train-mae-mean']].plot()
		print(cv_results)

	cvr=pd.concat(cv_results)
	cvr[['test-mae-mean','train-mae-mean']].plot()
	print (cvr)

#%%
TRAIN_WHOLE=True
if TRAIN_WHOLE:
	model = xgb.train(dict(xgb_params), dtrain, num_boost_round=NUM_ROUND)

	# Validate the model ------------------------------------------------------
	y_train_pre = model.predict(xgb.DMatrix(x_train_sc)).round(4)
	y_train_trim_pre = model.predict(xgb.DMatrix(x_train_trim_sc)).round(4)

	train_loss=y_train-y_train_pre
	train_trim_loss=y_train_trim-y_train_trim_pre
	print ("\nUnTrain Loss:\t", sum(abs(y_train)).round(),sum(y_train).round())
	print ("\nTrain Loss:\t",  sum(abs(train_loss)).round(),sum(train_loss).round())
	print ("\nTrim Tr Loss:\t",  sum(abs(train_trim_loss)).round(),sum(train_trim_loss).round())

#%%
TEST = True

if TEST:
	# Test -----------------------------------------------------------------------		x_test_sc=scalerX.transform(x_test.fillna(-1))  # fit your pipeline to scaled Y
	x_test_sc=scalerX.transform(x_test.fillna(-1))  # fit your pipeline to scaled Y
	y_pred = model.predict(xgb.DMatrix(x_test_sc))

#%%
sys.exit()


""" AuthenticationHandler """

from os import environ
from flask import request, g
from flask.wrappers import Request, Response
from flask_oidc import OpenIDConnect
from flask_nameko import FlaskPooledClusterRpcProxy
from requests import post, exceptions
from jwt import decode
from typing import Callable, Union

from .response import ResponseParser, APIException

class AuthenticationHandler:
    """The AuthenticationHandler connects to the user service and verifies if the bearer token
    send by the user is valid.
    """

    def __init__(self, response_handler: ResponseParser, rpc_proxy: FlaskPooledClusterRpcProxy, oidc: OpenIDConnect):
        self._res = response_handler
        self._rpc = rpc_proxy
        self._oidc = oidc

    def oidc(self, f):
        def decorator(*args, **kwargs):
            try:
                if g.oidc_id_token is None:
                    return self._oidc.redirect_to_auth_server(request.url)

                user_id = self._oidc.user_getfield("sub")

                if not self._oidc.user_getfield("email_verified"):
                    raise APIException(
                        msg="The email address of user {0} is not verified."\
                            .format(user_id),
                        code=401,
                        service="gateway",
                        internal=False)

                return f(user_id=user_id)
            except exceptions.HTTPError as exc:
                raise APIException(
                    msg=str(exc),
                    code=401,
                    service="gateway",
                    internal=False)
            except Exception as exc:
                return self._res.error(exc)
        return decorator

    def check_role(self, f, role):
        def decorator(user_id=None):
            try:
                token = self._parse_auth_header(request)
                roles = self._get_roles(token)

                if role not in roles:
                    raise APIException(
                        msg="The user {0} is not authorized to access this ressources."\
                            .format(user_id),
                        code=403,
                        service="gateway",
                        internal=False)

                return f(user_id=user_id)
            except Exception as exc:
                return self._res.error(exc)
        return decorator

    def _get_roles(self, token):
        roles = []
        if token:
            token_decode = decode(token, algorithms=['HS256'], verify=False)
            roles = token_decode["resource_access"]["openeo"]["roles"]
        return roles

    def _parse_auth_header(self, req: Request) -> Union[str, Exception]:
        """Parses and returns the bearer token. Raises an AuthenticationException if the Authorization
        header in the request is not correct.

        Arguments:
            req {Request} -- The Request object

        Returns:
            Union[str, Exception] -- Returns the bearer token as string or raises an exception
        """

        if "Authorization" not in req.headers or not req.headers["Authorization"]:
            raise APIException(
                msg="Missing 'Authorization' header.",
                code=400,
                service="gateway",
                internal=False)

        token_split = req.headers["Authorization"].split(" ")

        if len(token_split) != 2 or token_split[0] != "Bearer":
            raise APIException(
                msg="Invalid Bearer token.",
                code=401,
                service="gateway",
                internal=False)

        return token_split[1]

    # !!! INFO: Replaced Bearer Tokens with OIDC
    # def bearer(self, f: Callable) -> Union[Callable, Response]:
    #     """Decorator function to authenticate the user, using a bearer token
    #     that is contained within the response header. Returns a HTTP error if the
    #     bearer token is not valid.

    #     Arguments:
    #         f {Callable} -- The wrapped function

    #     Returns:
    #         Union[Callable, Response] -- Returns the decorator function or a HTTP error
    #     """
    #     # TODO: Decorator for authorization -> or (admin and not user["admin"])

    #     def decorator(*args, **kwargs):
    #         try:
    #             rpc_response = self._rpc.auth.identify(token)

    #             if rpc_response["status"] == "error":
    #                 raise self._res.error(rpc_response)

    #             user = rpc_response["data"]

    #             if not user["active"]:
    #                 raise APIException(
    #                     msg="User is not active.",
    #                     code=response.status_code,
    #                     servide="gateway",
    #                     internal=False)

    #             return f(user_id=user["user_id"])
    #         except exceptions.HTTPError as exc:
    #             raise APIException(
    #                 msg=str(exc),
    #                 code=response.status_code,
    #                 service="gateway",
    #                 internal=False)
    #         except Exception as exc:
    #             return self._res.error(exc)
    #     return decorator


from __future__ import absolute_import

from keras import backend as K
from keras import optimizers
from keras import initializers

from keras.metrics import binary_crossentropy, mean_squared_error

from scipy.stats.stats import pearsonr

from default_utils import set_seed as set_seed_defaultUtils

import warnings
with warnings.catch_warnings():
    warnings.filterwarnings("ignore", category=DeprecationWarning)
    from sklearn.metrics import r2_score

def set_seed(seed):
    set_seed_defaultUtils(seed)

    if K.backend() == 'tensorflow':
        import tensorflow as tf
        tf.set_random_seed(seed)
        # session_conf = tf.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1)
        # sess = tf.Session(graph=tf.get_default_graph(), config=session_conf)
        # K.set_session(sess)

        # Uncommit when running on an optimized tensorflow where NUM_INTER_THREADS and
        # NUM_INTRA_THREADS env vars are set.
        # session_conf = tf.ConfigProto(inter_op_parallelism_threads=int(os.environ['NUM_INTER_THREADS']),
        #	intra_op_parallelism_threads=int(os.environ['NUM_INTRA_THREADS']))
        # sess = tf.Session(graph=tf.get_default_graph(), config=session_conf)
        # K.set_session(sess)

def get_function(name):
    mapping = {}

    mapped = mapping.get(name)
    if not mapped:
        raise Exception('No keras function found for "{}"'.format(name))

    return mapped

def build_optimizer(type, lr, kerasDefaults):

    if type == 'sgd':
        return optimizers.SGD(lr=lr, decay=kerasDefaults['decay_lr'],
                              momentum=kerasDefaults['momentum_sgd'],
                              nesterov=kerasDefaults['nesterov_sgd'])#,
            #clipnorm=kerasDefaults['clipnorm'],
            #clipvalue=kerasDefaults['clipvalue'])

    elif type == 'rmsprop':
        return optimizers.RMSprop(lr=lr, rho=kerasDefaults['rho'],
                                  epsilon=kerasDefaults['epsilon'],
                                  decay=kerasDefaults['decay_lr'])#,
            #clipnorm=kerasDefaults['clipnorm'],
#clipvalue=kerasDefaults['clipvalue'])

    elif type == 'adagrad':
        return optimizers.Adagrad(lr=lr,
                                  epsilon=kerasDefaults['epsilon'],
                                  decay=kerasDefaults['decay_lr'])#,
                                  #clipnorm=kerasDefaults['clipnorm'],
                                  #clipvalue=kerasDefaults['clipvalue'])

    elif type == 'adadelta':
        return optimizers.Adadelta(lr=lr, rho=kerasDefaults['rho'],
                                   epsilon=kerasDefaults['epsilon'],
                                   decay=kerasDefaults['decay_lr'])#,
                                    #clipnorm=kerasDefaults['clipnorm'],
#clipvalue=kerasDefaults['clipvalue'])

    elif type == 'adam':
        return optimizers.Adam(lr=lr, beta_1=kerasDefaults['beta_1'],
                               beta_2=kerasDefaults['beta_2'],
                               epsilon=kerasDefaults['epsilon'],
                               decay=kerasDefaults['decay_lr'])#,
                               #clipnorm=kerasDefaults['clipnorm'],
                               #clipvalue=kerasDefaults['clipvalue'])

# Not generally available
#    elif type == 'adamax':
#        return optimizers.Adamax(lr=lr, beta_1=kerasDefaults['beta_1'],
#                               beta_2=kerasDefaults['beta_2'],
#                               epsilon=kerasDefaults['epsilon'],
#                               decay=kerasDefaults['decay_lr'])

#    elif type == 'nadam':
#        return optimizers.Nadam(lr=lr, beta_1=kerasDefaults['beta_1'],
#                               beta_2=kerasDefaults['beta_2'],
#                               epsilon=kerasDefaults['epsilon'],
#                               schedule_decay=kerasDefaults['decay_schedule_lr'])

def build_initializer(type, kerasDefaults, seed=None, constant=0.):

    if type == 'constant':
        return initializers.Constant(value=constant)

    elif type == 'uniform':
        return initializers.RandomUniform(minval=kerasDefaults['minval_uniform'],
                                  maxval=kerasDefaults['maxval_uniform'],
                                  seed=seed)

    elif type == 'normal':
        return initializers.RandomNormal(mean=kerasDefaults['mean_normal'],
                                  stddev=kerasDefaults['stddev_normal'],
                                  seed=seed)

# Not generally available
#    elif type == 'glorot_normal':
#        return initializers.glorot_normal(seed=seed)

    elif type == 'glorot_uniform':
        return initializers.glorot_uniform(seed=seed)

    elif type == 'lecun_uniform':
        return initializers.lecun_uniform(seed=seed)

    elif type == 'he_normal':
        return initializers.he_normal(seed=seed)

def xent(y_true, y_pred):
    return binary_crossentropy(y_true, y_pred)

def mse(y_true, y_pred):
    return mean_squared_error(y_true, y_pred)

def covariance(x, y):
    return K.mean(x * y) - K.mean(x) * K.mean(y)

def corr(y_true, y_pred):
    cov = covariance(y_true, y_pred)
    var1 = covariance(y_true, y_true)
    var2 = covariance(y_pred, y_pred)
    return cov / (K.sqrt(var1 * var2) + K.epsilon())

def evaluate_autoencoder(y_pred, y_test):
    mse = mean_squared_error(y_pred, y_test)
    r2 = r2_score(y_test, y_pred)
    corr, _ = pearsonr(y_pred.flatten(), y_test.flatten())
    # print('Mean squared error: {}%'.format(mse))
    return {'mse': mse, 'r2_score': r2, 'correlation': corr}