import os, uuid
import yaml
from shutil import copyfile
import numpy as np
import time

def __setup_directory():
    exp_dir = 'exp-' + str(uuid.uuid4())[0:8]
    os.mkdir(exp_dir)
    cwd = os.getcwd()
    return cwd + '/' + exp_dir

def __create_settings(exp_dir, loader_batch_size, loader_progress_thread,
                      pep_num_threads, pep_ibatch_size, pep_obatch_size):
#     settings_sh_in = os.path.dirname(os.path.abspath(__file__)) + '/scripts/settings.sh.in'
    settings_sh_in = '/lus/theta-fs0/projects/OptADDN/hepnos/github/HEPnOS-Autotuning/theta/scripts/settings.sh.in'
    settings_sh = exp_dir + '/settings.sh'
    copyfile(settings_sh_in, settings_sh)
    with open(settings_sh, 'a+') as f:
        f.write('\n')
        if loader_progress_thread:
            f.write('HEPNOS_LOADER_CLIENT_USE_PROGRESS_THREAD=-a\n')
        else:
            f.write('HEPNOS_LOADER_CLIENT_USE_PROGRESS_THREAD=\n')
        f.write('HEPNOS_LOADER_CLIENT_BATCH_SIZE=%d\n' % loader_batch_size)
        if None not in [pep_num_threads, pep_ibatch_size, pep_obatch_size]:
            f.write('HEPNOS_ENABLE_PEP=1\n')
            f.write('HEPNOS_PEP_THREADS=%d\n' % pep_num_threads)
            f.write('HEPNOS_PEP_IBATCH_SIZE=%d\n' % pep_ibatch_size)
            f.write('HEPNOS_PEP_OBATCH_SIZE=%d\n' % pep_obatch_size)
        else:
            f.write('HEPNOS_ENABLE_PEP=0\n')

def __generate_config_file(
        exp_dir='.',
        filename='config.yaml',
        threads=0,
        busy_spin=False,
        targets=1):
    config = dict()
    config['address'] = 'ofi+gni://'
    config['threads'] = int(threads)
    config['busy-spin'] = bool(busy_spin)
    config['databases'] = dict()
    config['databases']['datasets'] = dict()
    for k in ['datasets', 'runs', 'subruns', 'events', 'products']:
        config['databases'][k] = dict()
        d = config['databases'][k]
        d['name'] = 'hepnos-%s.$RANK.$PROVIDER.$TARGET' % k
        d['path'] = '/dev/shm/$RANK'
        d['type'] = 'map'
        d['targets'] = 1
        d['providers'] = 1
    config['databases']['events']['targets'] = int(targets)
    config['databases']['products']['targets'] = int(targets)
    with open(exp_dir+'/'+filename, 'w+') as f:
        f.write(yaml.dump(config))

def __parse_result(exp_dir):
    dataloader_time = 0
    pep_time = 0
    for line in open(exp_dir+'/dataloader-output.txt'):
        if 'real' in line:
            line = line.replace('s','')
            x = line.split()[1]
            m = int(x.split('m')[0])
            s = float(x.split('m')[1])
            dataloader_time = m*60 + s
    if os.path.isfile(exp_dir+'/pep-output.txt'):
        for line in open(exp_dir+'/pep-output.txt'):
            if 'real' in line:
                line = line.replace('s','')
                x = line.split()[1]
                m = int(x.split('m')[0])
                s = float(x.split('m')[1])
                pep_time = m*60 + s
    return (dataloader_time, pep_time)

def run(args):
    if len(args) == 5:
        args.extend([None, None, None])
    if len(args) != 8:
        raise RuntimeError("Expected 5 or 8 arguments in list, found %d" % len(args))
    hepnos_num_threads = args['num_threads'] # args[0]
    hepnos_num_databases = args['num_databases'] # args[1]
    busy_spin = args['busy_spin'] # args[2]
    loader_progress_thread = args['progress_thread'] # args[3]
    loader_batch_size = args['batch_size'] # args[4]
    pep_num_threads = args['num_threads_b'] #args[5]
    pep_ibatch_size = args['batch_size_in'] #args[6]
    pep_obatch_size = args['batch_size_out'] #args[7]
    print('Setting up experiment\'s directory')
    exp_dir = __setup_directory()
    print('Creating settings.sh')
    __create_settings(exp_dir,
                      loader_batch_size,
                      loader_progress_thread,
                      pep_num_threads,
                      pep_ibatch_size,
                      pep_obatch_size)
    print('Creating config.yaml')
    __generate_config_file(
            exp_dir,
            threads=hepnos_num_threads,
            busy_spin=busy_spin,
            targets=hepnos_num_databases)
    print('Submitting job')
#     submit_sh = os.path.dirname(os.path.abspath(__file__)) + '/scripts/submit.sh'
    submit_sh ='/lus/theta-fs0/projects/OptADDN/hepnos/github/HEPnOS-Autotuning/theta/scripts/submit.sh'
    os.system(submit_sh + ' ' + exp_dir)
    print('Parsing result')
    t = __parse_result(exp_dir)
    print('Done (loading time = %f, processing time = %f)' % (t[0], t[1]))
    return 1 / (t[0]+t[1])

if __name__ == '__main__':
#     run([ 31, 16, False, False, 1024 ])
    config = {
        'num_threads': 31,
        'num_databases': 1,
        'busy_spin': False,
        'progress_thread': False,
        'batch_size': 1024,
        'num_threads_b': 31,
        'batch_size_in': 32,
        'batch_size_out': 1024
    }
    objective = run(config)
    print('objective: ', objective)
    print('real run time: ', 1 / objective)
#     import matplotlib.pyplot as plt
#     plt.plot(HISTORY['val_r2'])
#     plt.xlabel('Epochs')
#     plt.ylabel('Objective: $R^2$')
#     plt.grid()
#     plt.show()


#!/usr/bin/env python3
# -*- coding: utf8 -*-

#YakDB commandline tool
import argparse
import sys
import code
import os
import os.path
import YakDB
import YakDB.Dump
from YakDB.Iterators import KeyIterator, KeyValueIterator

def info(db, args):
    print((db.serverInfo()).decode("utf-8"))

def tableInfo(db, args):
    print(db.tableInfo(args.table))

def stop(db, args):
    db.stopServer()

def read(db, args):
    tableNo = args.tableNo
    keys = args.keys
    output = db.read(tableNo, keys) #Always use custom mapping
    #Convert value-only to key-->value map if not --print-raw is set
    if args.printRaw:
        for line in output:
            print(line)
    else:
        outMap = {}
        for index, inval in enumerate(keys):
            outMap[inval] = output[index]
        print(outMap)

def put(db, args):
    tableNo = args.tableNo
    keysValues = args.keyValuePairs
    #Build the key/value dict
    if len(keysValues) % 2 != 0:
        print(("Didn't find a value for key %s, ignoring that key" % keysValues[-1]))
        keysValues = keysValues[0:-1]
    keys = keysValues[0::2]
    values = keysValues[1::2]
    putDict = dict(list(zip(keys, values)))
    db.put(tableNo, putDict)
    #Convert value-only to key-->value map
    if not args.quiet:
        for key, value in putDict.items():
            print(("Successfully put '%s' --> '%s'" % (key, value)))

def exists(db, args):
    tableNo = args.tableNo
    keys = args.keys
    result = db.exists(tableNo, keys)
    if args.printNumeric:
        result = [1 if x else 0 for x in result]
    #Convert value-only to key-->value map
    if args.printRaw:
        for value in result:
            print(value)
    else: #Print dictionary
        outMap = {}
        for index, inval in enumerate(keys):
            outMap[inval] = result[index]
        print(outMap)

def delete(db, args):
    tableNo = args.tableNo
    keys = args.keys
    db.delete(tableNo, keys)
    if not args.quiet:
        print(("Deleted [%s]" % ", ".join(keys)))

def deleteRange(db, args):
    tableNo = args.tableNo
    fromKey = args.fromKey
    toKey = args.toKey
    limit = args.scanLimit
    #Data is remapped into dictionary-form in connection class
    db.deleteRange(tableNo, fromKey, toKey, limit)

def scan(db, args):
    tableNo = args.tableNo
    #Override -t with positional argument, if any
    if args.table is not None:
        tableNo = args.table
    fromKey = args.fromKey
    toKey = args.toKey
    limit = args.scanLimit
    keyFilter = args.keyFilter
    valueFilter = args.valueFilter
    invert = args.invertDirection
    skip = args.skip
    #Lazily iterate over key value tuples
    it = KeyValueIterator(db, tableNo, fromKey, toKey, limit, keyFilter=keyFilter, valueFilter=valueFilter, skip=skip, invert=invert)
    for key, value in it:
        if args.onlyKeys:
            #Print key but strip "b'" and "'"
            print(str(key)[2:-1])
        else: #Print k,v
            print("{0},{1}".format(key, value))

def listKeysInRange(db, args):
    tableNo = args.tableNo
    #Override -t with positional argument, if any
    if args.table is not None:
        tableNo = args.table
    fromKey = args.fromKey
    toKey = args.toKey
    limit = args.scanLimit
    keyFilter = args.keyFilter
    valueFilter = args.valueFilter
    invert = args.invertDirection
    skip = args.skip
    codec = args.codec
    #Lazily iterate over the key list
    it = KeyIterator(db, tableNo, fromKey, toKey, limit, keyFilter=keyFilter, valueFilter=valueFilter, skip=skip, invert=invert)
    if codec == "binary":
        for key in it:
            print(key)
    else:
        for key in it:
            print(key.decode(codec))

def dump(db, args):
    tableNo = args.tableNo
    #Override -t with positional argument, if any
    if args.table is not None:
        tableNo = args.table
    fromKey = args.fromKey
    toKey = args.toKey
    limit = args.scanLimit
    outputFile = args.outputFile
    YakDB.Dump.dumpYDF(db, outputFile, tableNo, fromKey, toKey, limit)

def importDump(db, args):
    tableNo = args.tableNo
    #Override -t with positional argument, if any
    if args.table is not None:
        tableNo = args.table
    inputFile = args.inputFile
    if not os.path.exists(inputFile):
        print("Error: Input file '%s' does not exist!" % inputFile, file=sys.stderr)
        sys.exit(1)
    if not os.path.isfile(inputFile):
        print("Error: Input file '%s' is not a file!" % inputFile, file=sys.stderr)
        sys.exit(1)
    YakDB.Dump.importYDFDump(db, inputFile, tableNo)

def count(db, args):
    tableNo = args.tableNo
    #Override -t with positional argument, if any
    if args.table is not None:
        tableNo = args.table
    fromKey = args.fromKey
    toKey = args.toKey
    print((db.count(tableNo, fromKey, toKey)))

def compact(db, args):
    tables = [args.tableNo]
    fromKey = args.fromKey
    toKey = args.toKey
    if len(args.tables) > 0:
        tables = args.tables
    for table in tables:
        db.compactRange(table, fromKey, toKey)
        if not args.quiet:
            print("Compaction finished")

def openTable(db, args):
    compression = args.compression
    mergeOperator = args.mergeOperator
    lruCacheSize = args.lruCacheSize
    writeBufferSize = args.writeBufferSize
    blocksize = args.blocksize
    bloomFilterBitsPerKey = args.bloomFilterBitsPerKey
    for table in args.tables:
        db.openTable(table, lruCacheSize=lruCacheSize, writeBufferSize=writeBufferSize,
                    tableBlocksize=blocksize, bloomFilterBitsPerKey=bloomFilterBitsPerKey,
                    compression=compression, mergeOperator=mergeOperator)
        if not args.quiet:
            print("Opening table #%d finished" % table)

def closeTable(db, args):
    tables = [args.tableNo]
    if len(args.tables) > 0:
        tables = args.tables
    for table in tables:
        db.closeTable(table)
        if not args.quiet:
            print("Closing table #%d finished" % table)

def truncateTable(db, args):
    tables = [args.tableNo]
    if args.tables:
        tables = args.tables
    for table in tables:
        db.truncate(table)
        if not args.quiet:
            print("Truncating table #%d finished" % table)

def repl(db, args):
    code.InteractiveConsole(locals={"db":db}).interact("YakDB REPL -- Connection is available in 'db' variable")

def yakCLI():
    """
    Call this function to use the yak CLI on sys.argv.
    """
    if sys.version_info.major < 3:
        print("YakDB requires Python 3 to run. Please run yak using a python3k interpreter!")

    parser = argparse.ArgumentParser(description="YakDB client tool")
    #Server optionstype=int
    serverArgsGroup = parser.add_argument_group(parser, "Server options")
    serverArgsGroup.add_argument("-s","--server",
            help="Specifies server URL to connect to",
            default="tcp://localhost:7100",
            action='store',
            dest="serverURL")
    serverArgsGroup.add_argument("-g","--server-group",
            help="Specifies the server group to connect to for PUB/SUB connections",
            action='store',
            dest="serverGroup")
    serverArgsGroup.add_argument("-c","--connection-mode",
            choices=["PUB","PUSH","REQ"],
            default="REQ",
            action="store",
            dest="connectMode",
            help="The connection mode to use for requests. Most requests only work in REQ mode.")
    #Database options
    dbOptsGroup = parser.add_argument_group(parser, "Database options")
    dbOptsGroup.add_argument("-t","--table",
            help="The table number / ID to use",
            action="store",
            default=1,
            type=int,
            dest="tableNo")
    #CLI options
    cliOptsGroup = parser.add_argument_group(parser, "CLI options")
    #Data is remapped in connection class
    cliOptsGroup.add_argument("-q","--quiet",
            help="Don't print connection info",
            action="store_true",
            dest="quiet",
            default=False)
    cliOptsGroup.add_argument("-r","--repl",
            help="Start into a REPL where db is the database connection",
            action="store_true",
            default=False,
            dest="repl")
    ###
    #Create parsers for the individual commands
    ###
    subparsers = parser.add_subparsers(title="Commands")
    #Info
    parserInfo = subparsers.add_parser("info", description="Request server information (features and version number)")
    parserInfo.set_defaults(func=info)
    #Info
    parserTableInfo = subparsers.add_parser("table-info", description="Request table information")
    parserTableInfo.add_argument('table',
        type=int,
        nargs='?',
        action="store",
        help="The tables to get info from. Overrides -t option.")
    parserTableInfo.set_defaults(func=tableInfo)
    #Stop
    parserStop = subparsers.add_parser("stop", description="Stop the YakDB server")
    parserStop.set_defaults(func=stop)
    #Read
    parserRead = subparsers.add_parser("read", description="Read the values for one or more keys")
    parserRead.add_argument('keys',
            nargs='+',
            action="store",
            help="The keys to read")
    parserRead.add_argument('-p','--print-raw',
            dest="printRaw",
            action="store_true",
            default=False,
            help="Print only the values read (separated by newline), not a dictionary")
    parserRead.set_defaults(func=read)
    #Put
    parserPut = subparsers.add_parser("put", description="Write a single key-value pair")
    parserPut.add_argument('keyValuePairs',
            action="store",
            nargs="+",
            help="The keys / values to write, in alternating order (key value key2 value2 ...)")
    parserPut.set_defaults(func=put)
    #Exists
    parserExists = subparsers.add_parser("exists", description="Check if one or more keys exist in the table")
    parserExists.add_argument('keys',
            nargs='+',
            action="store",
            help="The keys to check for existence")
    parserExists.add_argument('-n','--numeric-boolean',
            dest="printNumeric",
            action="store_true",
            default=False,
            help="Instead of printing True or False, print 1 or 0")
    parserExists.add_argument('-p','--print-raw',
            dest="printRaw",
            action="store_true",
            default=False,
            help="Do not print a map from key to its existence value, but only the existence values in the same order as the keys")
    parserExists.set_defaults(func=exists)
    #Delete
    parserDelete = subparsers.add_parser("delete", description="Delete one or more keys")
    parserDelete.add_argument('keys',
            nargs='+',
            action="store",
            help="The keys to delete (keys that don't exist will be ignored)")
    parserDelete.set_defaults(func=delete)
    #Delete range
    parserDeleteRange = subparsers.add_parser("delete-range", description="Delete a range of keys")
    parserDeleteRange.add_argument('--from',
            action="store",
            dest="fromKey",
            default=None,
            help="The key to start deleting at (inclusive), default: start of table")
    parserDeleteRange.add_argument('--to',
            action="store",
            dest="toKey",
            default=None,
            help="The key to stop deleting at (exclusive), default: end of table")
    parserDeleteRange.add_argument('-l','--limit',
            action="store",
            dest="scanLimit",
            type=int,
            default=None,
            help="The maximum number of keys to delete")
    parserDeleteRange.set_defaults(func=deleteRange)
    #Scan
    parserScan = subparsers.add_parser("scan", description="Scan over a specified range in the table and return all key-value-pairs in that range")
    parserScan.add_argument('--from',
            action="store",
            dest="fromKey",
            default=None,
            help="The key to start scanning at (inclusive), default: Start of table")
    parserScan.add_argument('--to',
            action="store",
            dest="toKey",
            default=None,
            help="The key to stop scanning at (exclusive), default: end of table")
    parserScan.add_argument('-k','--key-filter',
            action="store",
            dest="keyFilter",
            default=None,
            help="The server-side key filter. Ignored KV pairs don't count when calculating the limit.")
    parserScan.add_argument('-v','--value-filter',
            action="store",
            dest="valueFilter",
            default=None,
            help="The server-side value filter. Ignored KV pairs don't count when calculating the limit.")
    parserScan.add_argument('-l','--limit',
            action="store",
            dest="scanLimit",
            type=int,
            default=None,
            help="The maximum number of keys to scan")
    parserScan.add_argument('-s','--skip',
            type=int,
            nargs='?',
            default=0,
            action="store",
            help="How many records to skip. Only filter-passing records count as skipped.")
    parserScan.add_argument('-i','--invert-direction',
            action="store_true",
            dest="invertDirection",
            default=False,
            help="Invert the scan direction")
    parserScan.add_argument('--only-keys',
            action="store_true",
            dest="onlyKeys",
            default=False,
            help="Only print keys, omit values completely")
    parserScan.add_argument('table',
            type=int,
            nargs='?',
            action="store",
            help="The tables to scan. Overrides -t option.")
    parserScan.set_defaults(func=scan)
    #List
    parserList = subparsers.add_parser("list", description="List keys in a specified range in the table and return all key-value-pairs in that range")
    parserList.add_argument('--from',
            action="store",
            dest="fromKey",
            default=None,
            help="The key to start listing at (inclusive), default: Start of table")
    parserList.add_argument('--to',
            action="store",
            dest="toKey",
            default=None,
            help="The key to stop listing at (exclusive), default: end of table")
    parserList.add_argument('-k','--key-filter',
            action="store",
            dest="keyFilter",
            default=None,
            help="The server-side key filter. Ignored KV pairs don't count when calculating the limit.")
    parserList.add_argument('-v','--value-filter',
            action="store",
            dest="valueFilter",
            default=None,
            help="The server-side value filter. Ignored KV pairs don't count when calculating the limit.")
    parserList.add_argument('-n','--limit',
            action="store",
            dest="scanLimit",
            type=int,
            default=None,
            help="The maximum number of keys to scan")
    parserList.add_argument('-s','--skip',
            type=int,
            nargs='?',
            default=0,
            action="store",
            help="How many records to skip. Only filter-passing records count as skipped.")
    parserList.add_argument('-i','--invert-direction',
            action="store_true",
            dest="invertDirection",
            default=False,
            help="Invert the scan direction")
    parserList.add_argument('-c','--codec',
            action="store_true",
            dest="codec",
            default="utf-8",
            help="Which codec to use for printing (or 'binary')")
    parserList.add_argument('table',
            type=int,
            nargs='?',
            action="store",
            help="The table to list. Overrides -t option.")
    parserList.set_defaults(func=listKeysInRange)
    #Count
    parserCount = subparsers.add_parser("count", description="Count how many keys exist in a specified range of the table")
    parserCount.add_argument('--from',
            action="store",
            dest="fromKey",
            default=None,
            help="The key to start scanning at (inclusive)")
    parserCount.add_argument('--to',
            action="store",
            dest="toKey",
            default=None,
            help="The key to stop scanning at (exclusive)")
    parserCount.add_argument('table',
            type=int,
            nargs='?',
            action="store",
            help="The tables to count in. Overrides -t option.")
    parserCount.set_defaults(func=count)
    #Compact
    parserCompact = subparsers.add_parser("compact", description="Compact a specified range of the table\nThis operation may take a long time, especially for large tables.")
    parserCompact.add_argument('tables',
            nargs='*',
            action="store",
            type=int,
            help="The tables to compact. Overrides -t option.")
    parserCompact.add_argument('--from',
            action="store",
            dest="fromKey",
            default=None,
            help="The key to start compacting at (inclusive), default: start of table")
    parserCompact.add_argument('--to',
            action="store",
            dest="toKey",
            default=None,
            help="The key to stop compacting at (exclusive), default: end of table")
    parserCompact.set_defaults(func=compact)
    #Open table
    parserOpenTable = subparsers.add_parser("open", description="Open a table.\nThis is only neccessary if you intend to use nonstandard open options.")
    parserOpenTable.add_argument('tables',
            nargs='+',
            type=int,
            action="store",
            help="The table ID to open")
    parserOpenTable.add_argument('-c','--compression',
            dest="compression",
            default="SNAPPY",
            help="Blocklevel compression code (NONE, SNAPPY, ZLIB, BZIP2, LZ4 or LZ4HC)")
    parserOpenTable.add_argument('-m','--merge-operator',
            dest="mergeOperator",
            default="REPLACE",
            help="Merge operator (e.g. REPLACE, INT64ADD, DMUL or APPEND)")
    parserOpenTable.add_argument('-l','--lru-cache-size',
            action="store",
            dest="lruCacheSize",
            type=int,
            default=None,
            help="The size of the LRU cache where uncompressed block data will be stored. Increasing this yields better performance for random-read-access-heavy workloads.")
    parserOpenTable.add_argument('-w','--write-buffer-size',
            action="store",
            dest="writeBufferSize",
            type=int,
            default=None,
            help="The size of the write buffer. Increasing this yields better write performance")
    parserOpenTable.add_argument('-b','--blocksize',
            action="store",
            dest="blocksize",
            type=int,
            default=None,
            help="The table blocksize. Larger blocks yield better compression and scan performance, but usually worse random-access performance. See LevelDB docs for details")
    parserOpenTable.add_argument('-f','--bloom-filter-bits-per-key',
            action="store",
            dest="bloomFilterBitsPerKey",
            type=int,
            default=None,
            help="The number of bloom filter bits per key. Increasing this improves read & exist performance for keys that don't exist, but increases memory usage.")
    parserOpenTable.set_defaults(func=openTable)
    #Close table
    parserCloseTable = subparsers.add_parser("close", description="Close a table\nThis is usually not neccessary, unless you want to save memory. ")
    parserCloseTable.add_argument('tables',
            nargs='*',
            type=int,
            action="store",
            help="The tables to compact. Overrides -t option.")
    parserCloseTable.set_defaults(func=closeTable)
    #Close table
    parserTruncateTable = subparsers.add_parser("truncate", description="Close and truncate a table. Deletes the table's files on filesystem-level.")
    parserTruncateTable.add_argument('tables',
            nargs='*',
            type=int,
            action="store",
            help="The tables to compact. Overrides -t option.")
    parserTruncateTable.add_argument('-y','--yes',
            dest="confirmed",
            action="store_true",
            default=False,
            help="Skip the 'Do you really want to truncate?' question")
    parserTruncateTable.set_defaults(func=truncateTable)
    #Dump
    parserDump = subparsers.add_parser("dump", description="Dump a specified range of a table (default: entire table) into a YDF format. Automatically creates a table snapshot for the dump.")
    parserDump.add_argument('-o','--output',
            action="store",
            dest="outputFile",
            required=True,
            help="The file to write the YDF dump to. Append .gz to use gzipped output (recommended)")
    parserDump.add_argument('--from',
            action="store",
            dest="fromKey",
            default=None,
            help="The key to start dumping at (inclusive), default: Start of table")
    parserDump.add_argument('--to',
            action="store",
            dest="toKey",
            default=None,
            help="The key to stop dumping at (exclusive), default: end of table")
    parserDump.add_argument('-l','--limit',
            action="store",
            dest="scanLimit",
            type=int,
            default=None,
            help="The maximum number of keys to scan")
    parserDump.add_argument('table',
            type=int,
            nargs='?',
            action="store",
            help="The table to dump. Overrides -t option.")
    parserDump.set_defaults(func=dump)
    #Import
    parserImport = subparsers.add_parser("import", description="Import a YDF dump")
    parserImport.add_argument('-i','--input',
            action="store",
            dest="inputFile",
            required=True,
            help="The file to read the YDF dump from. Append .gz to use transparent decompression.")
    parserImport.add_argument('table',
            type=int,
            nargs='?',
            action="store",
            help="The table to import to. Overrides -t option.")
    parserImport.set_defaults(func=importDump)
    #REPL
    parserREPL = subparsers.add_parser("repl", description="Start a Read-eval-print loop (REPL) for interactive DB usage")
    parserREPL.set_defaults(func=repl)
    ###
    #Parse and call the function
    ###
    args = parser.parse_args()
    db = YakDB.Connection()
    if not args.quiet:
        sys.stderr.write("Connecting to %s\n" % args.serverURL)
    if args.connectMode == "PUSH":
        db.usePushMode()
    elif args.connectMode == "PUB":
        db.usePubMode()
    else:
        db.useRequestReplyMode()
    db.connect(args.serverURL)
    #For some reason, the default=info setting only works with Python2
    try:
        args.func(db, args)
    except AttributeError as e:
        print(e)
        info(db, args)


import torch
if torch.__version__ >= "1.8":
    import torch_npu
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
import torch.backends.cudnn as cudnn
import torchvision
import torchvision.transforms as transforms
import models
import os
import time
import argparse
import numpy as np

from torch.autograd import Variable

import imdbfolder_coco as imdbfolder
import config_task
import utils_pytorch
import sgd
try:
    from apex import amp
except ImportError:
    apex = None

parser = argparse.ArgumentParser(description='PyTorch Residual Adapters training')
parser.add_argument('--dataset', default='cifar100', nargs='+', help='Task(s) to be trained')
parser.add_argument('--lr', default=0.1, type=float, help='initial learning rate')
parser.add_argument('--wd', default=1., type=float, help='weight decay for the classification layer')
parser.add_argument('--wd3x3', default=1., type=float, nargs='+', help='weight decay for the 3x3')
parser.add_argument('--wd1x1', default=1., type=float, nargs='+', help='weight decay for the 1x1')
parser.add_argument('--nb_epochs', default=120, type=int, help='nb epochs')
parser.add_argument('--step1', default=80, type=int, help='nb epochs before first lr decrease')
parser.add_argument('--step2', default=100, type=int, help='nb epochs before second lr decrease')
parser.add_argument('--mode', default='parallel_adapters', type=str, help='Task adaptation mode')
parser.add_argument('--proj', default='11', type=str, help='Position of the adaptation module')
parser.add_argument('--dropout', default='00', type=str, help='Position of dropouts')
parser.add_argument('--expdir', default='/scratch/shared/nfs1/srebuffi/exp/dem_learning/tmp/', help='Save folder')
parser.add_argument('--datadir', default='/scratch/local/ramdisk/srebuffi/decathlon/', help='folder containing data folder')
parser.add_argument('--imdbdir', default='/scratch/local/ramdisk/srebuffi/decathlon/annotations/', help='annotation folder')
parser.add_argument('--source', default='/scratch/shared/nfs1/srebuffi/exp/dem_learning/C100_alone/checkpoint/ckptpost11bnresidual11cifar1000.000180607060.t7', type=str, help='Network source')
parser.add_argument('--seed', default=0, type=int, help='seed')
parser.add_argument('--factor', default='1.', type=float, help='Width factor of the network')
# Mixed precision training parameters
parser.add_argument('--apex', action='store_true', help='Use apex for mixed precision training')
parser.add_argument('--apex-opt-level', default='O1', type=str,help='For apex mixed precision training'
                        'O0 for FP32 training, O1 for mixed precision training.'
                        'For further detail, see https://github.com/NVIDIA/apex/tree/master/examples/imagenet')
parser.add_argument('--loss-scale-value', default=1024., type=float,
                    help='loss scale using in amp, default -1 means dynamic')
# for ascend 910
parser.add_argument('--device_id', default=5, type=int, help='device id')
args = parser.parse_args()
args.archi ='default'
config_task.mode = args.mode
config_task.proj = args.proj
config_task.factor = args.factor
args.use_npu = torch.npu.is_available()

device = torch.device(f'npu:{args.device_id}')
torch.npu.set_device(device)
print("Use NPU: {} for training".format(args.device_id))

if type(args.dataset) is str:
    args.dataset = [args.dataset]

if type(args.wd3x3) is float:
    args.wd3x3 = [args.wd3x3]

if type(args.wd1x1) is float:
    args.wd1x1 = [args.wd1x1]

if not os.path.isdir(args.expdir):
    os.mkdir(args.expdir)

config_task.decay3x3 = np.array(args.wd3x3) * 0.0001
config_task.decay1x1 = np.array(args.wd1x1) * 0.0001
args.wd = args.wd *  0.0001

args.ckpdir = args.expdir + '/checkpoint/'
args.svdir  = args.expdir + '/results/'

if not os.path.isdir(args.ckpdir):
    os.mkdir(args.ckpdir)

if not os.path.isdir(args.svdir):
    os.mkdir(args.svdir)

config_task.isdropout1 = (args.dropout[0] == '1')
config_task.isdropout2 = (args.dropout[1] == '1')

#####################################

# Prepare data loaders
train_loaders, val_loaders, num_classes = imdbfolder.prepare_data_loaders(args.dataset,args.datadir,args.imdbdir,True)
args.num_classes = num_classes

# Create the network
net = models.resnet26(num_classes)

start_epoch = 0
best_acc = 0  # best test accuracy
results = np.zeros((4,start_epoch+args.nb_epochs,len(args.num_classes)))
all_tasks = range(len(args.dataset))
np.random.seed(1993)

if args.use_npu:
    net.npu()
    cudnn.benchmark = True

args.criterion = nn.CrossEntropyLoss()
optimizer = sgd.SGD(filter(lambda p: p.requires_grad, net.parameters()), lr=args.lr, momentum=0.9, weight_decay=args.wd)

if args.apex:
    net, optimizer = amp.initialize(net, optimizer,
                                      opt_level=args.apex_opt_level,
                                      loss_scale=args.loss_scale_value,
                                      combine_grad=True)

print("Start training")
for epoch in range(start_epoch, start_epoch+args.nb_epochs):
    training_tasks = utils_pytorch.adjust_learning_rate_and_learning_taks(optimizer, epoch, args)
    st_time = time.time()

    # Training and validation
    train_acc, train_loss = utils_pytorch.train(epoch, train_loaders, training_tasks, net, args, optimizer)
    test_acc, test_loss, best_acc = utils_pytorch.test(epoch,val_loaders, all_tasks, net, best_acc, args, optimizer)

    # Record statistics
    for i in range(len(training_tasks)):
        current_task = training_tasks[i]
        results[0:2,epoch,current_task] = [train_loss[i].cpu(),train_acc[i].cpu()]
    for i in all_tasks:
        results[2:4,epoch,i] = [test_loss[i].cpu(),test_acc[i].cpu()]
    np.save(args.svdir+'/results_'+'adapt'+str(args.seed)+args.dropout+args.mode+args.proj+''.join(args.dataset)+'wd3x3_'+str(args.wd3x3)+'_wd1x1_'+str(args.wd1x1)+str(args.wd)+str(args.nb_epochs)+str(args.step1)+str(args.step2),results)
    print('Epoch lasted {0}'.format(time.time()-st_time))


from django.http import HttpResponse
from django.shortcuts import render
from django.contrib.auth.decorators import login_required
from django.shortcuts import render, redirect, get_object_or_404
from change_tasks.models import News, Tasks, Contact
from django.contrib.auth.models import User
from change_tasks.forms import UpdateTaskForm, UserCreationForm, TaskFormAdd
from django.contrib import messages

# главная страница
def home(request):
    # Получаем все новости и сортируем их по дате в обратном порядке
    news = News.objects.order_by("-data")

    # Формируем контекст для шаблона
    context = {
        "news": news,  # передаем список новостей в контекст
    }

    # Рендерим шаблон и передаем ему контекст
    return render(request, "home.html", context)

# регистрация
def register(request):
    # Проверяем метод запроса (POST - создание нового пользователя)
    if request.method == 'POST':
        # Создаем объект формы UserCreationForm и передаем данные из запроса
        form = UserCreationForm(request.POST)
        # Проверяем форму на валидность
        if form.is_valid():
            # Сохраняем данные формы и создаем новый объект пользователя
            form.save()
            # Получаем имя пользователя
            username = form.cleaned_data.get('username')
            # Выводим сообщение об успешной регистрации и перенаправляем на главную страницу
            messages.success(request, f'Аккаунт создан для юзера {username}!')
            return redirect('home')
    else:
        # Если метод GET, то создаем пустую форму
        form = UserCreationForm()
    # Сохраняем форму в контексте и рендерим шаблон регистрации
    context = {
        'form': form
    }
    return render(request, 'register.html', context)

# список пользователей
def list_users(request):
    # Получаем всех пользователей из базы данных
    users = User.objects.all()
    # Создаем словарь с контекстом для передачи в шаблон
    context = {
        "users": users
    }
    # Рендерим шаблон с контекстом
    return render(request, 'list_users.html', context)

# задачи пользователя
# проверяем авторизованы ли мы с помощью декоратора login_required, если да, то заходим на страницу пользователя
@login_required
def user_page(request):
    # Получаем текущего пользователя
    user = request.user
    # Фильтруем задачи, которые назначены на текущего пользователя
    tasks = Tasks.objects.filter(assigned_to=user)
    # Формируем словарь контекста
    context = {
        "tasks": tasks
    }
    # возвращаем ответ с отрисованным шаблоном и переданным контекстом
    return render(request, 'user_page.html', context)

# делегировать задачу
# проверяем авторизованы ли мы с помощью декоратора login_required, если да, то заходим на страницу изменения задачи
@login_required
def UpdateView(request, task_id):
    # получает объект задания (Tasks) по его уникальному идентификатору (task_id), или возвращает страницу 404 ошибки, если такой объект не существует.
    task = get_object_or_404(Tasks, pk=task_id)
    # проверяет, был ли отправлен POST-запрос пользователем.
    if request.method == 'POST':
        # создает экземпляр формы UpdateTaskForm с данными, которые были отправлены пользователем, и существующим экземпляром задания в качестве instance.
        form = UpdateTaskForm(request.POST, instance=task)
        if form.is_valid(): # проверяет не было ли пропущено обязательное поле
            task = form.save() # сохраняем форму
            messages.success(request, 'Передача задания прошла успешно!') # добавляет сообщение об успешном обновлении задания в очередь сообщений пользователя.
            # return redirect('task_edit', task_id=task.id) # перенаправляет пользователя на страницу редактирования задания с обновленными данными.
            return redirect('user_page')
    else:
        form = UpdateTaskForm(instance=task)

    context = {
        'form': form,
        'task': task,
    }
    return render(request, 'task_edit.html', context) # возвращает шаблон task_edit.html с переданным контекстом данных.

# создание задачи
# проверяем авторизованы ли мы с помощью декоратора login_required, если да, то заходим на страницу создания задачи
@login_required
def create_task(request):
    if request.method == 'POST': #если метод запроса POST, значит форма отправлена, нужно сохранить задачу в БД
        form = TaskFormAdd(request.POST, request.FILES) # создаем экземпляр формы и заполняем ее данными из запроса
        if form.is_valid(): # проверяем, валидна ли форма
            task = form.save(commit=False) # создаем объект задачи, но не сохраняем его в БД, используя commit=False
            task.save()  # сохраняем объект `Tasks` в базе данных

            assigned_users = form.cleaned_data.get('assigned_to')
            task.created_by.set([request.user])  # добавляем создателя задачи

            # добавляем назначенных пользователей
            for assigned_user in assigned_users:
                task.assigned_to.add(assigned_user)
             # выводим сообщение об успешном создании задачи
            messages.success(request, 'Задача успешно создана!')
            return redirect('home')
    else:
        form = TaskFormAdd()

    return render(request, 'task_add.html', {'form': form})

# страница контактов, где мы можем отправить сообщению админу и получить ответ
def contact(request):
    if request.method == 'POST':# Проверяем, если запрос был отправлен методом POST
        contact = Contact(contact_info=request.user)  # Создаем экземпляр модели Contact, связанный с текущим пользователем
        name = request.POST.get('name') # Получаем данные из формы
        email = request.POST.get('email')# Получаем данные из формы
        phone = request.POST.get('phone')# Получаем данные из формы
        subject = request.POST.get('subject')# Получаем данные из формы
        contact_info = request.user # Получаем данные из формы
        contact.name = name # Получаем данные из формы
        contact.email = email # Получаем данные из формы
        contact.phone = phone # Получаем данные из формы
        contact.subject = subject # Получаем данные из формы

        contact.save()# Сохраняем объект `Contact` в базе данных
        messages.success(
            request, f"Добавилась новая заявка! от пользователя  {contact_info}") # Отображаем сообщение об успешном сохранении заявки

        contact.save()  # Снова сохраняем объект `Contact` в базе данных
        return redirect('/', f"Добавилась новая заявка! ")  # Перенаправляем пользователя на домашнюю страницу
    return render(request, 'contact.html') # Если запрос был отправлен методом GET, отображаем страницу с формой

# проверяем авторизованы ли мы с помощью декоратора login_required, если да, то заходим на страницу заявок
@login_required
def contacts(request):

    zayzvka = Contact.objects.all().filter(contact_info=request.user) #выбираем все объекты модели Contact, которые относятся к текущему пользователю.

    return render(request,
                  'user_contacts.html',
                  {'zayzvka': zayzvka})

# удаление заявки
def delete_contact(request, pk=None): #Функция delete_contact принимает запрос request и необязательный параметр pk, который является идентификатором объекта Contact, который нужно удалить
    Contact.objects.get(id=pk).delete() #получает объект Contact с помощью метода get() и передает значение параметра pk. Затем вызывается метод delete() для удаления объекта из базы данных.
    messages.success(
        request, f'Заявка успешно удалена пользователем!{request.user.username} !')
    return redirect('contacts')

# Отображение детальной информации по задаче с id = task_id
def card_detail(request, task_id):

    task = get_object_or_404(Tasks, pk=task_id) # Получаем объект задачи из базы данных по её id
    # Отображаем страницу с подробной информацией о задаче
    return render(request, "card_detail.html", {"task": task})

def hide_task(request, pk=None):
    # Получаем объект задачи или возвращаем 404 ошибку, если такой задачи нет
    task = get_object_or_404(Tasks, id=pk)

    # Если метод запроса POST, то изменяем статус скрытости задачи и сохраняем ее
    if request.method == 'POST':
        task.is_hidden = not task.is_hidden
        task.save()
        # Перенаправляем пользователя на главную страницу
        # return redirect('home')
        return redirect('user_page')


# Required packages
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

import scipy.stats as stats
import math

# Change directory
import os
#%%
file_path = "C://Users//vojha//Dropbox//0_Programming//ESUM_Experiments//Participant_Data//gps_sensor_data//5s//smoot_signals//complied_data_filtered_with_ID.csv"
# Read in the data: Participants Biofeedback
#xl = pd.ExcelFile("Participants_Bio_feedback_data_dynamics.xlsx")
#xl.sheet_names for calling the name of the sheet. "Sheet1" is it in this excel sheet.
#bio = xl.parse("Sheet1")
file_path_demograph = "C://Users//vojha//Dropbox//0_Programming//ESUM_Experiments//Participant_Data//demography"

data = pd.read_csv(file_path)
#del data["Unnamed: 0"]#deleting unneccasry column
#histogram
#data = data["Area"].tolist()
#fig = plt.figure(figsize=(3.5, 3.8))
#hist, bins = np.histogram(data)
#width = 0.9 * (bins[1] - bins[0])
#center = (bins[:-1] + bins[1:]) / 2
#plt.bar(center, hist, align='center', color = "royalblue", width=width)
##plt.title("Light value distribution")
##plt.xlabel("Sound")
##plt.xlabel("Dust")
##plt.xlabel("Temperature")
##plt.xlabel("Relative Humidity")
#plt.xlabel("Area")
##plt.xlabel("Area")
##plt.ylabel("Frequency")
#fig.savefig('hist_area.pdf')
#fig.savefig('hist_area.jpeg')
#plt.show()

#result = data.sort(["participant"], ascending=[1,1])
#grouped_sum = data.groupby(['ID']).sum()
grouped_mean = data.groupby(['ID']).mean()
x_y_phase_peak = grouped_mean[['X','Y','Sound','Dust','TempEN','RH','Light','EDA_phase','EDA_peaks']]
x_y_phase_peak.to_csv("per_xy_pahse_mean_all_var.csv")
#normalized_grouped_mean =(x_y_phase_peak-x_y_phase_peak.min())/(x_y_phase_peak.max()-x_y_phase_peak.min())
cols_to_norm = ['Sound','Dust','TempEN','RH','Light','EDA_phase','EDA_peaks']
x_y_phase_peak[cols_to_norm] = x_y_phase_peak[cols_to_norm].apply(lambda x: (x - x.min()) / (x.max() - x.min()))
normalized_grouped_mean = x_y_phase_peak
normalized_grouped_mean.to_csv("per_xy_pahse_mean_norm_all_var.csv")

#slicing the data frame participants wise
#[6,7,9,10,13,24,27,28,34,35]#native
#[8,11,16,23,25,26,29,30,31,32]#non-native
#[6,8,9,10,11,16,23,25,27,28,29,30,31]#20-30
#[7,13,23,24,26,32,34,35]#30 and above

#%%
cols_to_norm = ['EDA_phase','EDA_peaks']
data[cols_to_norm] = data[cols_to_norm].apply(lambda x: (x - x.min()) / (x.max() - x.min()))

#%%all participants
grouped_mean = data.groupby(['X', 'Y']).mean()
grouped_mean = grouped_mean[['EDA_phase','EDA_peaks']]
#grouped_mean.to_csv("per_xy_pahse_norm_mean.csv")

#%%
native = data[(data.participant == 6) |
        (data.participant == 7)|
        (data.participant == 9)|
        (data.participant == 10)|
        (data.participant == 13)|
        (data.participant == 24)|
        (data.participant == 27)|
        (data.participant == 28)|
        (data.participant == 34)|
        (data.participant == 35)]

native_grouped_mean = native.groupby(['ID']).mean()
#native_grouped_mean = native_grouped_mean[['EDA_phase','EDA_peaks']]
native_grouped_mean = native_grouped_mean[['X','Y','EDA_phase','EDA_peaks']]

non_native = data[(data.participant == 8)|
        (data.participant == 11)|
        (data.participant == 16)|
        (data.participant == 23)|
        (data.participant == 25)|
        (data.participant == 26)|
        (data.participant == 29)|
        (data.participant == 30)|
        (data.participant == 31)|
        (data.participant == 32)]

non_native_grouped_mean = non_native.groupby(['ID']).mean()
#non_native_grouped_mean = non_native_grouped_mean[['EDA_phase','EDA_peaks']]
non_native_grouped_mean = non_native_grouped_mean[['X','Y','EDA_phase','EDA_peaks']]

min_a = native_grouped_mean[['EDA_phase','EDA_peaks']].min(axis=0).tolist()
min_b = non_native_grouped_mean[['EDA_phase','EDA_peaks']].min(axis=0).tolist()

max_a = native_grouped_mean[['EDA_phase','EDA_peaks']].max(axis=0).tolist()
max_b = non_native_grouped_mean[['EDA_phase','EDA_peaks']].max(axis=0).tolist()

min_phase = min([min_a[0],min_b[0]])
min_peak = min([min_a[1],min_b[1]])
max_phase = max([max_a[0],max_b[0]])
max_peak = max([max_a[1],max_b[1]])

cols_to_norm = ['EDA_phase']
native_grouped_mean["EDA_phase_norm"] = native_grouped_mean[cols_to_norm].apply(lambda x: (x - min_phase) / (max_phase - min_phase))
non_native_grouped_mean["EDA_phase_norm"] = non_native_grouped_mean[cols_to_norm].apply(lambda x: (x - min_phase) / (max_phase - min_phase))

cols_to_norm = ['EDA_peaks']
native_grouped_mean["EDA_peaks_norm"] = native_grouped_mean[cols_to_norm].apply(lambda x: (x - min_peak) / (max_peak - min_peak))
non_native_grouped_mean["EDA_peaks_norm"] = non_native_grouped_mean[cols_to_norm].apply(lambda x: (x - min_peak) / (max_peak - min_peak))

native_grouped_mean.to_csv("per_xy_pahse_norm_mean_native.csv")
non_native_grouped_mean.to_csv("per_xy_pahse_norm_mean_non_native.csv")

#%%age
age_20_30 = data[(data.participant == 6)|
        (data.participant == 8)|
        (data.participant == 9)|
        (data.participant == 10)|
        (data.participant == 11)|
        (data.participant == 16)|
        (data.participant == 23)|
        (data.participant == 25)|
        (data.participant == 27)|
        (data.participant == 28)|
        (data.participant == 29)|
        (data.participant == 30)|
        (data.participant == 31)]

age_20_30_grouped_mean = age_20_30.groupby(['ID']).mean()
#age_20_30_grouped_mean = age_20_30_grouped_mean[['EDA_phase','EDA_peaks']]
age_20_30_grouped_mean = age_20_30_grouped_mean[['X','Y','EDA_phase','EDA_peaks']]

age_30_above = data[(data.participant == 7)|
        (data.participant == 13)|
        (data.participant == 23)|
        (data.participant == 24)|
        (data.participant == 26)|
        (data.participant == 32)|
        (data.participant == 34)|
        (data.participant == 35)]

age_30_above_grouped_mean = age_30_above.groupby(['ID']).mean()
#age_30_above_grouped_mean = age_30_above_grouped_mean[['EDA_phase','EDA_peaks']]
age_30_above_grouped_mean = age_30_above_grouped_mean[['X','Y','EDA_phase','EDA_peaks']]

min_a = age_20_30_grouped_mean[['EDA_phase','EDA_peaks']].min(axis=0).tolist()
min_b = age_30_above_grouped_mean[['EDA_phase','EDA_peaks']].min(axis=0).tolist()

max_a = age_20_30_grouped_mean[['EDA_phase','EDA_peaks']].max(axis=0).tolist()
max_b = age_30_above_grouped_mean[['EDA_phase','EDA_peaks']].max(axis=0).tolist()

min_phase = min([min_a[0],min_b[0]])
min_peak = min([min_a[1],min_b[1]])
max_phase = max([max_a[0],max_b[0]])
max_peak = max([max_a[1],max_b[1]])

cols_to_norm = ['EDA_phase']
age_20_30_grouped_mean["EDA_phase_norm"] = age_20_30_grouped_mean[cols_to_norm].apply(lambda x: (x - min_phase) / (max_phase - min_phase))
age_30_above_grouped_mean["EDA_phase_norm"] = age_30_above_grouped_mean[cols_to_norm].apply(lambda x: (x - min_phase) / (max_phase - min_phase))

cols_to_norm = ['EDA_peaks']
age_20_30_grouped_mean["EDA_peaks_norm"] = age_20_30_grouped_mean[cols_to_norm].apply(lambda x: (x - min_peak) / (max_peak - min_peak))
age_30_above_grouped_mean["EDA_peaks_norm"] = age_30_above_grouped_mean[cols_to_norm].apply(lambda x: (x - min_peak) / (max_peak - min_peak))

age_20_30_grouped_mean.to_csv("per_xy_pahse_norm_mean_age_20_30.csv")
age_30_above_grouped_mean.to_csv("per_xy_pahse_norm_mean_age_30_above.csv")

#%%place
village_town = data[(data.participant == 6)|
        (data.participant == 7)|
        (data.participant == 24)|
        (data.participant == 27)|
        (data.participant == 28)|
        (data.participant == 32)|
        (data.participant == 34)|
        (data.participant == 35)]

village_town_mean = village_town.groupby(['ID']).mean()
#village_town_mean = village_town_mean[['EDA_phase','EDA_peaks']]
village_town_mean = village_town_mean[['X','Y','EDA_phase','EDA_peaks']]

city = data[(data.participant == 9)|
        (data.participant == 10)|
        (data.participant == 11)|
        (data.participant == 13)|
        (data.participant == 26)|
        (data.participant == 30)|
        (data.participant == 29)|
        (data.participant == 31)]

city_mean = city.groupby(['ID']).mean()
#city_mean = city_mean[['EDA_phase','EDA_peaks']]
city_mean = city_mean[['X','Y','EDA_phase','EDA_peaks']]

metropolis = data[(data.participant == 8)|
        (data.participant == 16)|
        (data.participant == 23)|
        (data.participant == 25)]

metropolis_mean = metropolis.groupby(['ID']).mean()
#metropolis_mean = metropolis_mean[['EDA_phase','EDA_peaks']]
metropolis_mean = metropolis_mean[['X','Y','EDA_phase','EDA_peaks']]

min_a = village_town_mean[['EDA_phase','EDA_peaks']].min(axis=0).tolist()
min_b = city_mean[['EDA_phase','EDA_peaks']].min(axis=0).tolist()
min_c = metropolis_mean[['EDA_phase','EDA_peaks']].min(axis=0).tolist()

max_a = village_town_mean[['EDA_phase','EDA_peaks']].max(axis=0).tolist()
max_b = city_mean[['EDA_phase','EDA_peaks']].max(axis=0).tolist()
max_c = metropolis_mean[['EDA_phase','EDA_peaks']].max(axis=0).tolist()

min_phase = min([min_a[0],min_b[0],min_c[0]])
min_peak = min([min_a[1],min_b[1],min_c[1]])
max_phase = max([max_a[0],max_b[0],max_c[0]])
max_peak = max([max_a[1],max_b[1],max_c[1]])

cols_to_norm = ['EDA_phase']
village_town_mean["EDA_phase_norm"] = village_town_mean[cols_to_norm].apply(lambda x: (x - min_phase) / (max_phase - min_phase))
city_mean["EDA_phase_norm"] = city_mean[cols_to_norm].apply(lambda x: (x - min_phase) / (max_phase - min_phase))
metropolis_mean["EDA_phase_norm"] = metropolis_mean[cols_to_norm].apply(lambda x: (x - min_phase) / (max_phase - min_phase))

cols_to_norm = ['EDA_peaks']
village_town_mean["EDA_peaks_norm"] = village_town_mean[cols_to_norm].apply(lambda x: (x - min_peak) / (max_peak - min_peak))
city_mean["EDA_peaks_norm"] = city_mean[cols_to_norm].apply(lambda x: (x - min_peak) / (max_peak - min_peak))
metropolis_mean["EDA_peaks_norm"] = metropolis_mean[cols_to_norm].apply(lambda x: (x - min_peak) / (max_peak - min_peak))

village_town_mean.to_csv("per_xy_pahse_norm_mean_village.csv")
city_mean.to_csv("per_xy_pahse_norm_mean_city.csv")
metropolis_mean.to_csv("per_xy_pahse_norm_mean_metro.csv")

#%% Stat
list_stat = []
A = native_grouped_mean['EDA_phase'].tolist()
B = non_native_grouped_mean['EDA_phase'].tolist()
stats.ttest_ind(a= A,b= B, equal_var=False)    # Assume samples have equal variance?
#stats.ks_2samp(A, B)

A = age_20_30_grouped_mean['EDA_phase'].tolist()
B = age_30_above_grouped_mean['EDA_phase'].tolist()
stats.ttest_ind(a= A,b= B, equal_var=False)    # Assume samples have equal variance?
stats.ks_2samp(A, B)

A = village_town_mean['EDA_phase'].tolist()
B = city_mean['EDA_phase'].tolist()
stats.ttest_ind(a= A,b= B, equal_var=False)    # Assume samples have equal variance?
#stats.ks_2samp(A, B)

A = village_town_mean['EDA_phase'].tolist()
B = metropolis_mean['EDA_phase'].tolist()
stats.ttest_ind(a= A,b= B, equal_var=False)    # Assume samples have equal variance?
#stats.ks_2samp(A, B)

A = city_mean['EDA_phase'].tolist()
B = metropolis_mean['EDA_phase'].tolist()
stats.ttest_ind(a= A,b= B, equal_var=False)    # Assume samples have equal variance?
#stats.ks_2samp(A, B)
#%%mean

list_sums = []
colum_to_sum = "EDA_peaks_norm"
list_sums.append(native_grouped_mean[colum_to_sum].mean())
list_sums.append(non_native_grouped_mean[colum_to_sum].mean())
list_sums.append(age_20_30_grouped_mean[colum_to_sum].mean())
list_sums.append(age_30_above_grouped_mean[colum_to_sum].mean())
list_sums.append(village_town_mean[colum_to_sum].mean())
list_sums.append(city_mean[colum_to_sum].mean())
list_sums.append(metropolis_mean[colum_to_sum].mean())
list_sums
#%%Combination point to point
result_ethenic =  pd.concat([native_grouped_mean, non_native_grouped_mean], axis=1, join_axes=[native_grouped_mean.index])
result_ethenic.columns = ["native_phase","native_peak","non_native_phase","non_native_peak"]

result_age =  pd.concat([age_20_30_grouped_mean, age_30_above_grouped_mean], axis=1, join_axes=[age_20_30_grouped_mean.index])
result_age.columns = ["age_tt_phase","age_tt_peak","age_ta_phase","age_ta_peak"]

result_place =  pd.concat([village_town_mean,city_mean, metropolis_mean], axis=1, join_axes=[village_town_mean.index])
result_place.columns = ["vilage_phase","vilage_peak","city_phase","city_peak","metro_phase","metro_peak"]

#%%line plot

plt.style.use("default")
fig, ax = plt.subplots(nrows=1, ncols=1, figsize=[5,2])
plt.plot(grouped_mean['EDA_phase'].tolist())
#plt.plot(result_ethenic['native_phase'].tolist(), label = 'native (0.13)', color = 'seagreen', linewidth=1)
#plt.plot(result_ethenic['non_native_phase'].tolist(), label = 'non-native (0.15)', color = 'red',linewidth=1)
#plt.plot(result_age['age_tt_phase'].tolist(), label = 'age 20-29 (0.12)', color = 'dimgray',linewidth=1)
#plt.plot(result_age['age_ta_phase'].tolist(), label = 'age 30+ (0.18)', color = 'orange',linewidth=1)
#plt.plot(result_place['vilage_phase'].tolist(), label = 'village (0.13)', color = 'olivedrab',linewidth=1)
#plt.plot(result_place['city_phase'].tolist(), label = 'city (0.17)', color = 'orange',linewidth=1)
#plt.plot(result_place['metro_phase'].tolist(), label = 'metropolis (0.11)', color = 'royalblue',linewidth=1)

#ax.set_ylim([0,1])
plt.xticks([])
plt.ylabel('Normalized mean')
#plt.xlabel('Locations')
#plt.title('All variable normalized mean signal analysis')
plt.legend()
fig.savefig('demograpgy_all.pdf')
fig.savefig('demograpgy_all.jpeg')
plt.show()
plt.close(fig)

#%%bar plot data

list1 = result_ethenic["native_peak"].tolist()
list2 = result_ethenic["non_native_peak"].tolist()

list1 = result_age["age_tt_phase"].tolist()
list2 = result_age["age_ta_phase"].tolist()

#%% create bar plot
length = len(list1)

fig, ax = plt.subplots()
index = np.arange(length)
bar_width = 1
opacity = 0.8

rects1 = plt.bar(index,list1 , bar_width,
                 alpha=opacity,
                 color='b',
                 label='age 20-29')

rects2 = plt.bar(index , list2, bar_width,
                 alpha=opacity,
                 color='g',
                 label='age 30+')

#plt.xlabel('Person')
#plt.ylabel('Scores')
#plt.title('Scores by person')
#plt.xticks(index + bar_width, ('A', 'B', 'C', 'D'))
plt.xticks([])
plt.legend()

plt.tight_layout()
plt.show()


#inheritance
class Phone:#base class/parent class
    def __init__(self,model_name,brand,price):
        self.model_name=model_name
        self.brand=brand
        self._price=max(price,0)
    def full_name(self):
        return f"{self.model_name} {self.brand}"
    def calling_no(self,phone_no):
        return f"calling {phone_no}"

class Smartphone(Phone):#derived class/child class
    def __init__(self,model_name,brand,price,ram,internal_memory,rear_camera):
        Phone.__init__(self,model_name,brand,price)
        # super().__init__(model_name,brand,price) #same as above method to call parent class
        self.ram=ram
        self.internal_memory=internal_memory
        self.rear_camera=rear_camera

p1=Phone("nokia","567f",980)
p2=Smartphone("1+","17vggfhmzz",34000,3,4,"3ds")
print(p1.full_name())
print(p2.full_name())
print(p2.__dict__)
print(p2._price)


"""Question-answering with sources over a vector database."""

import warnings
from typing import Any, Dict, List

from langchain_core.documents import Document
from langchain_core.pydantic_v1 import Field, root_validator
from langchain_core.vectorstores import VectorStore

from langchain.callbacks.manager import (
    AsyncCallbackManagerForChainRun,
    CallbackManagerForChainRun,
)
from langchain.chains.combine_documents.stuff import StuffDocumentsChain
from langchain.chains.qa_with_sources.base import BaseQAWithSourcesChain

class VectorDBQAWithSourcesChain(BaseQAWithSourcesChain):
    """Question-answering with sources over a vector database."""

    vectorstore: VectorStore = Field(exclude=True)
    """Vector Database to connect to."""
    k: int = 4
    """Number of results to return from store"""
    reduce_k_below_max_tokens: bool = False
    """Reduce the number of results to return from store based on tokens limit"""
    max_tokens_limit: int = 3375
    """Restrict the docs to return from store based on tokens,
    enforced only for StuffDocumentChain and if reduce_k_below_max_tokens is to true"""
    search_kwargs: Dict[str, Any] = Field(default_factory=dict)
    """Extra search args."""

    def _reduce_tokens_below_limit(self, docs: List[Document]) -> List[Document]:
        num_docs = len(docs)

        if self.reduce_k_below_max_tokens and isinstance(
            self.combine_documents_chain, StuffDocumentsChain
        ):
            tokens = [
                self.combine_documents_chain.llm_chain._get_num_tokens(doc.page_content)
                for doc in docs
            ]
            token_count = sum(tokens[:num_docs])
            while token_count > self.max_tokens_limit:
                num_docs -= 1
                token_count -= tokens[num_docs]

        return docs[:num_docs]

    def _get_docs(
        self, inputs: Dict[str, Any], *, run_manager: CallbackManagerForChainRun
    ) -> List[Document]:
        question = inputs[self.question_key]
        docs = self.vectorstore.similarity_search(
            question, k=self.k, **self.search_kwargs
        )
        return self._reduce_tokens_below_limit(docs)

    async def _aget_docs(
        self, inputs: Dict[str, Any], *, run_manager: AsyncCallbackManagerForChainRun
    ) -> List[Document]:
        raise NotImplementedError("VectorDBQAWithSourcesChain does not support async")

    @root_validator()
    def raise_deprecation(cls, values: Dict) -> Dict:
        warnings.warn(
            "`VectorDBQAWithSourcesChain` is deprecated - "
            "please use `from langchain.chains import RetrievalQAWithSourcesChain`"
        )
        return values

    @property
    def _chain_type(self) -> str:
        return "vector_db_qa_with_sources_chain"


# -*- coding: utf-8 -*-
"""
Created on Mon Oct 09 15:00:29 2017

@author: dyin

Leetcode Problem 2: Add two numbers

You are given two non-empty linked lists representing two non-negative integers. The digits are stored in reverse order and each of their nodes contain a single digit. Add the two numbers and return it as a linked list.

You may assume the two numbers do not contain any leading zero, except the number 0 itself.

Input: (2 -> 4 -> 3) + (5 -> 6 -> 4)
Output: 7 -> 0 -> 8

"""

class ListNode(object):
    def __init__(self, x):
        self.val=x
        self.next=None

class Solution(object):
    def addTwoNumbers(self, l1, l2):
        """
        :type l1: ListNode
        :type l2: ListNode
        :rtype: ListNode
        """
        dummy=ListNode(0)
        carry, current=0, dummy
        while l1 or l2:
            val=carry
            if l1:
                val+=l1.val
                l1=l1.next
            if l2:
                val+=l2.val
                l2=l2.next
            carry=val / 10
            val=val % 10
            current.next=ListNode(val)
            current=current.next
        # Check if the sum have an extra carry of one at the end
        if carry == 1:
            current.next=ListNode(1)
        return dummy.next

class Solution(object):
    def addTwoNumbers(self, l1, l2):
        """
        :type l1: ListNode
        :type l2: ListNode
        :rtype: ListNode
        """
        # first take two numbers out
        num1,num2=0,0
        i,j=0,0
        while l1:
            num1+=l1.val*10**i
            l1=l1.next
            i+=1
        while l2:
            num2+=l2.val*10**j
            l2=l2.next
            j+=1
        string=reversed(str(num1+num2))
        dummy=res=ListNode(0)
        print(res)
        for s in string:
            res.next=ListNode(int(s))
            res=res.next
        return dummy.next


import numpy as np
import time

def make_bg(phase):
	# Triangle size
	v_size = 46
	h_size = 80

	# Gradiente parameters
	max_grad = 10
	sd_op = 0.1

	# Wave parameters
	length = 16
	phase = phase +0 % length
	freq = 8
	amp = 0.5
	height = 0.3
	wave = amp*(np.sin(phase+np.arange(0,16)*np.pi/freq)+1)-height
	wave_op = wave*0.1
	wave_cl = length*wave*0.3

	color_map_default = np.array(['216171','367A99','41A3CD','67C2D6','FFFFFF'])
	#np.array(['0F128F','0F1F81','0F1281','0F1ADC','364ADC','3647F5','93C7F5','93CCF9','D0ECF9','D0E1FE','E0F1FE','E0FFFF','FFFFFF'])
	color_map = color_map_default#[np.array([0,2,1,4,3,6,5,8,7,10,9,12,11])]
	opacity_range = np.arange(1,0,-0.1)

	def format_line(p1,p2,p3,color,opacity):
		p1_str = "M"+str(int(p1[0]))+","+str(int(p1[1]))+" "
		p2_str = "L"+str(int(p2[0]))+","+str(int(p2[1]))+" "
		p3_str = "L"+str(int(p3[0]))+","+str(int(p3[1]))+" "

		return("<path d='"+p1_str+p2_str+p3_str+"Z"+"' style='fill:#"+color+";stroke-width:0;opacity:"+str(opacity)+"'/>")

	def line_type_one(v_plus,color,opacity):
		script = ""
		for i in np.arange(16):
			h_plus = h_size*i
			p1 = [h_plus,v_plus]
			p2 = [h_plus,v_plus+v_size]
			p3 = [h_plus+h_size/2,v_plus+v_size/2]
			op = np.clip(opacity-np.random.rand()*sd_op-wave_op[i],0,1)
			cl = color_map[np.clip(color+int(-1*np.random.rand()*2-wave_cl[i]),0,color_map.size-1)]
			script = script + format_line(p1,p2,p3,cl,op)
			p1 = [h_plus+h_size/2,v_plus+v_size/2]
			p2 = [h_plus+h_size,v_plus+v_size]
			p3 = [h_plus+h_size,v_plus]
			op = np.clip(opacity-np.random.rand()*sd_op-wave_op[i],0,1)
			cl = color_map[np.clip(color+int(-1*np.random.rand()*2-wave_cl[i]),0,color_map.size-1)]
			script = script + format_line(p1,p2,p3,cl,op)
		return(script)

	def line_type_two(v_plus,color,opacity):
		script = ""
		for i in np.arange(16):
			h_plus = h_size*i - h_size/2
			p1 = [h_plus+h_size/2,v_plus+v_size/2]
			p2 = [h_plus+h_size,v_plus+v_size]
			p3 = [h_plus+h_size,v_plus]
			op = np.clip(opacity-np.random.rand()*sd_op-wave_op[i],0,1)
			cl = color_map[np.clip(color+int(-1*np.random.rand()*2-wave_cl[i]),0,color_map.size-1)]
			script = script + format_line(p1,p2,p3,cl,op)
			h_plus = h_size*i + h_size/2
			p1 = [h_plus,v_plus]
			p2 = [h_plus,v_plus+v_size]
			p3 = [h_plus+h_size/2,v_plus+v_size/2]
			op = np.clip(opacity-np.random.rand()*sd_op-wave_op[i],0,1)
			cl = color_map[np.clip(color+int(-1*np.random.rand()*2-wave_cl[i]),0,color_map.size-1)]
			script = script + format_line(p1,p2,p3,cl,op)
		return(script)

	script = ""
	for i in np.arange(0,max_grad):
		pos_y = (i-1)*v_size
		script = script + line_type_two(pos_y - v_size/2,i,1)#(max_grad-i)/max_grad)
		script = script + line_type_one(pos_y,i,1)#(max_grad-i)/max_grad)

	header = "<!DOCTYPE html><svg width='1280' height='480'>"

	bottom = "</svg>"

	file = open("bg.html","w")
	file.write(header+script+bottom)
	file.close()

for i in np.arange(1):
	#time.sleep(1)
	make_bg(i)
	i = i+1


def absValue(x):
    if x < 0:
        res= x*-1
        return res
    else:
        return x

def power(x,p):
    res= x
    if p == 0:
        return 1
    for d in range(1,p):
        res = res * x
    return res

def isPrime(x):
    if x < 2:
        return False
    for d in range(2, x//2 +1):
        if x % d == 0:
            return False
    return True

def slowFib(x):
    if x == 0:
        return 0
    elif x == 1:
        return 1
    else :
        res= slowFib(x-1)+slowFib(x-2)
        return res

def quickFib(x):
    d= [-1 for _ in range(x+1)]
    res= quickFib2(x,d)
    return res

def quickFib2(x,d):
    if x == 0:
        d.insert(x,0)
        return d[0]
    elif x == 1:
        d.insert(x,1)
        return d[1]
    else :
        if(d[x-1] != -1):
            res1=d[x-1]
        else:
            res1= quickFib2(x-1,d)
        if(d[x-2] != -1):
            res2=d[x-2]
        else:
            res2= quickFib2(x-2,d)

        d.insert(x,res1+res2)
        return d[x]


import itertools

n, d = map(int, input().split())

arr = list(map(int, input().rstrip().split()))

counter = 0

for i in range(0, n - 2):
    subsequence = list(itertools.takewhile(lambda x: x - arr[i] <= 2 * d, arr[i:]))

    if len(subsequence) > 2:
        # set, otherwise: 3x(7 10 13) in 1 6 7 7 8 10 12 13 14 19 sequence
        for ai, aj, ak in set(itertools.combinations(subsequence, 3)):
            if aj - ai == d and ak - aj == d:
                counter += 1

print(counter)


#Prob4 : https://iitmcvg.github.io/problem_statements/Problem_statements/

import numpy as np
from cv2 import *
kernel = np.ones((5,5))
cap = cv2.VideoCapture('track.mp4')
target = cv2.imread('target.png')
#equ = cv2.equalizeHist(target)
hsvt = cv2.cvtColor(target , cv2.COLOR_BGR2HSV)
hist = cv2.calcHist([hsvt],[0,1],None,[180,256],[0,180,0,256])
cv2.normalize(hist,hist,0,255,cv2.NORM_MINMAX)

while(cap.isOpened()):
    ret, frame = cap.read()
    hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)

    res = cv2.calcBackProject([hsv],[0,1],hist,[0,180,0,256],1)
    disc = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5,5))
    res = cv2.filter2D(res, -1, disc)

    ret,thresh = cv2.threshold(res,100,255,0)
    #erode=cv2.erode(thresh,kernel)
    #dilate=cv2.dilate(erode,kernel)
    #thresh = dilate
    final = cv2.merge((thresh,thresh,thresh))

    out = cv2.bitwise_and(frame,final)

    cv2.imshow('frame',out)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

cap.release()
cv2.destroyAllWindows()


import pandas as pd
from scraper import scrapper
from summarizer import summarizer
from IPython.display import display

def launch_excel_storage_bot():
    topics = {'WORLD':4, 'NATION':5, 'BUSINESS':2, 'ENTERTAINMENT':2, 'SPORTS':3, 'SCIENCE':2, 'HEALTH':2}

    news = pd.DataFrame(columns = ['Title', 'Content', 'Summary', 'Keywords'])

    for topic in topics.keys():
        headlines, content = scrapper(topic, topics[topic])
        if len(headlines) == 0:
            continue
        for n in range(topics[topic]):
            summary, keywords = summarizer(content[n])
            news.loc[len(news.index)] = [headlines[n], content[n], summary, keywords]

    display(news)
    news.to_csv('daily_news.csv')

launch_excel_storage_bot()


from typing import List, Tuple, Optional

from collections import defaultdict,deque
from functools import cache
import heapq
from heapq import heappop,heappush
from sortedcontainers import SortedDict,SortedList

class Solution:
    def longestValidSubstring(self, word: str, forbidden: List[str]) -> int:
        mx = 0
        n = len(word)
        dic ={}
        for f in forbidden:
            dic[f] = 1
        left=0
        def verify(i):
            ls = word[left:i+1][-10:]
            m = len(ls)
            for i in range(m):
                if ls[i:] in dic:
                    return False
            return True
        for i,a in enumerate(word):
            while not verify(i):
                left +=1
            mx = max(mx,i+1-left)
        return mx

re =Solution().longestValidSubstring(word = "cbaaaabc", forbidden = ["aaa","cb"])
print(re)


import os
import azure.cognitiveservices.speech as speechsdk
import json

os.environ['SPEECH_KEY'] = "f8d0a97015eb44c89965625a6ba923c4"
os.environ['SPEECH_REGION'] = "westus"

def pron_check_and_read_from_file(audio_path='test.wav'):
    speech_config = speechsdk.SpeechConfig(subscription=os.environ.get('SPEECH_KEY'), region=os.environ.get('SPEECH_REGION'))
    audio_config = speechsdk.AudioConfig(filename=audio_path)
    speech_recognizer = speechsdk.SpeechRecognizer(speech_config=speech_config, audio_config=audio_config)
    result = speech_recognizer.recognize_once_async().get()
    if result.reason == speechsdk.ResultReason.RecognizedSpeech:
        print("Recognized: {}".format(result.text))
    elif result.reason == speechsdk.ResultReason.NoMatch:
        print("No speech could be recognized: {}".format(result.no_match_details))
    elif result.reason == speechsdk.ResultReason.Canceled:
        cancellation_details = result.cancellation_details
        print("Speech Recognition canceled: {}".format(cancellation_details.reason))
        if cancellation_details.reason == speechsdk.CancellationReason.Error:
            print("Error details: {}".format(cancellation_details.error_details))
            print("Did you set the speech resource key and region values?")
    ref_text = result.text # Speech to text
    speech_recognizer2 =  speechsdk.SpeechRecognizer(speech_config=speech_config, audio_config=audio_config)
    pronunciation_assessment_config = speechsdk.PronunciationAssessmentConfig(json_string="{\"referenceText\":\"%s\",\"gradingSystem\":\"HundredMark\",\"granularity\":\"Phoneme\"}" %ref_text)
    pronunciation_assessment_config.apply_to(speech_recognizer2)
    result2 = speech_recognizer2.recognize_once()
    pronunciation_assessment_result_json = result2.properties.get(speechsdk.PropertyId.SpeechServiceResponse_JsonResult)
    json_results_pron = json.loads(pronunciation_assessment_result_json) #json version of Pronunciation check results.
    # print(json.dumps(json_results_pron, indent =4))

    return ref_text, json_results_pron

def pron_check_from_microphone():
        pronunciation_assessment_config = speechsdk.PronunciationAssessmentConfig(json_string="{\"ReferenceText\":\"Hello my name is George\",\"gradingSystem\":\"HundredMark\",\"granularity\":\"Phoneme\"}")
        speech_config = speechsdk.SpeechConfig(subscription=os.environ.get('SPEECH_KEY'), region=os.environ.get('SPEECH_REGION'))
        speech_config.speech_recognition_language="en-US"
        audio_config = speechsdk.audio.AudioConfig(use_default_microphone=True)
        speech_recognizer = speechsdk.SpeechRecognizer(speech_config=speech_config, audio_config=audio_config)

        #pronunciation_assessment_config.apply_to(speech_recognizer)
        print("speak into the microphone:")
        speech_recognition_result = speech_recognizer.recognize_once_async().get()
        print(speech_recognition_result.text)
        # The pronunciation assessment result as a JSON string
        pronunciation_assessment_result_json = speech_recognition_result.properties.get(speechsdk.PropertyId.SpeechServiceResponse_JsonResult)
        print(pronunciation_assessment_result_json)

if __name__ == '__main__':
    pron_check_and_read_from_file()


from typing import TYPE_CHECKING, Optional, Tuple, Union

from .base import MATCH_WINDOW_OPTION, ArgsType, Boss, PayloadGetType, PayloadType, RCOptions, RemoteCommand, ResponseType, Window

if TYPE_CHECKING:
    from kitty.cli_stub import ScrollWindowRCOptions as CLIOptions

class ScrollWindow(RemoteCommand):

    protocol_spec = __doc__ = '''
    amount+/list.scroll_amount: The amount to scroll, a two item list with the first item being \
             either a number or the keywords, start and end. \
             And the second item being either 'p' for pages or 'l' for lines or 'u'
             for unscrolling by lines.
    match/str: The window to scroll
    '''

    short_desc = 'Scroll the specified windows'
    desc = (
        'Scroll the specified windows, if no window is specified, scroll the window this command is run inside.'
        ' :italic:`SCROLL_AMOUNT` can be either the keywords :code:`start` or :code:`end` or an'
        ' argument of the form :italic:`<number>[unit][+-]`. For example, :code:`30` will scroll down 30 lines, :code:`2p-`'
        ' will scroll up 2 pages and :code:`0.5p`will scroll down half page. :code:`3u` will *unscroll* by 3 lines, which means that 3 lines will move from the'
        ' scrollback buffer onto the top of the screen.'
    )
    options_spec = MATCH_WINDOW_OPTION + '''\n
--no-response
type=bool-set
default=false
Don't wait for a response indicating the success of the action. Note that
using this option means that you will not be notified of failures.
'''
    args = RemoteCommand.Args(spec='SCROLL_AMOUNT', count=1, special_parse='parse_scroll_amount(args[0])', json_field='amount')

    def message_to_kitty(self, global_opts: RCOptions, opts: 'CLIOptions', args: ArgsType) -> PayloadType:
        if len(args) < 1:
            self.fatal('Scroll amount must be specified')
        amt = args[0]
        amount: Tuple[Union[str, float], Optional[str]] = (amt, None)
        if amt not in ('start', 'end'):
            pages = 'p' in amt
            unscroll = 'u' in amt
            mult = -1 if amt.endswith('-') and not unscroll else 1
            q = float(amt.rstrip('+-plu'))
            if not pages and not q.is_integer():
                self.fatal('The number must be an integer')
            amount = q * mult, 'p' if pages else ('u' if unscroll else 'l')

        # defaults to scroll the window this command is run in
        return {'match': opts.match, 'amount': amount, 'self': True}

    def response_from_kitty(self, boss: Boss, window: Optional[Window], payload_get: PayloadGetType) -> ResponseType:
        amt = payload_get('amount')
        for window in self.windows_for_match_payload(boss, window, payload_get):
            if window:
                if amt[0] in ('start', 'end'):
                    getattr(window, {'start': 'scroll_home'}.get(amt[0], 'scroll_end'))()
                else:
                    amt, unit = amt
                    if unit == 'u':
                        window.screen.reverse_scroll(int(abs(amt)), True)
                    else:
                        unit = 'page' if unit == 'p' else 'line'
                        if unit == 'page' and not isinstance(amt, int) and not amt.is_integer():
                            amt = round(window.screen.lines * amt)
                            unit = 'line'
                        direction = 'up' if amt < 0 else 'down'
                        func = getattr(window, f'scroll_{unit}_{direction}')
                        for i in range(int(abs(amt))):
                            func()
        return None

scroll_window = ScrollWindow()


import os
from sys import argv
from json import dump
from skimage import io, transform

from find import *

def angles_poll(angles):
    real_angles = [a for a in angles if a is not None]
    if len(real_angles) > 0:
        return np.mean(real_angles)
    return None

def main():
    # bamboo.py <data_path> <out_path> <show>
    data = argv[1]  # path to the data directory
    out = argv[2]   # path to the output directory
    show = len(argv) == 4 and argv[3] == "-s"  # plots the found bamboo lines
    group_dirs = [gd for gd in os.listdir(data) if gd.startswith("group")]
    for group in group_dirs:
        print("---" + group + "---")
        group_data = os.path.join(data, group)
        group_out = os.path.join(out, group)
        if not os.path.exists(group_out):
            os.mkdir(group_out)
        file_names = [fn for fn in os.listdir(group_data) if fn.endswith(OBJ)]
        for fn in file_names:
            print(fn)
            data_file_path = os.path.join(group_data, fn)
            fragment = Render(data_file_path, pose='backside')
            bamboo_angles = [find_bamboo_lines_angle(fragment, show) for i in range(10)]
            chosen_angle = angles_poll(bamboo_angles)
            out_file_path = os.path.join(group_out, fn.removesuffix(OBJ) + JSON)
            with open(out_file_path, mode='w') as out_file:
                dump(chosen_angle, out_file)

def my_main():
    # ...09-...11 contains bamboo lines
    # ...12 doesn't contain bamboo lines
    path = "data/RPf_00311.obj"
    frag = Render(path, pose='backside')
    frag.plot()
    angle = find_bamboo_lines_angle(frag, show=True)
    if angle is not None:
        print(np.rad2deg(angle))
        frag.planar_rotation(angle)
        frag.plot()

    # Previous attempt:
    # real_angles = [a for a in angles if a is not None]
    # if len(real_angles) == 0:
    #     return None
    # angle_votes = np.zeros_like(real_angles)
    # for index_1, angle_1 in enumerate(real_angles):
    #     for index2, angle_2 in enumerate(real_angles):
    #         if index_1 != index2 and abs(angle_1 - angle_2) < epsilon:
    #             angle_votes[index_1] += 1
    # print(angle_votes)
    # return real_angles[np.argmax(angle_votes)]

def align_intact(frag_path, intact_path, show=False):
    frag = Render(frag_path, pose='backside')
    if show:
        frag.plot()
    angles = [find_bamboo_lines_angle(frag, show=show) for i in range(10)]
    angle = angles_poll(angles)
    if angle is not None:
        intact = io.imread(intact_path)
        rotated = transform.rotate(intact, np.rad2deg(angle))
        if show:
            io.imshow(rotated)
        return rotated
    return None

def my_main_2():
    for frag_id in ["09"]+([str(i) for i in range(10, 21)]):
        frag_path = "data/group39/RPf_003"+frag_id+".obj"
        intact_path = "data/group39/RPf_003"+frag_id+"_intact_mesh.png"
        aligned = align_intact(frag_path, intact_path)
        if aligned is not None:
            aligned = np.array(aligned * 255, dtype="uint8")
            io.imsave("out/RPf_003"+frag_id+"_aligned.png", aligned)

if __name__ == '__main__':
    main()


# Primary and Secondary Method Functions + Maps ==================================================

def convertToStr(addr):
    if not isinstance(addr, str):
        return ''
    else:
        return addr

def remPunctuation(addrString):
    newAddrString = addrString
    for index, char in enumerate(addrString):
        if char == '-' or char == '.' or char == ',':
            newAddrString = newAddrString[:index] + ' ' + newAddrString[index + 1:]
    return newAddrString

def postCodeLookup(distr, postCodeMap, cityList, cityListSimplified):
    # returns tuple (matchFound, city string)
    for districts in postCodeMap:
        if distr in districts:
            cityFromPC = remPunctuation(postCodeMap[districts].lower())
            for index, cityFromList in enumerate(cityListSimplified):
                if cityFromPC == cityFromList:
                    return True, cityList[index] # properly capitalized city
    return False, None

abbrMap = {
    "M'well": 'Motherwell',
    'Northants': 'Northamptonshire',
    'Berwick': 'Berwick-upon-Tweed',
    'Holme-on-spalding': 'Holme-on-Spalding-Moor',
    'Holy': 'Holytown',
    'Spald': 'Holme-on-Spalding-Moor',
    'Soton': 'Southampton',
    'Glas': 'Glasgow',
    'Eastleig': 'Eastleigh',
    'Doncs': 'Doncaster',
    'Billinigham': 'Billingham',
    'Lcester': 'Leicester',
    'Doncaste': 'Doncaster',
}

def checkAbbr(word):  # check list of exceptions/abbreviations
    if word in abbrMap:
        return abbrMap[word]
    return word

def cityStringParse(addr, cityList, cityListSimplified):
    matchesList = []
    addrSimplified = remPunctuation(addr.lower())
    for index, citySimple in enumerate(cityListSimplified):
        if citySimple in addrSimplified:
            matchesList.append(cityList[index])
    # Alternative:
    #matchesList = [cityList[index] for index, citySimple in enumerate(cityListSimplified) if
                   #citySimple in addrSimplified]

    if len(matchesList) == 0:
        return False, None
    else:
        return True, max(matchesList, key=len)


import pickle
from Schema.DAO import uWaterlooMath136FileNamesDAO
from probCalculation import the_def_score_pairs
from Schema.DAO import uWaterlooMath136ScoreDAO

sessionFileName = uWaterlooMath136FileNamesDAO.uWaterlooMath136FileNamesDAO()
sessionScore = uWaterlooMath136ScoreDAO.uWaterlooMath136ScoreDAO()
fileScorePair = the_def_score_pairs.the_def_score_paris

with open("../pickles/KeyToFileNames", 'rb') as loadKeyToFileDict:
    d = pickle.load(loadKeyToFileDict);
    for key,value in d.items():
        files = []

        for each in value:
            files.append(each.split(".")[0])

        sessionFileName.insert(key, ",".join(files))

#
# for key,value in fileScorePair.items():
#     sessionScore.insert(key,value)