# File naming and handling procedures for the Bpilnu project
# C. DeTar 5 April 2018

# Python 3 version

import os, sys, re, subprocess

verbose = False

######################################################################
def remoteRootDir(root, lrun):
    """Root of remote files for the current ensemble"""
    return os.path.join( root, lrun )

######################################################################
def archiveRootDir(root, afm, lrun):
    """Root of remote files for the current ensemble"""
    return os.path.join( root, afm, lrun )

######################################################################
def addRootPaths(param):
    """Add parameter-dependent archive and remote paths to dictionary"""

    ensemble = param['ensemble']
    run = ensemble['run']
    lrun = 'l' + run
    afm = 'a' + ensemble['afm']
    root = param['files']['root']
    remoteRoot = remoteRootDir(param['remoteRoot'], lrun )
    archiveRoot = archiveRootDir(param['archiveRoot'], afm, lrun)
    # Add to the parameter dictionary
    root['remote'] = remoteRoot
#    try:
#        root['remoteSSD'] = os.environ['DW_JOB_STRIPED']
#    except KeyError:
#        print("Warning, DW_JOB_STRIPED not defined")
#        root['remoteSSD'] = "DW_JOB_STRIPED"
    root['archive'] = archiveRoot

######################################################################
def localPath(filename):
    """Generic path for local files"""
    return os.path.join( localRoot, filename )

#####################################################################
def buildPath(root, dirs):
    """Build path from root plus a list of subdirs"""
    r = root
    if dirs != None:
        for d in dirs:
            r = os.path.join(r, d)
    return r

#####################################################################
def makePath(path):
    """Make directories"""
    try:
        stat = os.stat(path)
    except OSError:
        try:
            os.makedirs(path)
        except OSError:
            print("WARNING: Can't create directories", path)
            subprocess.call("ls -l /mnt/bb/detar", shell = True)
            # sys.exit(1)

#####################################################################
def checkSSDList(path):
    """See if 'path' is in the list of files presumably on the SSD"""
    return False

#####################################################################
# Class for staging files from the remote path to a local path for MPP execution
# Also used to archive (store) result files
# For multijob running for stdin, stdout, stdeff we use symlinks to map the
# internally constructed names with ".jnn" suffixes to the actual file names.

# 'w' in "mode" means don't fetch.  Store if it doesn't exist.
# 'x' in "mode" means scratch only. Fetch if available. Don't store
# 'r' in "mode" means must fetch.  Store if it doesn't exist.
class StageFile:
    """Staging and saving files"""
    def __init__(self, localDir, localSubdirs, remoteDir, remoteSubdirs, name, mode,
                 multiJobName, createLink):
        self.localDir = localDir
        self.remoteDir = remoteDir
        self.localSubdirs = localSubdirs
        self.remoteSubdirs = remoteSubdirs
        self.fileName = name
        self.mode = mode
        self.multiJobName = multiJobName
        self.createLink = createLink

        self.staged = False
        self.fd = None

        self.remoteDirs = buildPath(self.remoteDir, self.remoteSubdirs)
        makePath(self.remoteDirs)
        self.pathRemote = os.path.join(self.remoteDirs, self.fileName)

        # If no local directory specified, then use remote path
        if self.localDir == None:
            self.localDirs = None
            self.pathLocal = self.pathRemote
            self.pathSymLink = self.remoteDirs
        else:
            self.localDirs = buildPath(self.localDir, self.localSubdirs)
            makePath(self.localDirs)
            self.pathLocal = os.path.join(self.localDirs, self.fileName)
            self.pathSymLink = self.localDirs

        # Create symlink to file if requested
        if self.multiJobName != None and self.createLink:
            self.pathSymLink = os.path.join(self.pathSymLink, self.multiJobName)
            if verbose:
                print("Creating symlink to", self.pathLocal, "from", self.pathSymLink)
            # Create the target file if it doesn't exist
            if self.mode == 'w' and not os.access(self.pathLocal, os.R_OK):
                f = open(self.pathLocal, 'w')
                f.close()
            try:
                os.symlink(self.pathLocal, self.pathSymLink)
            except OSError:
                print("WARNING: ln -s failed for", self.pathSymLink)
        else:
            self.pathSymLink = None

        if not 'w' in self.mode:
            # Is the file already local?
            if os.access(self.pathLocal, os.R_OK):
                if verbose:
                    print("Using existing",self.pathLocal)
                self.staged = True

            # Perhaps it is in partfile format locally
            # We check only vol0000 and assume all other parts are in place
            elif os.access(self.pathLocal+'.vol0000', os.R_OK):
                if verbose:
                    print("Using existing partfile",self.pathLocal)
                self.staged = True

            elif checkSSDList(self.pathLocal):
                if verbose:
                    print("SSD list has",self.pathLocal)
                self.staged = True

            # If not local, then fetch it
            elif os.access(self.pathRemote, os.R_OK):
                cmd = ' '.join(('/usr/bin/rsync -auv', self.pathRemote, self.pathLocal))
                if verbose:
                    print('#',cmd)
                try:
                    subprocess.check_output(cmd, shell = True)
                except subprocess.CalledProcessError as e:
                    print("WARNING: rsync failed for", e.cmd)
                    print("return code", e.returncode)
                self.staged = True
            else:
                self.staged = False
                if 'r' in self.mode:
                    if verbose:
                        print("WARNING: can't find", self.pathRemote)

    def openwrite(self):
        if self.fd == None:
            try:
                self.fd = open(self.pathLocal,'w')
            except IOError:
                print("WARNING: Can't open", self.pathLocal)
                self.fd = None
        return self.fd

    def close(self):
        try:
            self.fd.close()
        except:
            print("WARNING: Close failed for", self.fileName)
        self.fd = None

    def exist(self):
        return self.staged

    def name(self):
        return self.fileName

    def path(self):
        if self.pathSymLink != None:
            # Strip any .jnn from the symlink name
            return re.sub('\.j\d\d$','',self.pathSymLink)
        else:
            return self.pathLocal

    def pathremote(self):
        return self.pathRemote

    def dirLocal(self):
        return self.localDirs

    def dirRemote(self):
        return self.remoteDirs

    def bzip2(self):
        filebz2 = re.compile(".*\.bz2$")
        if not filebz2.search(self.pathLocal):
            if os.access(self.pathLocal, os.R_OK):
                cmd = ' '.join(('bzip2', self.pathLocal))
                if verbose:
                    print("#", cmd)
                subprocess.check_output(cmd, shell = True)
            self.pathLocal = self.pathLocal + '.bz2'
            if not filebz2.search(self.pathRemote):
                self.pathRemote = self.pathRemote + '.bz2'

    def store(self):
        # Don't store if it doesn't exist
        if not self.staged:
            return
        # Reconcile multijob symlink with permanent name
        if self.pathSymLink != None:
            # If the symlink was not created, change the name instead
            if not self.createLink:
                try:
                    os.rename(self.pathSymLink, self.pathLocal)
                except OSError:
                    print("WARNING: rename failed for", self.pathSymLink)
            # Otherwise, just remove the symlink
            else:
                if verbose:
                    print("Removing sym link", self.pathSymLink)
                try:
                    os.remove(self.pathSymLink)
                except OSError:
                    print("WARNING: remove failed for", self.pathSymLink)

        # Copy file to archive if needed
        if verbose:
            print("#", self.pathLocal, "->", self.pathRemote)
        if not 'x' in self.mode and os.access(self.pathLocal, os.R_OK):
            if(self.pathLocal != self.pathRemote):
                cmd = ' '.join(('/usr/bin/rsync -auv --no-p --no-o --no-g', self.pathLocal, self.pathRemote))
                if verbose:
                    print(cmd)
                try:
                    subprocess.check_output(cmd, shell = True)
                except subprocess.CalledProcessError as e:
                    print("WARNING: rsync failed for", e.cmd)
                    print("return code", e.returncode)
        else:
            if verbose:
                print(self.pathLocal, "not stored, because either mode is", self.mode,"or read access is", os.access(self.pathLocal, os.R_OK))
            pass

    def delete_staged(self):
        if(self.staged and self.pathLocal != self.pathRemote):
            cmd = ' '.join(('/bin/rm', self.pathLocal))
            if verbose:
                print(cmd)
            try:
                subprocess.check_output(cmd, shell = True)
            except subprocess.CalledProcessError as e:
                print("WARNING: /bin/rm failed for", e.cmd)
                print("return code", e.returncode)

    pass

######################################################################
def codeCfg(suffix, cfg):
    """Encode tsrc and cfg for file names"""
    if suffix == '' or suffix == None:
        series = 'a'
    else:
        series = suffix
    return "{0:s}{1:06d}".format(series,int(cfg))

######################################################################
def codeTsrcCfg(tsrcConfigId):
    """Encode tsrc and cfg for file names"""
    (tsrc, suffix, cfg) = tsrcConfigId
    configId = codeCfg(suffix, cfg)
    return "t{0:d}.{1:s}".format(tsrc, configId)
######################################################################
def codeTsrcSym(tsrcConfigId,kjob):
    """Encode tsrc and kjob for file names"""
    (tsrc, suffix, cfg) = tsrcConfigId
    return "t{0:d}.j{1:02d}".format(tsrc, kjob)
######################################################################
def ensFile(pfx, run, tsrcConfigId):
    """Standard file name pattern containing the ensemble name, time source, and configuration"""
    sfx = codeTsrcCfg(tsrcConfigId)
    return  "{0:s}_{1:s}_{2:s}".format(pfx, run, sfx)

######################################################################
def milc2FNAL(suffix, cfg):
    """Convert MILC cfg to FNAL"""
    leading = 0
    if suffix == 'b':
        leading = 5
    elif suffix == 'c':
        leading = 6
    return "{0:d}{1:05d}".format(leading, cfg)

######################################################################
def rndFile(sq, run, tsrcConfigId):
    """Construct random source name"""
    return ensFile('rnd' + sq, run, tsrcConfigId)

######################################################################
def latFileCoul( run, suffix, cfg):
    """Convention for Coulomb-gauge-fixed lattice file names"""
    return "l{0:s}{1:s}-Coul.{2:d}.ildg".format(run, suffix, int(cfg))

######################################################################
def latFileEig( run, suffix, cfg):
    """Convention for eigenpair file names"""
    return "eigPRIMME{0:s}nv600er10{1:s}.{2:d}".format(run, suffix, int(cfg))

######################################################################
def latFileMILCv5( run, suffix, cfg):
    """Convention for MILC v5 lattice file names"""
    return "l{0:s}{1:s}.{2:d}".format(run, suffix, int(cfg))

######################################################################
def propNameKS(qkKey, run, tsrcConfigId):
    """Construct KS propagator file name"""
    return ensFile( 'prop_' + qkKey, run, tsrcConfigId )

######################################################################
def propNameClover(qkKey, run, tsrcConfigId):
    """Construct KS propagator file name"""
    return ensFile( 'prop_' + qkKey, run, tsrcConfigId )

######################################################################
def corr3ptFileName(chan, run, extT, tsrcConfigId):
    """Construct 3pt correlator file name"""
    (tsrc, suffix, cfg) = tsrcConfigId
    return "corr3pt_T{0:d}_{1:s}".format(extT,  codeCfg(suffix, cfg))

######################################################################
def corr2ptFileName(chan, run, tsrcConfigId):
    """Construct 2pt correlator file name"""
    (tsrc, suffix, cfg) = tsrcConfigId
    return "corr2pt_{0:s}".format(codeCfg(suffix, cfg))

######################################################################
def massLabel(qk, mk):
    """Construct mass label"""
    if qk['type'] == 'clover':
        return "k" + mk
    elif qk['type'] == 'KS':
        return "m" + mk

######################################################################
def momLabel(mom):
    """Construct momentum label"""
    return "p{0:d}{1:d}{2:d}".format(*tuple(mom))

######################################################################
def massSubdir2pt(mQk, mAQk, mom):
    """Construct subdirectory name for 2pt correlator files"""
    return '-'.join([mQk, mAQk, momLabel(mom)])

######################################################################
def massSubdir3pt(mQkS, mQkP, mQkD, mom):
    """Construct subdirectory name for 2pt correlator files"""
    return '-'.join([mQkP, mQkS, mQkD, momLabel(mom)])

######################################################################
def logFileName(run, tsrcConfigId, jobid, tag, seqno):
    """Construct log file (stdout) name"""
    (tsrc, suffix, cfg) = tsrcConfigId
    # return "logJob{0:s}_{1:s}_{2:s}".format(jobid, seqno, codeTsrcCfg(tsrcConfigId))
    return "logJob{0:s}{1:s}.{2:s}".format(tag, jobid, codeCfg(suffix, cfg))

######################################################################
def logFileSymLink(run, tsrcConfigId, jobid, tag, seqno, kjob, njobs):
    """Construct log file (stdout) name"""
    (tsrc, suffix, cfg) = tsrcConfigId
    if njobs == 1:
        return None
    else:
        return "logJob{0:s}{1:s}_{2:s}_{3:s}".format(tag, jobid, seqno,
                codeTsrcSym(tsrcConfigId, kjob))

######################################################################
def outFileName(run, tsrcConfigId, jobid, tag, seqno):
    """Construct log file (stdout) name"""
    return "outJob{0:s}{1:s}_{2:s}_{3:s}".format(tag, jobid, seqno, codeTsrcCfg(tsrcConfigId))

######################################################################
def outFileSymLink(run, tsrcConfigId, jobid, tag, seqno, kjob, njobs):
    """Construct log file (stdout) name"""
    if njobs == 1:
        return None
    else:
        return "outJob{0:s}{1:s}_{2:s}_{3:s}".format(tag, jobid, seqno, codeTsrcSym(tsrcConfigId, kjob))

######################################################################
def errFileName(run, tsrcConfigId, jobid, tag, seqno):
    """Construct error file (stderr) name"""
    return "errJob{0:s}{1:s}_{2:s}_{3:s}".format(tag, jobid, seqno, codeTsrcCfg(tsrcConfigId))

######################################################################
def errFileSymLink(run, tsrcConfigId, jobid, tag, seqno, kjob, njobs):
    """Construct error file (stderr) name"""
    if njobs == 1:
        return None
    else:
        return "errJob{0:s}{1:s}_{2:s}_{3:s}".format(tag, jobid, seqno, codeTsrcSym(tsrcConfigId, kjob))

######################################################################
def inFileName(run, tsrcConfigId, jobid, tag, seqno):
    """Construct input file (stdin) name"""
    return "inJob{0:s}{1:s}_{2:s}_{3:s}".format(tag, jobid, seqno, codeTsrcCfg(tsrcConfigId))

######################################################################
def inFileSymLink(run, tsrcConfigId, jobid, tag, seqno, kjob, njobs):
    """Construct input file (stdin) name"""
    if njobs == 1:
        return None
    else:
        return "inJob{0:s}{1:s}_{2:s}_{3:s}".format(tag, jobid, seqno, codeTsrcSym(tsrcConfigId, kjob))

######################################################################
def tarFileName(configId, jobid, tag):
    """Construct tar file name"""
    (suffix, cfg) = configId
    return "Job{0:s}{1:s}_{2:s}.tar.bz2".format(tag, jobid, codeCfg(suffix, cfg))

######################################################################
def prefixSpinTaste( pfx, spinTaste ):
    """Convert spin-taste prefix from gamma notation to bilinear notation
    For example, GXT-GT is converted to T14-V4"""
    gSpin, gTaste = spinTaste.split("-")
    pSpin = pfx[gSpin]
    pTaste = pfx[gTaste]
    prefix = '-'.join([ pSpin, pTaste])
    return prefix

######################################################################
def prefix3pt(pfx, stP, stC, extT, mQkP):
    """Make prefix for 3pt correlator"""
    prefixP = prefixSpinTaste(pfx, stP)
    prefixC = prefixSpinTaste(pfx, stC)
    prefix = '_'.join([prefixP, prefixC, 'T'+str(extT), mQkP])
    return prefix

######################################################################
def prefix2pt(pfx, stSnk):
    """Make prefix for 2pt correlator"""
    return prefixSpinTaste(pfx, stSnk)


from tfnn.body.network import Network
import tfnn

class ClfNetwork(Network):
    def __init__(self, n_inputs, n_outputs, method='softmax', do_dropout=False, do_l2=False, seed=None):
        if method == 'softmax':
            input_dtype = tfnn.float32
            output_dtype = tfnn.float32
            output_activator = tfnn.nn.softmax
        elif method == 'sigmoid':
            input_dtype = tfnn.float32
            output_dtype = tfnn.float32
            output_activator = tfnn.sigmoid
        else:
            raise ValueError("method should be one of ['softmax', 'sigmoid']")
        super(ClfNetwork, self).__init__(
            n_inputs, n_outputs, input_dtype, output_dtype, output_activator,
            do_dropout, do_l2, seed)
        self.method = method
        self.name = 'ClassificationNetwork'

    def _init_loss(self):
        with tfnn.name_scope('predictions'):
            if self.method == 'softmax':
                self.predictions = tfnn.nn.softmax(self.layers_final_output.iloc[-1], name='predictions')
            elif self.method == 'sigmoid':
                self.predictions = tfnn.nn.sigmoid(self.layers_final_output.iloc[-1], name='predictions')
        with tfnn.name_scope('loss'):
            if self.method == 'softmax':
                self.cross_entropy = tfnn.nn.softmax_cross_entropy_with_logits(
                    self.layers_final_output.iloc[-1],
                    self.target_placeholder,
                    name='xentropy')
            elif self.method == 'sigmoid':
                self.cross_entropy = tfnn.nn.sigmoid_cross_entropy_with_logits(
                    self.layers_final_output.iloc[-1],
                    self.target_placeholder,
                    name='xentropy')
            else:
                raise ValueError("method should be one of ['sparse_softmax', 'softmax', 'sigmoid']")
            self.loss = tfnn.reduce_mean(self.cross_entropy, name='xentropy_mean')

            if self.reg == 'l2':
                with tfnn.name_scope('l2_reg'):
                    regularizers = 0
                    for W in self.Ws:
                        regularizers += tfnn.nn.l2_loss(W, name='l2_reg')
                    regularizers *= self.l2_placeholder
                with tfnn.name_scope('l2_loss'):
                    self.loss += regularizers

            tfnn.scalar_summary('loss', self.loss)


# uncompyle6 version 3.7.4
# Python bytecode 2.6 (62161)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: build\bdist.win32\egg\dao\base.py
# Compiled at: 2011-11-04 04:58:46

def is_subclass(sub, sup):
    try:
        if sup in sub.__bases__:
            return True
    except:
        return False
    else:
        for klass in sub.__bases__:
            if is_subclass(klass, sup):
                return True

class Parser:

    def parse(self, exp):
        try:
            exp_parse = exp.___parse___
        except:
            if isinstance(exp, list):
                return [ self.parse(e) for e in exp ]
            else:
                if isinstance(exp, tuple):
                    return tuple(self.parse(e) for e in exp)
                return exp

        try:
            return exp_parse(self)
        except TypeError:
            return exp

def preparse(exp):
    return Parser().parse(exp)

class LoopExitNextTagger:
    """ use tagger to preprocess before solve expression"""
    surfix = '$'

    def __init__(self):
        self.new_label_id = 1
        self.labels = {}

    def make_label(self, label):
        if label is None:
            label = '$' + str(self.new_label_id)
            self.new_label_id += 1
        return label

    def push_label(self, control_struct_type, label):
        self.labels.setdefault(control_struct_type, []).append(label)
        self.labels.setdefault(None, []).append(label)
        return

    def pop_label(self, control_struct_type):
        self.labels[control_struct_type].pop()
        self.labels[None].pop()
        return

    def tag_loop_label(self, exp):
        try:
            exp_tag_loop_label = exp.tag_loop_label
        except:
            if isinstance(exp, list):
                return [ self.tag_loop_label(e) for e in exp ]
            else:
                if isinstance(exp, tuple):
                    return tuple(self.tag_loop_label(e) for e in exp)
                return exp

        try:
            return exp_tag_loop_label(self)
        except TypeError:
            return exp

def tag_loop_label(exp):
    return LoopExitNextTagger().tag_loop_label(exp)

def dao_repr(exp):
    try:
        exp_____repr____ = exp.____repr____
    except:
        if isinstance(exp, list) or isinstance(exp, tuple):
            return (',').join([ dao_repr(e) for e in exp ])
        else:
            return repr(exp)

    try:
        return exp_____repr____()
    except TypeError:
        return repr(exp)

def apply_generators(generators):
    length = len(generators)
    if length == 0:
        yield True
        return
    if length == 1:
        for _ in generators[0]:
            yield True

        return
    i = 0
    while i < length:
        try:
            generators[i].next()
            if i == length - 1:
                yield True
                return
            i += 1
        except StopIteration:
            if i == 0:
                return
            i -= 1
        except GeneratorExit:
            raise

def unify(x, y, env, occurs_check=False):
    try:
        x_unify = x.unify
    except AttributeError:
        try:
            y_unify = y.unify
        except AttributeError:
            if (isinstance(x, list) or isinstance(x, tuple)) and (isinstance(y, list) or isinstance(y, tuple)):
                for _ in unify_list(x, y, env, occurs_check):
                    yield True

            if x == y:
                yield True
            return
        else:
            for _ in y_unify(x, env, occurs_check):
                yield True

        return
    else:
        for _ in x_unify(y, env, occurs_check):
            yield True

def unify_list(list1, list2, env, occurs_check=False):
    """unify list1 with list2 in env."""
    if len(list1) != len(list2):
        return
    for _ in apply_generators(tuple(unify(x, y, env, occurs_check) for (x, y) in zip(list1, list2))):
        yield True

def deref(x, env):
    try:
        x_deref = x.deref
    except AttributeError:
        if isinstance(x, list):
            return [ deref(e, env) for e in x ]
        else:
            if isinstance(x, tuple):
                return tuple(deref(e, env) for e in x)
            return x

    return x_deref(env)

def getvalue(x, env):
    try:
        x_getvalue = x.getvalue
    except AttributeError:
        if isinstance(x, list):
            return [ getvalue(e, env) for e in x ]
        else:
            if isinstance(x, tuple):
                return tuple(getvalue(e, env) for e in x)
            return x

    return x_getvalue(env)

def apply_generators_list(generators):
    length = len(generators)
    if length == 0:
        yield []
        return
    i = 0
    result = []
    while i < length:
        try:
            result.append(generators[i].next())
            if i == length - 1:
                yield result
                return
            i += 1
        except StopIteration:
            if i == 0:
                return
            i -= 1
        except GeneratorExit:
            raise

def peek_value(exp, env):
    try:
        exp_take_value = exp.peek_value
    except AttributeError:
        if isinstance(exp, list):
            return [ peek_value(e) for e in exp ]
        else:
            if isinstance(exp, tuple):
                return tuple(peek_value(e) for e in exp)
            return exp

    return exp_take_value(env)

def copy(exp, memo):
    try:
        exp_copy = exp.copy
    except AttributeError:
        if isinstance(exp, list):
            return [ getvalue(e, memo) for e in exp ]
        else:
            if isinstance(exp, tuple):
                return tuple(getvalue(e, memo) for e in exp)
            return exp

    return exp_copy(memo)

def copy_rule_head(arg_exp, env):
    try:
        arg_exp_copy_rule_head = arg_exp.copy_rule_head
    except AttributeError:
        if isinstance(arg_exp, list):
            return [ copy_rule_head(e, env) for e in arg_exp ]
        else:
            if isinstance(arg_exp, tuple):
                return tuple(copy_rule_head(e, env) for e in arg_exp)
            return arg_exp

    return arg_exp_copy_rule_head(env)

def match_list(list1, list2):
    if len(list1) != len(list2):
        return False
    for (x, y) in zip(list1, list2):
        if not match(x, y):
            return False

    return True

def match(x, y):
    try:
        x_match = x.match
    except AttributeError:
        if isinstance(x, list) or isinstance(x, tuple) or isinstance(y, list) and isinstance(y, tuple):
            return match_list(x, y)
        else:
            return x == y

    return x_match(y)

def contain_var(x, y):
    try:
        return x.contain_var(x, y)
    except:
        return False

def closure(exp, env):
    try:
        exp_closure = exp.closure
    except AttributeError:
        if isinstance(exp, list) or isinstance(exp, tuple):
            return tuple(closure(e, env) for e in exp)
        else:
            return exp

    return exp_closure(env)


import ast
from flax_core.base.base_image_processor import BaseImageProcessor
from .src.template_process import TemplateProcess

class TemplateMatch(BaseImageProcessor):
    config = None
    default_config = {
        "extract": 4,
        "log": False,
        "shift": 90,
        "match_thres": 40,
        "match_thres_all_template": 40
    }

    def __init__(self, config=None):
        BaseImageProcessor.__init__(self)
        self.config = self.default_config
        if config is not None:
            if isinstance(config, str):
                self.config = ast.literal_eval(config)
            else:
                self.config = config

        self.template_process = TemplateProcess(self.config)
        self.img_path = None
        self.tpl_files = []

    def imread(self, img_path, tpl_files=[]):
        self.img_path = img_path
        self.tpl_files = tpl_files

    def process(self):
        """
        :param self:
        :return: {
                    'best_template': {},
                    'list_template': []
                }
        """
        # requirement must run template pre-processing before running matching function
        result = self.template_process.get_templates_matched(self.img_path, self.tpl_files)
        return result

    def save(self, dir, prefix):
        pass


##
# Copyright (c) 2005-2016 Apple Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
##

"""
Extensions to twisted.internet.ssl.
"""

__all__ = [
    "ChainingOpenSSLContextFactory",
    "simpleClientContextFactory",
]

import OpenSSL
from OpenSSL.SSL import Context as SSLContext, SSLv23_METHOD, OP_NO_SSLv2, \
    OP_CIPHER_SERVER_PREFERENCE, OP_NO_SSLv3, VERIFY_NONE, VERIFY_PEER, \
    VERIFY_FAIL_IF_NO_PEER_CERT, VERIFY_CLIENT_ONCE

from twisted.internet.ssl import DefaultOpenSSLContextFactory
from twisted.internet._sslverify import Certificate, _tolerateErrors, VerificationError, verifyHostname
from twisted.python.failure import Failure

import uuid

_OP_NO_COMPRESSION = getattr(OpenSSL.SSL, 'OP_NO_COMPRESSION', 0x00020000)
SSL_CB_HANDSHAKE_DONE = 0x20

class ChainingOpenSSLContextFactory (DefaultOpenSSLContextFactory):

    def __init__(
        self, privateKeyFileName, certificateFileName,
        sslmethod=SSLv23_METHOD,
        certificateChainFile=None, keychainIdentity=None,
        passwdCallback=None, ciphers=None,
        verifyClient=False, requireClientCertificate=False,
        verifyClientOnce=True, verifyClientDepth=9,
        clientCACertFileNames=[], sendCAsToClient=True,
        peerName=None
    ):
        self.certificateChainFile = certificateChainFile
        self.keychainIdentity = keychainIdentity
        self.passwdCallback = passwdCallback
        self.ciphers = ciphers

        self.peerName = peerName

        self.verifyClient = verifyClient
        self.requireClientCertificate = requireClientCertificate
        self.verifyClientOnce = verifyClientOnce
        self.verifyClientDepth = verifyClientDepth
        self.clientCACertFileNames = clientCACertFileNames
        self.sendCAsToClient = sendCAsToClient

        DefaultOpenSSLContextFactory.__init__(
            self,
            privateKeyFileName,
            certificateFileName,
            sslmethod=sslmethod
        )

    def cacheContext(self):
        # Unfortunate code duplication.
        ctx = SSLContext(self.sslmethod)

        # Always disable SSLv2/SSLv3/Compression
        ctx.set_options(OP_NO_SSLv2)
        ctx.set_options(OP_NO_SSLv3)
        ctx.set_options(_OP_NO_COMPRESSION)

        if self.ciphers is not None:
            ctx.set_cipher_list(self.ciphers)
            ctx.set_options(OP_CIPHER_SERVER_PREFERENCE)

        if self.passwdCallback is not None:
            ctx.set_passwd_cb(self.passwdCallback)

        if self.keychainIdentity and hasattr(ctx, "use_keychain_identity"):
            ctx.use_keychain_identity(self.keychainIdentity)
        else:
            if self.certificateFileName:
                ctx.use_certificate_file(self.certificateFileName)
            if self.privateKeyFileName:
                ctx.use_privatekey_file(self.privateKeyFileName)
            if self.certificateChainFile:
                ctx.use_certificate_chain_file(self.certificateChainFile)

        verifyFlags = VERIFY_NONE
        if self.verifyClient:
            verifyFlags = VERIFY_PEER
            if self.requireClientCertificate:
                verifyFlags |= VERIFY_FAIL_IF_NO_PEER_CERT
            if self.verifyClientOnce:
                verifyFlags |= VERIFY_CLIENT_ONCE
            if self.clientCACertFileNames:
                store = ctx.get_cert_store()
                for cert in self.clientCACertFileNames:
                    with open(cert) as f:
                        certpem = f.read()
                    cert = Certificate.loadPEM(certpem)
                    store.add_cert(cert.original)
                    if self.sendCAsToClient:
                        ctx.add_client_ca(cert.original)

            # When a client certificate is used we also need to set a session context id
            # to avoid openssl SSL_F_SSL_GET_PREV_SESSION,SSL_R_SESSION_ID_CONTEXT_UNINITIALIZED
            # errors
            ctx.set_session_id(str(uuid.uuid4()).replace("-", ""))

        # It'd be nice if pyOpenSSL let us pass None here for this behavior (as
        # the underlying OpenSSL API call allows NULL to be passed).  It
        # doesn't, so we'll supply a function which does the same thing.
        def _verifyCallback(conn, cert, errno, depth, preverify_ok):
            return preverify_ok
        ctx.set_verify(verifyFlags, _verifyCallback)

        if self.verifyClientDepth is not None:
            ctx.set_verify_depth(self.verifyClientDepth)

        if self.peerName:
            if hasattr(ctx, "set_peer_name"):
                ctx.set_peer_name(self.peerName)
            elif hasattr(ctx, "set_info_callback"):
                ctx.set_info_callback(
                    _tolerateErrors(self._identityVerifyingInfoCallback)
                )
            else:
                raise ValueError("No suitable SSL API for verifying the peer host name")

        self._context = ctx

    def _identityVerifyingInfoCallback(self, connection, where, ret):
        """
        U{info_callback
        <http://pythonhosted.org/pyOpenSSL/api/ssl.html#OpenSSL.SSL.Context.set_info_callback>
        } for pyOpenSSL that verifies the hostname in the presented certificate
        matches the one passed to this L{ClientTLSOptions}.

        @param connection: the connection which is handshaking.
        @type connection: L{OpenSSL.SSL.Connection}

        @param where: flags indicating progress through a TLS handshake.
        @type where: L{int}

        @param ret: ignored
        @type ret: ignored
        """
        if where & SSL_CB_HANDSHAKE_DONE:
            try:
                hostname = self.peerName.decode("utf-8") if isinstance(self.peerName, str) else self.peerName
                verifyHostname(connection, hostname)
            except VerificationError:
                f = Failure()
                transport = connection.get_app_data()
                transport.failVerification(f)

def simpleClientContextFactory(hostname):
    """
    Get a client context factory.
    """
    return ChainingOpenSSLContextFactory(
        "", "",
        certificateChainFile="",
        keychainIdentity="",
        peerName=hostname,
    )


# -*- coding: utf-8 -*-
"""
Created on Fri Oct 27 13:43:47 2017

@author: Rebecca
"""

import numpy as np
from linearRegCostFunction import linearRegCostFunction as computarCusto
from linearRegGD import gd

def learningCurve(x,y,xval,yval,initialTheta,alpha,lamda):
    trainingError = []
    valError = []
    for i in range(1,len(x)):
        x_tmp = x[:i,:]
        y_tmp = y[:i]
        theta,Jtheta,converged,iterations = gd(x_tmp,y_tmp,initialTheta,alpha,lamda)
        if not converged:
            raise Exception('GD with ' + str(i) + ' training examples did not converged.')
        errorT,grad = computarCusto(x_tmp,y_tmp,theta,0)
        trainingError.append(errorT)
        errorVal,grad = computarCusto(xval,yval,theta,0)
        valError.append(errorVal)

    return np.array(trainingError),np.array(valError)


"""Standard FAUCET pipeline."""

# Copyright (C) 2015 Brad Cowie, Christopher Lorier and Joe Stringer.
# Copyright (C) 2015 Research and Education Advanced Network New Zealand Ltd.
# Copyright (C) 2015--2018 The Contributors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

class ValveTableConfig: # pylint: disable=too-few-public-methods,too-many-instance-attributes
    """Configuration for a single table."""

    def __init__(self, name, exact_match=None, meter=None, output=True, # pylint: disable=too-many-arguments
                 miss_goto=None, size=None, match_types=None, set_fields=None, dec_ttl=None,
                 vlan_port_scale=None):
        self.name = name
        self.exact_match = exact_match
        self.meter = meter
        self.output = output
        self.miss_goto = miss_goto
        self.size = size
        self.match_types = match_types
        self.set_fields = set_fields
        self.dec_ttl = dec_ttl
        self.vlan_port_scale = vlan_port_scale

    def __str__(self):
        field_strs = ' '.join([
            '%s: %s' % (key, val) for key, val in sorted(self.__dict__.items())])
        return 'table config %s' % field_strs

    def __repr__(self):
        return self.__str__()

    def __hash__(self):
        return hash(self.__str__())

    def __eq__(self, other):
        return self.__hash__() == other.__hash__()

    def __lt__(self, other):
        return self.__hash__() < other.__hash__()

def _fib_table(ipv):
    return ValveTableConfig(
        'ipv%u_fib' % ipv,
        match_types=(('eth_type', False), ('ipv%u_dst' % ipv, True), ('vlan_vid', False)),
        set_fields=('eth_dst', 'eth_src', 'vlan_vid'),
        dec_ttl=True,
        vlan_port_scale=3.1)

# TODO: implement an eth_type table before VLAN. This would enable interception
# of control protocols and simplify matches in vlan/eth_src, enabling use of exact_match.
FAUCET_PIPELINE = (
    ValveTableConfig(
        'port_acl'),
    ValveTableConfig(
        'vlan',
        match_types=(('eth_dst', True), ('eth_type', False),
                     ('in_port', False), ('vlan_vid', False)),
        set_fields=('vlan_vid',),
        vlan_port_scale=1.1),
    ValveTableConfig(
        'vlan_acl'),
    ValveTableConfig(
        'eth_src',
        miss_goto='eth_dst',
        match_types=(('eth_dst', True), ('eth_src', False), ('eth_type', False),
                     ('in_port', False), ('vlan_vid', False)),
        set_fields=('vlan_vid', 'eth_dst'),
        vlan_port_scale=4.1),
    _fib_table(4),
    _fib_table(6),
    ValveTableConfig(
        'vip',
        match_types=(('arp_tpa', False), ('eth_dst', False), ('eth_type', False),
                     ('icmpv6_type', False), ('ip_proto', False))),
    ValveTableConfig(
        'eth_dst',
        exact_match=True,
        miss_goto='flood',
        match_types=(('eth_dst', False), ('vlan_vid', False)),
        vlan_port_scale=4.1),
    ValveTableConfig(
        'flood',
        match_types=(('eth_dst', True), ('in_port', False), ('vlan_vid', False)),
        vlan_port_scale=2.1),
)


## FT991a operating manual: http://www.hfelectronics.nl/docs/Manuals/FT-991A_Manual.pdf
## CAT commands manual: https://www.yaesu.com/downloadFile.cfm?FileID=13370&FileCatID=158&FileName=FT%2D991A%5FCAT%5FOM%5FENG%5F1711%2DD.pdf&FileContentType=application%2Fpdf
## TODO: Add try except blocks around send commands
## TODO: Add custom exceptions for each command function

from rsutils import send_rig_cmd, rslog, strip_prefix, strip_menu_return #project import

#Yaesu F991a globals
#TODO: Maybe move to a config file
VFO_A = "A"
VFO_B = "B"
VALID_VFO = [VFO_A, VFO_B]
VFO_FREQ_MIN_HZ = 30000         #.03Mhz
VFO_FREQ_MAX_HZ = 470000000     #470MHz
AF_GAIN_MIN = 0
AF_GAIN_MAX = 254
CAT_RATES = {
    0: 4800,
    1: 9600,
    2: 19200,
    3: 38400,
}

## SWAP VFOS - TODO: combine into one functions?
# {"catcmd": "AB", "desc": "VFO-A TO VFO-B"}
def atob():
    cmd = "AB"
    out = send_rig_cmd(cmd)

# {"catcmd": "BA", "desc": "VFO-B TO VFO-A"},
def btoa():
    cmd = "BA"
    send_rig_cmd(cmd)

## Antenna tuner control
# {"catcmd": "AC", "desc": "ANTENNA TUNER CONTROL"}
def get_at():
    cmd =   "AC"
    out = send_rig_cmd(cmd)
    if(out == "AC001"):
        return True
    elif(out == "AC000"):
        return False

def set_at(at_state):
    cmd = ""
    if at_state == True:
        cmd = "AC001"
    elif at_state == False:
        cmd = "AC000"
    else:
        rslog("Invalid command for set_at(). Options are True or False (on/off)")
    out = send_rig_cmd(cmd)
    return out

def start_at():
    #To be tested. No dummy load for the rig :-(
    cmd = "AC002"
    out = send_rig_cmd(cmd)
    return out

## AF Gain controls
# {"catcmd": "AG", "desc": "AF GAIN"}
def get_afgain():
    cmd = "AG0"
    out = send_rig_cmd(cmd)
    #strip command prefix and convert to int
    out = out[3:]
    out = int(out)
    return(out)

# Sets  the  receiver  audio  volume  level.
def set_afgain(gain):
    #check arg input between 0 - 254
    if(gain >= AF_GAIN_MIN and gain <= AF_GAIN_MAX):
        #convert to 3 digits and string
        gain = ('{0:03d}'.format(gain))
        gain = str(gain)
        cmd = "AG0" + str(gain)
        out = send_rig_cmd(cmd)
        return True
    else:
        rslog("set_afgain() value not in range")
        return False

# "400":   {"catcmd": "AI", "desc": "AUTO INFORMATION"},
# "500":   {"catcmd": "AM", "desc": "VFO-A TO MEMORY CHANNEL"},
# "700":   {"catcmd": "BC", "desc": "AUTO NOTCH"},
# "800":   {"catcmd": "BD", "desc": "BAND DOWN"},
# "900" :  {"catcmd": "BI", "desc": "BREAK-IN"},
# "1000" : {"catcmd": "BP", "desc": "MANUAL NOTCH"},
# "1100" : {"catcmd": "BS", "desc": "BAND SELECT"},
# "1200" : {"catcmd": "BU", "desc": "BAND UP"},
# "1300" : {"catcmd": "BY", "desc": "BUSY"},
# "1400" : {"catcmd": "CH", "desc": "CHANNEL UP/DOWN"},
# "1500" : {"catcmd": "CN", "desc": "CTCSS/DCS NUMBER"},
# "1600" : {"catcmd": "CO", "desc": "CONTOUR"},
# "1700" : {"catcmd": "CS", "desc": "CW SPOT"},
# "1800" : {"catcmd": "CT", "desc": "CTCSS"},
# "1900" : {"catcmd": "DA", "desc": "DIMMER"},
# "2000" : {"catcmd": "DN", "desc": "DOWN"},
# "2100" : {"catcmd": "DT", "desc": "DATE AND TIME"},
# "2200" : {"catcmd": "ED", "desc": "ENCORDER DOWN"},
# "2300" : {"catcmd": "EK", "desc": "ENT KEY"},
# "2400" : {"catcmd": "EU", "desc": "ENCORDER UP"},

# {"catcmd": "EX", "desc": "MENU"}
## Menu item cmd format EX12312345678

#menu item 031 CAT RATE
def get_catrate():
    cmd = "EX031"
    out = send_rig_cmd(cmd)
    out = strip_prefix(out)
    out = strip_menu_return(out)
    out = CAT_RATES[int(out)]

    return out

#def set_catrate():
#    return False

## Get and set VFO frequencies
# {"catcmd": "FA", "desc": "FREQUENCY VFO-A"}
# {"catcmd": "FB", "desc": "FREQUENCY VFO-B"}
def get_vfo(vfo, format="hz"):
    vfo = vfo.upper()

    if(vfo in VALID_VFO):
        cmd = "F" + vfo
        out = send_rig_cmd(cmd)

        if(format == "hz"):
            out = out[2:]
            out = int(out)
            return out
        elif(format == "string"):
            out = out[2:]
            #make a nice string
            mhz = str(int(out[0:3]))
            khz = "{}".format(out[3:6])
            hz = "{}".format(out[6:9])
            freqstring = mhz + "." + khz + hz + "MHz"
            return freqstring
        elif(format == "raw"):
            return out
    else:
        rslog("Not a valid VFO for get_vfo()")
        return False

#Set VFO frequency in Hz
def set_vfo(vfo, freq_hz):
    vfo = vfo.upper()
    if(freq_hz >= VFO_FREQ_MIN_HZ and freq_hz <= VFO_FREQ_MAX_HZ):
        if(vfo in VALID_VFO):
            freq_padded = "{:0>9d}".format(freq_hz)
            cmd = "F" + vfo
            cmd = cmd + str(freq_padded)
            out = send_rig_cmd(cmd)

            #No output from radio
            return True
        else:
            rslog("Not a valid VFO for get_vfo()")
            return False
    else:
        rslog("Invalid frequency range for set_vfo() " + str(VFO_FREQ_MIN_HZ) + " - " + str(VFO_FREQ_MAX_HZ))

# "2800" : {"catcmd": "FS", "desc": "FAST STEP"},
# "2900" : {"catcmd": "FT", "desc": "FUNCTION TX"},
# "3000" : {"catcmd": "GT", "desc": "AGC FUNCTION"},

## Radio ID
# Usage: get_id(format) where format=num returns int with ID,
# format="name" returns name string, format="raw" returns raw CAT cmd return (minus the ;)
# {"catcmd": "ID", "desc": "IDENTIFICATION"}
def get_id(format="name"):
    cmd = "ID"
    out = send_rig_cmd(cmd)

    if(format == "num"):
        #strip command prefix
        out = out[3:]           #TODO: use rsutils strip_prefix
        out = int(out)          #TODO: move this value to a config file
        return out
    elif(format == "name"):
        return "FT-991A"        #TODO: move this value to a config file
    elif(format == "raw"):
        return out

#{"catcmd": "IF", "desc": "INFORMATION"}
def get_info():
    cmd = "IF"
    out = send_rig_cmd(cmd)
    raw = out #save the raw response for later
    ret = {}

    #strip command prefix
    out = out[2:]

    #3 chars current memory channel
    ret["active_mem"] = out[0:3]
    ret["vfoa_freq"] = int(out[3:12])
    ret["clar_offset"] = int(out[12:17])
    ret["rx_clar_on"] = bool(out[17:18])  #0 = OFF, 1 = ON
    ret["tx_clar_on"] = bool(out[18:19])  #0 = OFF, 1 = ON

    op_mode = out[19:20]
    if(op_mode == "1"):
        ret["op_mode"] = "LSB"
    elif(op_mode == "2"):
        ret["op_mode"] = "USB"
    elif(op_mode == "3"):
        ret["op_mode"] = "CW"
    elif(op_mode == "4"):
        ret["op_mode"] = "FM"
    elif(op_mode == "5"):
        ret["op_mode"] = "AM"
    elif(op_mode == "6"):
        ret["op_mode"] = "RTTY-LSB"
    elif(op_mode == "7"):
        ret["op_mode"] = "CW-R"
    elif(op_mode == "8"):
        ret["op_mode"] = "DATA-LSB"
    elif(op_mode == "9"):
        ret["op_mode"] = "RTTY-USB"
    elif(op_mode == "A"):
        ret["op_mode"] = "DATA-FM"
    elif(op_mode == "B"):
        ret["op_mode"] = "FM-N"
    elif(op_mode == "C"):
        ret["op_mode"] = "DATA-USB"
    elif(op_mode == "D"):
        ret["op_mode"] = "AM-N"
    elif(op_mode == "E"):
        ret["op_mode"] = "C4FM"
    else:
        ret["op_mode"] = "Error?"

    mem_mode = out[20:21] #TODO: find a better name for this
    if(mem_mode == "0"):
        ret["mem_mode"] = "VFO"
    elif(mem_mode == "1"):
        ret["mem_mode"] = "Memory"
    elif(mem_mode == "2"):
        ret["mem_mode"] = "Memory Tune"
    elif(mem_mode == "3"):
        ret["mem_mode"] = "Quick Memory Bank (QMB)"
    else:
        ret["mem_mode"] = "Error?"

    code_type = out[21:22]
    if(code_type == "0"):
        ret["code_type"] = "CTCSS OFF"
    elif(code_type == "1"):
        ret["code_type"] = "CTCSS ENC/DEC"
    elif(code_type == "2"):
        ret["code_type"] = "CTCSS ENC"
    elif(code_type == "3"):
        ret["code_type"] = "DCS ENC/DEC"
    elif(code_type == "4"):
        ret["code_type"] = "DCS ENC"
    else:
        ret["code_type"] = "Error?"

    simplex = out[24:25]
    if(simplex == "0"):
        ret["simplex"] = "Simplex"
    elif(simplex == "1"):
        ret["simplex"] = "Plus Shift"
    elif(simplex == "2"):
        ret["simplex"] = "Minus Shift"
    else:
        ret["simplex"] = "Error?"

    ret["raw_output"] = raw

    return ret

# {"catcmd": "IS", "desc": "IF-SHIFT"}
def get_ifshift():
    return False

def set_ifshift():
    return False

# "3400" : {"catcmd": "KM", "desc": "KEYER MEMORY"},
# "3500" : {"catcmd": "KP", "desc": "KEY PITCH"},
# "3600" : {"catcmd": "KR", "desc": "KEYER"},
# "3700" : {"catcmd": "KS", "desc": "KEY SPEED"},
# "3800" : {"catcmd": "KY", "desc": "CW KEYING"},
# "3900" : {"catcmd": "LK", "desc": "LOCK"},
# "4100" : {"catcmd": "LM", "desc": "LOAD MESSEGE"},
# "4200" : {"catcmd": "MA", "desc": "MEMORY CHANNEL TO VFO A"},
# "4300" : {"catcmd": "MC", "desc": "MEMORY CHANNEL"},
# "4400" : {"catcmd": "MD", "desc": "MODE"},
# "4500" : {"catcmd": "MG", "desc": "MIC GAIN"},
# "4600" : {"catcmd": "ML", "desc": "MONITOR LEVEL"},
# "4700" : {"catcmd": "MR", "desc": "MEMORY READ"},
# "4800" : {"catcmd": "MS", "desc": "METER SW"},
# "4900" : {"catcmd": "MT", "desc": "MEMORY CHANNEL WRITE/TAG"},
# "5100" : {"catcmd": "MW", "desc": "MEMORY WRITE"},
# "5200" : {"catcmd": "MX", "desc": "MOX SET"},
# "5300" : {"catcmd": "NA", "desc": "NARROW"},
# "5400" : {"catcmd": "NB", "desc": "NOISE BLANKER"},
# "5500" : {"catcmd": "NL", "desc": "NOISE BLANKER LEVEL"},
# "5600" : {"catcmd": "NR", "desc": "NOISE REDUCTION"},
# "5700" : {"catcmd": "OI", "desc": "OPPOSITE BAND INFORMATION"},
# "5800" : {"catcmd": "SO", "desc": "FFSET (Repeater Shift)"},
# "5900" : {"catcmd": "PA", "desc": "PRE-AMP (IPO)"},
# "6100" : {"catcmd": "PB", "desc": "PLAY BACK"},
# "6200" : {"catcmd": "PC", "desc": "POWER CONTROL"},
# "6300" : {"catcmd": "PL", "desc": "SPEECH PROCESSOR LEVEL"},
# "6400" : {"catcmd": "PR", "desc": "SPEECH PROCESSOR"},
# "6500" : {"catcmd": "PS", "desc": "POWER SWITCH"},
# "6600" : {"catcmd": "QI", "desc": "QMB STORE"},
# "6700" : {"catcmd": "QR", "desc": "QMB RECALL"},
# "6800" : {"catcmd": "QS", "desc": "QUICK SPLIT"},
# "6900" : {"catcmd": "RA", "desc": "RF ATTENUATOR"},
# "7100" : {"catcmd": "RC", "desc": "CLAR CLEAR"},
# "7200" : {"catcmd": "RD", "desc": "CLAR DOWN"},
# "7300" : {"catcmd": "RG", "desc": "RF GAIN"},
# "7400" : {"catcmd": "RI", "desc": "RADIO INFORMATION"},
# "7500" : {"catcmd": "RL", "desc": "NOISE REDUCTION LEVEL"},
# "7600" : {"catcmd": "RM", "desc": "READ METER"},
# "7700" : {"catcmd": "RS", "desc": "RADIO STATUS"},
# "7800" : {"catcmd": "RT", "desc": "CLAR"},
# "7900" : {"catcmd": "RU", "desc": "CLAR UP"},
# "8100" : {"catcmd": "SC", "desc": "SCAN"},
# "8200" : {"catcmd": "SD", "desc": "SEMI BREAK-IN DELAY TIME"},
# "8300" : {"catcmd": "SH", "desc": "WIDTH"},
# "8400" : {"catcmd": "SM", "desc": "S METER"},
# "8500" : {"catcmd": "SQ", "desc": "SQUELCH LEVEL"},
# "8600" : {"catcmd": "SV", "desc": "SWAP VFO"},
# "8700" : {"catcmd": "TS", "desc": "TXW"},
# "8800" : {"catcmd": "TX", "desc": "TX SET"},
# "8900" : {"catcmd": "UL", "desc": "UNLOCK"},
# "9100" : {"catcmd": "UP", "desc": "UP"},
# "9200" : {"catcmd": "VD", "desc": "VOX DELAY TIME"},
# "9300" : {"catcmd": "VG", "desc": "VOX GAIN"},
# "9400" : {"catcmd": "VM", "desc": "[V/M] KEY FUNCTION"},
# "9500" : {"catcmd": "VX", "desc": "VOX"},
# "9600" : {"catcmd": "XT", "desc": "TX CLAR"},
# "9700" : {"catcmd": "ZI", "desc": "ZERO IN"}


from django.conf.urls.defaults import *

urlpatterns = patterns('memo.views',
    url(r'^$', 'memos', name='memos'),
    url(r'^memorando/', 'memorando', name='memorando'),
    url(r'^pesquisa/', 'search_form', name='search_form'),
    url(r'^search/', 'search', name='search'),
    #url(r'^(\d+)$', 'memo_types', name='memo_types'),
    #url(r'^([-\w]+)/$','speaker',name='speaker'),
)


# coding=utf-8
# 将NSL_KDD数据向量化
import csv
import pickle

import numpy as np

# 获取数据集中所有的非数字标签,方面向量化
def get_sym():
    sym = {"protocol_type": [], "service": [], "flag": [], "result": []}
    with open("KDDTrain+.csv") as f:
        reader = csv.reader(f)
        for row in reader:
            if row[1] not in sym["protocol_type"]:
                sym["protocol_type"].append(row[1])
            if row[2] not in sym["service"]:
                sym["service"].append(row[2])
            if row[3] not in sym["flag"]:
                sym["flag"].append(row[3])
            if row[41] not in sym["result"]:
                sym["result"].append(row[41])
    with open("KDDTest+.csv") as f:
        reader = csv.reader(f)
        for row in reader:
            if row[1] not in sym["protocol_type"]:
                sym["protocol_type"].append(row[1])
            if row[2] not in sym["service"]:
                sym["service"].append(row[2])
            if row[3] not in sym["flag"]:
                sym["flag"].append(row[3])
            if row[41] not in sym["result"]:
                sym["result"].append(row[41])
    with open("sym", 'wb') as f:
        pickle.dump(sym, f)

def vector_data():
    Train_list = []
    TrainTFlabel_list = []
    TrainALLlabel_list = []
    Test_list = []
    TestTFlabel_list = []
    TestALLlabel_list = []
    with open("sym", "rb") as f:
        sym = pickle.load(f)
    with open("KDDTrain+.csv") as f:
        reader = csv.reader(f)
        for row in reader:
            row[1] = sym["protocol_type"].index(row[1])
            row[2] = sym["service"].index(row[2])
            row[3] = sym["flag"].index(row[3])
            row[41] = sym["result"].index(row[41])
            Train_list.append(row[0:41])
            TrainALLlabel_list.append(row[41])
            if row[41] == 0:  # 0表示normal,其他表示为攻击流量
                TrainTFlabel_list.append(0)
            else:
                TrainTFlabel_list.append(1)
    with open("KDDTest+.csv") as f:
        reader = csv.reader(f)
        for row in reader:
            row[1] = sym["protocol_type"].index(row[1])
            row[2] = sym["service"].index(row[2])
            row[3] = sym["flag"].index(row[3])
            row[41] = sym["result"].index(row[41])
            Test_list.append(row[0:41])
            TestALLlabel_list.append(row[41])
            if row[41] == 0: # 0表示normal,其他表示为攻击流量
                TestTFlabel_list.append(0)
            else:
                TestTFlabel_list.append(1)
    x_train = np.array(Train_list)
    y_train = np.array(TrainTFlabel_list)
    z_train = np.array(TrainALLlabel_list)
    x_test = np.array(Test_list)
    y_test = np.array(TestTFlabel_list)
    z_test = np.array(TestALLlabel_list)
    np.savez_compressed("./NSL_KDD.npz", x_train=x_train, y_train=y_train, z_train=z_train, x_test=x_test,
                        y_test=y_test, z_test=z_test)

vector_data()


from .Tab import *
import wx.lib.agw.aui.dockart as da

class MainFrame(wx.Frame):

    def __init__(self, parent, title, tabart):#, dockart):
        super(MainFrame, self).__init__(parent, title=title, size=wx.Size(800, 600), pos=wx.Point(300, 400))
        self.basecolor = wx.SystemSettings.GetColour(wx.SYS_COLOUR_WINDOW)
        self.tabart = tabart
        self.dockart = da.AuiDefaultDockArt()
        self.dockart.SetMetric(aui.AUI_DOCKART_GRADIENT_TYPE, aui.AUI_GRADIENT_NONE)
        self.dockart.SetMetric(aui.AUI_DOCKART_SASH_SIZE, 3)
        self.dockart.SetMetric(aui.AUI_DOCKART_PANE_BORDER_SIZE, 0)
        self.dockart.SetColor(aui.AUI_DOCKART_SASH_COLOUR, wx.SystemSettings.GetColour(wx.SYS_COLOUR_BACKGROUND))
        self.dockart.SetColor(aui.AUI_DOCKART_HINT_WINDOW_COLOUR, wx.Colour(255,255,255))
        # self.dockart.SetColor(aui.AUI_DOCKART_BORDER_COLOUR, wx.Colour(0,0,0,0))
        self.initUI()

    def initUI(self):
        self._nb = JRLNotebook(self, agwStyle= aui.AUI_NB_DEFAULT_STYLE | aui.AUI_NB_CLOSE_ON_ALL_TABS | aui.AUI_NB_TAB_FLOAT | aui.AUI_NB_TAB_FIXED_WIDTH)
        self._nb.SetArtProvider(self.tabart)
        self._nb.GetAuiManager().SetArtProvider(self.dockart)

        btm = wx.Bitmap()
        btm.LoadFile("assets/close.png")
        plusButton_btm = wx.Bitmap()
        plusButton_btm.LoadFile("assets/tab_new.png")
        self._nb.AddPage(Tab(self._nb, wx.NewIdRef()), caption="Что-то очень длинное", select=False, bitmap=btm)
        self._nb.AddPage(Tab(self._nb, wx.NewIdRef()), caption="漢字できますか？", select=False)
        self._nb.AddPage(Tab(self._nb, wx.NewIdRef()), caption="ハイ、デキマス", select=True)
        self._nb.AddTabAreaButton(wx.ID_ANY, wx.CENTER, normal_bitmap=plusButton_btm, name="Add")
        self._nb.Bind(aui.EVT_AUINOTEBOOK_BUTTON, self.OnAddButton)
        self._nb.Bind(aui.EVT_AUINOTEBOOK_BG_MIDDLE_UP, self.OnAddButton)

    def OnExit(self,event):
        self.Close(True)  # Close the frame.

    def OnAddButton(self, event):
        if event.GetInt() == aui.AUI_BUTTON_CLOSE:
            event.Skip()
            return
        tctrl = self._nb.FindWindow(event.GetId())
        page = tctrl.GetPage(tctrl.GetActivePage())
        self._nb.SetSelectionToPage(page)
        self._nb.AddPage(Tab(self._nb, wx.NewIdRef()), "LOL", select=True)


import Skype4Py
import re
import threading
import logging

import oracle

logging.basicConfig(level=logging.INFO)

class GifBot(object):

    Modules = [
        oracle.Urban, oracle.Gif,
        oracle.GitList,
    ]

    def __init__(self):
        self.modules = [M() for M in self.Modules]

        # Create an instance of the Skype class.
        self.skype = Skype4Py.Skype(Events=self)

        # Connect the Skype object to the Skype client.
        self.skype.Attach()

        logging.info("*** Started!")

    def handle_message(self, text):
        match = re.match("@(?P<at>[a-z0-9]+)( (?P<text>.*))?",
                         text, re.IGNORECASE)
        if not match:
            return

        d = match.groupdict()
        at = d["at"]
        text = d.get("text", "")
        for M in self.modules:
            func = getattr(M, "at_{}".format(at), None)
            if callable(func):
                break
        else:
            return

        return func(text)

    def MessageStatus(self, msg, status):
        if status not in {Skype4Py.cmsReceived, Skype4Py.cmsSent}:
            return

        logging.info("[%s] %s: %s", msg.Chat.Topic,
                     msg.FromDisplayName, msg.Body)

        response = self.handle_message(msg.Body)
        if response:
            msg.Chat.SendMessage(response)

    def get_skype_thread(self):
        for t in threading.enumerate():
            if t.name == "Skype4Py API thread":
                return t
        else:
            raise LookupError("Where is the Skype thread?")

    def repl(self):
        while True:
            try:
                text = raw_input("> ")
                response = self.handle_message(text)
                if response:
                    print(response)
            except (KeyboardInterrupt, EOFError) as e:
                exit()

if __name__ == "__main__":
    bot = GifBot()
    bot.repl()


import io, sys
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf8')
sys.setrecursionlimit(100000)

num = int(input());

def recursion(n):
    if(n == 1 or n == 2):
        return 1
    return recursion(n-1) + recursion(n-2)

print(recursion(num) % 10009)


from Task_class import *
from Area_class import *

class CLE:

    def __init__(self,areas,temps_vecteur,entry,out):
        self.areas=areas
        self.temps_vecteur=temps_vecteur
        self.entry=entry
        self.out=out

    def apply_task(self,task):
        D=self.areas[task.d_area]
        F=self.areas[task.f_area]

        # Plutot mettre des raiseError

        if max(D.can_remove_car(task.car_type,1),F.can_add_car(task.car_type,1)):
            D.remove_car(task.car_type,1)
            F.add_car(task.car_type,1)

    def affichage_remplissage(self):
        for area_name in self.areas:
            print(area_name,self.areas[area_name].filling)


#!/usr/bin/env python
# -*- coding: UTF-8 -*-
import os

import sys
from lxml import etree

import requests

from tkinter import *

user_id = 3877667526

def write_to_html_file(html_content):
    file_dir = os.path.split(os.path.realpath(__file__))[0] + os.sep + "weibo"
    if not os.path.isdir(file_dir):
        os.mkdir(file_dir)
    file_path = file_dir + os.sep + "%d" % user_id + ".html"
    f = open(file_path, "wb")
    f.write(html_content.encode(sys.stdout.encoding))
    f.close()
    print(u"微博写入文件完毕，保存路径:")
    print(file_path)

class Weibo:
    cookie = {
        "Cookie": "_T_WM=be77132bf44cce11cd210cbe564e616b; WEIBOCN_FROM=1110006030; SSOLoginState=1534390759; ALF=1536982759; SCF=AnnzUkj-MBJbwZ5cWKMGT968UJIB8DukUnLz7dKqnfCyTt6Yd29cQm5cNggoUncqY12KNBAsqe6189rnsPLgzgY.; SUB=_2A252cIG3DeRhGeBI6FoW9CnLzj-IHXVVmi__rDV6PUNbktANLUfNkW1NRp-UfHnKswIua2nfxv1R3k9iBwNCNuq4; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9W5JxPr-4Ynnija8BNTMMVgj5JpX5KMhUgL.Foqce0nNShMNSKe2dJLoIp7LxKML1KBLBKnLxKqL1hnLBoMcSoeRS0BNS0-0; SUHB=0wn1cPVBNulhYJ; MLOGIN=1; M_WEIBOCN_PARAMS=fid%3D1005055030120112%26uicode%3D10000011"}  # 将your cookie替换成自己的cookie

    def __init__(self, profile_ftype, is_ori):
        self.user_id = user_id
        self.profile_ftype = profile_ftype
        self.username = ''
        self.weibo_content = []
        self.publish_time = []
        self.publish_tool = []

    def get_username(self):
        url = "https://weibo.com/u/%d?profile_ftype=1&is_ori=1#_0" % self.user_id
        html = requests.get(url, cookies=self.cookie)
        content = html.content
        result = etree.HTML(content)
        content_html = etree.tostring(result).decode('utf-8')
        # print(content_html)
        write_to_html_file(content_html)

def main():
    user_id = 3877667526
    profile_ftype = 1
    is_ori = 1
    wb = Weibo(profile_ftype, is_ori)
    wb.get_username()

if __name__ == '__main__':
    main()


#!/usr/bin/env python3

from __future__ import print_function

import fasm
import argparse
import json
import os
import os.path

from prjxray import fasm_assembler
from prjxray.db import Database
from prjxray.roi import Roi

class FASMSyntaxError(SyntaxError):
    pass

def dump_frames_verbose(frames):
    print()
    print("Frames: %d" % len(frames))
    for addr in sorted(frames.keys()):
        words = frames[addr]
        print(
            '0x%08X ' % addr + ', '.join(['0x%08X' % w for w in words]) +
            '...')

def dump_frames_sparse(frames):
    print()
    print("Frames: %d" % len(frames))
    for addr in sorted(frames.keys()):
        words = frames[addr]

        # Skip frames without filled words
        for w in words:
            if w:
                break
        else:
            continue

        print('Frame @ 0x%08X' % addr)
        for i, w in enumerate(words):
            if w:
                print('  % 3d: 0x%08X' % (i, w))

def dump_frm(f, frames):
    '''Write a .frm file given a list of frames, each containing a list of 101 32 bit words'''
    for addr in sorted(frames.keys()):
        words = frames[addr]
        f.write(
            '0x%08X ' % addr + ','.join(['0x%08X' % w for w in words]) + '\n')

def find_pudc_b(db):
    """ Find PUDC_B pin func in grid, and return the tile and site prefix.

    The PUDC_B pin is a special 7-series pin that controls unused pin pullup.

    If the PUDC_B is unused, it is configured as an input with a PULLUP.

    """
    grid = db.grid()

    pudc_b_tile_site = None
    for tile in grid.tiles():
        gridinfo = grid.gridinfo_at_tilename(tile)

        for site, pin_function in gridinfo.pin_functions.items():
            if 'PUDC_B' in pin_function:
                assert pudc_b_tile_site == None, (
                    pudc_b_tile_site, (tile, site))
                iob_y = int(site[-1]) % 2

                pudc_b_tile_site = (tile, 'IOB_Y{}'.format(iob_y))

    return pudc_b_tile_site

def run(
        db_root,
        filename_in,
        f_out,
        sparse=False,
        roi=None,
        debug=False,
        emit_pudc_b_pullup=False):
    db = Database(db_root)
    assembler = fasm_assembler.FasmAssembler(db)

    if emit_pudc_b_pullup:
        pudc_b_in_use = False
        pudc_b_tile_site = find_pudc_b(db)

        def check_for_pudc_b(set_feature):
            parts = set_feature.feature.split('.')

            if parts[0] == pudc_b_tile_site[0] and parts[
                    1] == pudc_b_tile_site[1]:
                nonlocal pudc_b_in_use
                pudc_b_in_use = True

        if pudc_b_tile_site is not None:
            assembler.set_feature_callback(check_for_pudc_b)

    extra_features = []
    if roi:
        with open(roi) as f:
            roi_j = json.load(f)
        x1 = roi_j['info']['GRID_X_MIN']
        x2 = roi_j['info']['GRID_X_MAX']
        y1 = roi_j['info']['GRID_Y_MIN']
        y2 = roi_j['info']['GRID_Y_MAX']

        assembler.mark_roi_frames(Roi(db=db, x1=x1, x2=x2, y1=y1, y2=y2))

        if 'required_features' in roi_j:
            extra_features = fasm.parse_fasm_string(
                '\n'.join(roi_j['required_features']))

    assembler.parse_fasm_filename(filename_in, extra_features=extra_features)

    if emit_pudc_b_pullup and not pudc_b_in_use and pudc_b_tile_site is not None:
        # Enable IN-only and PULLUP on PUDC_B IOB.
        #
        # TODO: The following FASM string only works on Artix 50T and Zynq 10
        # fabrics.  It is known to be wrong for the K70T fabric, but it is
        # unclear how to know which IOSTANDARD to use.
        missing_features = []
        for line in fasm.parse_fasm_string("""
{tile}.{site}.LVCMOS12_LVCMOS15_LVCMOS18_LVCMOS25_LVCMOS33_LVTTL_SSTL135.IN_ONLY
{tile}.{site}.LVCMOS25_LVCMOS33_LVTTL.IN
{tile}.{site}.PULLTYPE.PULLUP
""".format(
                tile=pudc_b_tile_site[0],
                site=pudc_b_tile_site[1],
        )):
            assembler.add_fasm_line(line, missing_features)

        if missing_features:
            raise fasm_assembler.FasmLookupError('\n'.join(missing_features))

    frames = assembler.get_frames(sparse=sparse)

    if debug:
        dump_frames_sparse(frames)

    dump_frm(f_out, frames)

def main():
    parser = argparse.ArgumentParser(
        description=
        'Convert FPGA configuration description ("FPGA assembly") into binary frame equivalent'
    )

    database_dir = os.getenv("XRAY_DATABASE_DIR")
    database = os.getenv("XRAY_DATABASE")
    db_root_kwargs = {}
    if database_dir is None or database is None:
        db_root_kwargs['required'] = True
    else:
        db_root_kwargs['required'] = False
        db_root_kwargs['default'] = os.path.join(database_dir, database)

    parser.add_argument('--db-root', help="Database root.", **db_root_kwargs)
    parser.add_argument(
        '--sparse', action='store_true', help="Don't zero fill all frames")
    parser.add_argument(
        '--roi',
        help="ROI design.json file defining which tiles are within the ROI.")
    parser.add_argument(
        '--emit_pudc_b_pullup',
        help="Emit an IBUF and PULLUP on the PUDC_B pin if unused",
        action='store_true')
    parser.add_argument(
        '--debug', action='store_true', help="Print debug dump")
    parser.add_argument('fn_in', help='Input FPGA assembly (.fasm) file')
    parser.add_argument(
        'fn_out',
        default='/dev/stdout',
        nargs='?',
        help='Output FPGA frame (.frm) file')

    args = parser.parse_args()
    run(
        db_root=args.db_root,
        filename_in=args.fn_in,
        f_out=open(args.fn_out, 'w'),
        sparse=args.sparse,
        roi=args.roi,
        debug=args.debug,
        emit_pudc_b_pullup=args.emit_pudc_b_pullup)

if __name__ == '__main__':
    main()


import pandas as pd
import re
from decimal import Decimal, ROUND_HALF_UP
from elasticsearch import Elasticsearch
from elasticsearch import helpers

# def parse_league_field(league_str):
#     l = league_str.split(',')
#     l = [x.lstrip() for x in l]
#     return l
from utils.Const import FIFALABEL

def parse_league_s(l):
    l.update(l.str.split(','))
    s = []
    for i in l.values:
        for index, item in enumerate(i):
            i[index] = item.lstrip()
        i.pop(0)
        s.append(i.pop(1))
    l.update(l.astype(str))
    l.update(l.str.lstrip('[\'').str.rstrip('\']'))
    return s

def add_s_to_df(d, l, name, index):
    d.insert(index, name, l)

def parse_workrate(s):
    defense = s.str.findall(r'/([A-z]+)$')
    defense.update(defense.apply(s_from_one_elem_l))
    s.update(s.str.findall(r'([A-z]+)/'))
    s.update(s.apply(s_from_one_elem_l))
    return defense

def parse_rating_position_s(s):
    r = s.str.findall(r'[0-9]+')
    r.update(r.apply(int_from_one_elem_l))
    s.update(s.str.findall(r'[A-Z]{2,3}'))
    s.update(s.apply(s_from_one_elem_l))
    return r

def s_from_one_elem_l(l):
    l = str(l[0])
    return l

def int_from_one_elem_l(l):
    l = int(l[0])
    return l

def replace_height_s(s):
    s.update(s.apply(height_int))

def height_int(s):
    return int(re.findall(r'\((\d+) cm\)', s)[0])

def replace_weight_s(s):
    s.update(s.apply(weight_int))

def weight_int(s):
    return int(re.findall(r'^[0-9]+', s)[0])

def replace_foot_s(s):
    s.replace(to_replace={'Left': 'l', 'Right': 'r'}, inplace=True)

def replace_weak_f_s(s):
    s.update(s.apply(weak_int))

def weak_int(s):
    return int(re.findall(r'\((\d)\)', s)[0])

def count_l_foot(s):
    return s.loc[s == 'Left'].count()

def get_filtered(df, fields, filters):
    """
    take 1-d array(n) with fields and matrix(n1) with values for that fields row by row putting into the 1-d array
    give a suitable part of df
    принимает массив полей и матрицу фильтров, где каждая строка матрицы фильтров это возможные значения
    для соответствующего поля
    :param df:
    :param fields: filtering will be by these fields
    :param filters: matrix: in each rows there are possible values for fields
    :return: suitable rows of df
    """
    map = []
    for i in df[fields].iterrows():
        row = []
        for index, j in enumerate(filters):
            row.append(i[1][index] in j)
        map.append(False not in row)
    return df.loc[map]

def __filter_keys__(d, k):
    return {key: d[key] for key in k}

def doc_generator(df):
    print('Exporting rows in es started...')
    df_iter = df.iterrows()
    keys = df.columns.values
    for i, d in df_iter:
        yield {
            "type": "players",
            "_index": "fifaportal",
            "_id": f"{d['ID']}",
            "_source": __filter_keys__(d, keys),
        }
    print('Exporting rows in es finished...')
    # raise StopIteration

def safe_value(field_val):
    return field_val if not pd.isna(field_val) else 0

def rename_c(df, o_c_name, n_c_name):
    df.update(df.rename(columns={o_c_name: n_c_name}, inplace=True))

fifa_data_set = pd.read_csv('../data/fifa_players_data.csv', engine='c', chunksize=1000)
# fifa_data = pd.read_csv('test.csv', engine='c')
es_client = Elasticsearch(http_compress=True)
#es_client.indices.delete(index=FIFALABEL)

fifa_ind_body = {
    "settings": {
        "analysis": {
            # put char_filter here if need
            "tokenizer": "standard",
            "filter": {
                "stopwords_removal": {
                    "type": "stop",
                    "stopwords": [
                        "_brazilian_",
                        "_english_",
                        "_german_",
                        "_portuguese_",
                        "_spanish_",
                        "_turkish_"
                    ]
                }
                # "hunspell_": {
                #             "type": "hunspell",
                #             "locale": "en_US"
                #         },
            },
            "analyzer": {
                "custom_analyzer_for_key_text_fields": {
                    "type": "custom",
                    "tokenizer": "whitespace",
                    "filter": [
                        "lowercase",
                        "asciifolding",
                        "stopwords_removal"
                    ]
                },
                "custom_analyzer_for_nations_fields": {
                    "type": "custom",
                    "tokenizer": "whitespace",
                    "filter": [
                        "lowercase",
                        "asciifolding",
                        # "hunspell_",
                        "stemmer",  # can be removed, just for example
                        "stop"
                    ]
                }
            },
            "normalizer": {
                "key_normalizer": {
                    "type": "custom",
                    "char_filter":
                        {
                            "type": "mapping",
                            "mappings": [
                                "left => l",
                                "right => r"
                            ]
                        }
                    ,
                    "filter": ["lowercase"]
                }
            }
        }
    },
}
es_client.indices.create(index=FIFALABEL, body=fifa_ind_body)

int_key_fields = ['Weight', 'Rating', 'Height']  # add more in future
int_fields = ['Weak Foot', 'PACE', 'Acceleration',
              'Shot Power', 'Long Shot', 'Volleys', 'Penalties', 'PASSING', 'Vision',
              'Free Kick', 'Short Passing', 'Long Passing', 'Agility', 'Balance',
              'Ball Control', 'Dribbling', 'Interceptions', 'Heading', 'PHYSICAL',
              'Jumping', 'Strength', 'Finishing']

key_fields = ['ID', 'Position', 'Foot', 'Work in ATT', 'Work in DEF']

key_unsearchable_fields = ['Celebration', 'URL', 'Image_URL']

key_text_fields_not_nation = ['Name', 'Club', 'League']

key_text_fields_nation = ['Nation', 'National team']

mapping_key_prop = {
    x: {
        "type": "keyword",
        "normalizer": "key_normalizer"
    } for x in key_fields
}

mapping_key_unsearchable_prop = {
    x: {
        "type": "keyword",
        "index": "false"
    } for x in key_unsearchable_fields
}

mapping_key_text_prop_not_nation = {
    x: {
        "type": "text",
        "analyzer": "custom_analyzer_for_key_text_fields",
        "norms": "false",  # for optimization and connected with project specific
        "fields": {
            "keyword": {
                "type": "keyword"
            }
        }
    } for x in key_text_fields_not_nation
}

mapping_key_text_prop_nation = {
    x: {
        "type": "text",
        "analyzer": "custom_analyzer_for_nations_fields",
        "norms": "false",  # for optimization and connected with project specific
        "fields": {
            "keyword": {
                "type": "keyword"
            }
        }
    } for x in key_text_fields_nation
}

mapping_int_key_prop = {
    x: {
        "type": "short",
        "coerce": 'false',
        # "ignore_malformed": 'true',
        "fields": {
            "keyword": {
                "type": "keyword"
            }
        }
    } for x in int_key_fields
}

mapping_int_prop = {
    x: {
        "type": "short",
        "coerce": 'false',  # Try to convert strings to numbers and truncate fractions for integers. Accepts true (
        # default) and false. Not applicable for unsigned_long. "ignore_malformed": 'true',
    }
    for x in int_fields
}

mapping_for_prop = {}
mapping_for_prop.update(**mapping_int_key_prop, **mapping_int_prop, **mapping_key_prop,
                        **mapping_key_text_prop_not_nation, **mapping_key_text_prop_nation,
                        **mapping_key_unsearchable_prop)

es_client.indices.put_mapping(
    index=FIFALABEL,
    body={
        "properties":
            mapping_for_prop
    }
)

for fifa_data in fifa_data_set:
    # delete useless columns
    fifa_data = fifa_data[
        list(filter(lambda x: x not in ['Skillboost', 'Source', 'Sprint Speed', 'Crossing', 'Curve', 'AGILITY',
                                        'Reactions', 'DEFENDING', 'Marking', 'Stand Tackle', 'Sliding Tackle',
                                        'Aggression', 'SHOOTING', 'Positioning', 'KICKING', 'DIVING', 'POSITIONING',
                                        'REFLEXES', 'Reflexes', 'GK Diving', 'GK Kicking', 'GK Positioning',
                                        'HANDLING', 'Handling', 'Special Trait', 'Swipe Skill Move', 'Tap Skill Move'],
                    fifa_data.columns))]

    # print('Amount of rows with rows with nan: ', fifa_data['ID'].count())
    fifa_data.dropna(inplace=True)
    print('Amount of rows to import from current chunk: ', fifa_data['ID'].count())

    replace_height_s(fifa_data['Height'])
    add_s_to_df(fifa_data, parse_rating_position_s(fifa_data['Position_Rating']), 'Rating', 5)
    rename_c(fifa_data, 'Position_Rating', 'Position')
    add_s_to_df(fifa_data, parse_workrate(fifa_data['Workrates (ATT/DEF)']), 'Work in DEF', 12)
    rename_c(fifa_data, 'Workrates (ATT/DEF)', 'Work in ATT')
    add_s_to_df(fifa_data, parse_league_s(fifa_data['League']), 'National team', 4)
    replace_foot_s(fifa_data['Foot'])
    replace_weak_f_s(fifa_data['Weak Foot'])
    replace_weight_s(fifa_data['Weight'])
    # print(fifa_data['Work in ATT'])
    # print(fifa_data['Work in DEF'])

    fifa_data[int_fields + int_key_fields] = fifa_data[int_fields + int_key_fields].astype(int)

    # print(fifa_data.dtypes)

    # for i in fifa_data.columns.values:
    #     fifa_data[i] = fifa_data[i].apply(safe_value)

    # to es

    helpers.bulk(es_client, doc_generator(fifa_data))

# какое количество левоногих в каждой лиге

# amount of left-foot players in each league
# df_eng_pr_l = get_filtered(fifa_da иta, ['League'], [['England Premier League']])
# print('Процент левшей в Английской Премьер Лиге:', Decimal(count_l_foot(df_eng_pr_l['Foot']) /
#                                                       df_eng_pr_l['Foot'].count() * 100).quantize(Decimal('0.01'),
#                                                                                rounding=
#                                                                                ROUND_HALF_UP))
# # amount of each nationality in each league
# print('Процент французов в Английской Премьер Лиге:', Decimal(get_filtered(df_eng_pr_l, ['Nation'],
#                                                                            [['France']])['Nation'].count() /
#                                                               df_eng_pr_l['Nation'].count() * 100).quantize(
#                                                                                Decimal('0.01'),
#                                                                                rounding=
#                                                                                ROUND_HALF_UP))


from __future__ import division

import json
import time
import math
import file_handler

class InvertedIndex():

	def __init__(self, trainset):
		self.table = {}
		self.tokens = []
		self.documents = trainset

		start = time.time()

		for fileid in trainset:
			file = file_handler.readfile(fileid)
			processed = file_handler.preprocess(file)

			for token in processed:

				if token not in self.table:
					self.table[token] = dict()
					self.tokens.append(token)

				token_data = self.table[token]
				if fileid not in token_data:
					token_data[fileid] = 1
				else:
					token_data[fileid] += 1

		finish = time.time()
		elapsed = finish - start
		#print "Built inverted index from "+str(len(self.documents))+" documents in "+str(elapsed)+" seconds."

	def __str__(self):
		return json.dumps(self.table, indent=1)

	def __repr__(self):
		return json.dumps(self.table, indent=1)

	def tf_idf(self, idset=[]):
		if len(idset) == 0:
			start = time.time()
			matrix = [[0 for x in range(len(self.tokens))] for y in range(len(self.documents))]
			classes = []

			step = 100.0/len(self.documents)
			for i in range(0, len(self.documents)):
				for j in range(0,len(self.tokens)+1):
					if j == len(self.tokens):
						if self.documents[i][1] == 'j':
							classes.append(0)
						else:
							classes.append(1)
					else:
						tf = self._tf(self.tokens[j], self.documents[i])
						idf = self._idf(self.tokens[j])

						matrix[i][j] = tf*idf

				val = i*step
				#print ""+str(val)+"% complete..."

			finish = time.time()
			elapsed = finish - start
			#print "Built tf-idf matrix in "+str(elapsed)+" seconds."

			return matrix, classes

		else:
			start = time.time()
			matrix = [[0 for x in range(len(self.tokens))] for y in range(len(idset))]

			step = 100.0/len(idset)
			for i in range(0, len(idset)):
				file = file_handler.readfile(idset[i])
				processed = file_handler.preprocess(file)

				for j in range(0,len(self.tokens)):
					tf = self._tf(self.tokens[j], processed)
					idf = self._idf(self.tokens[j])

					matrix[i][j] = tf*idf

				val = i*step
				#print ""+str(val)+"% complete..."

			finish = time.time()
			elapsed = finish - start
			#print "Built tf-idf matrix in "+str(elapsed)+" seconds."

			return matrix

	def _tf(self, term, document):
		if isinstance(document, list):
			if term in document:
				frequency = 0
				for token in document:
					if token == term:
						frequency += 1
				return 1 + math.log10(frequency)
			else:
				return 0
		else:
			if document in self.table[term]:
				frequency = self.table[term][document]
				return 1 + math.log10(frequency)
			else:
				return 0

	def _idf(self, term):
		N = len(self.tokens)
		document_frequency = len(self.table[term].keys())
		return math.log10(N/1 + document_frequency)

	def trim(self, ratio=0.5):
		for token in self.tokens:
			docs = self.table[token]
			js, gs = 0, 0
			for doc in docs:
				if doc[1] == 'j':
					js += 1
				else:
					gs += 1
			major = gs if gs > js else js
			fraction = major / (gs+js)
			if len(docs) == 1 or fraction <= ratio:
				del self.table[token]

		self.tokens = self.table.keys()


static = ARGUMENTS.get('static', 0)

ccflags="-Ofast"
if int(static):
    ccflags="-Ofast -static"

txt = Program("mandeltext", "main.cpp", CCFLAGS=ccflags)
Program("mandelcurses", "main_pd.cpp", LIBS="pdcurses", CCFLAGS=ccflags)
ppm = Program("mandelppm", "main_ppm.cpp", CCFLAGS=ccflags)

Clean(txt, Split("out.txt"))
Clean(ppm, Split("mandelbrot.ppm"))


class Kreciolek():
    def __init__(self, x, y, segmenty):
        self.x = x
        self.y = y
        self.segmenty = segmenty
        self.niebieski = 100
        self.kierunek = True
        self.var = (2*PI)/self.segmenty
        self.start = 0
        self.stop = self.var

    def sketch(self):
        pushMatrix()
        strokeWeight(10)
        fill(50)
        self.start = (self.start + self.var) % (2*PI)
        self.stop = (self.stop + self.var) % (2*PI)
        self.niebieski += self.kierunek
        stroke(0, 0, self.niebieski)
        arc(self.x, self.y, 150, 150, self.start, self.stop, OPEN)
        if self.niebieski > 200 or self.niebieski < 50: self.kierunek *= -1
        popMatrix()

def setup():
    size(1200, 1200)
    background(0)
    global pierwszy, drugi, trzeci
    pierwszy = Kreciolek(200, 200, 100)
    drugi = Kreciolek(500, 500, 150)
    trzeci = Kreciolek(500, 200, 500)

def draw():
    pierwszy.sketch()
    drugi.sketch()
    trzeci.sketch()

#2pkt


# configuring your own logging using log handlers
# -----------------------------------------------
from ats import aetest
from ats.topology import loader
import logging

from ats.log import ScreenHandler, TaskLogHandler

# create a logger
logger = logging.getLogger(__name__)

# create a handler
screen_handler = ScreenHandler()
tasklog_handler = TaskLogHandler('log_file.txt')

# attach to your logger
logger.addHandler(screen_handler)
logger.addHandler(tasklog_handler)

# set log level to show everything
logger.setLevel(logging.DEBUG)
testbed = loader.load("testbed_1.yaml")

class TestcaseOne(aetest.Testcase):
    server = None

    @aetest.setup
    def setup(self):
        self.server = testbed.devices['vm-a']
        self.server.connect()
        logger.warning("Connected to server")

    @aetest.test
    def test_one(self):
        logger.critical('Check the results')
        virtual_machine_name = self.server.execute('hostname')
        assert len(virtual_machine_name) == 'Hello'

    @aetest.cleanup
    def cleanup(self):
        logger.error('Close connection to Server')

if __name__ == '__main__':
    aetest.main()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('herders', '0007_monster_archetype'),
    ]

    operations = [
        migrations.RemoveField(
            model_name='monster',
            name='awakens_to',
        ),
        migrations.AddField(
            model_name='monster',
            name='awakens_from',
            field=models.ForeignKey(to='herders.Monster', null=True),
        ),
    ]


# -*- encoding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>). All Rights Reserved
#    $Id$
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from osv import fields, osv

class task(osv.osv):
    _name = 'project.task'
    _inherit = 'project.task'

    def default_user_id(self, cr, uid, context={}):
        return False

    def default_task_stage_ids(self, cr, uid, context={}):
        obj_task_type = self.pool.get('project.task.type')
        result = []

        kriteria = [('state','=','open')]
        task_type_ids = obj_task_type.search(cr, uid, kriteria)

        if task_type_ids:
            for task_type in obj_task_type.browse(cr, uid, task_type_ids):
                val =   {
                        'stage_id' : task_type.id,
                        }
                result.append(val)

        return result

    _columns =  {
                'task_stage_ids' : fields.one2many(string='Task Stage', obj='project.task_stage', fields_id='task_id'),
                }

    _defaults = {
                'user_id' : default_user_id,
                'task_stage_ids' : default_task_stage_ids,
                }

    def write(self, cr, uid, ids, value, context=None):
        obj_task_stage = self.pool.get('project.task_stage')

        if value.get('stage_id', False):
            kriteria = [('stage_id','=',value['stage_id']),('task_id','=',ids[0])]
            task_stage_ids = obj_task_stage.search(cr, uid, kriteria)
            if task_stage_ids:
                task_stage = obj_task_stage.browse(cr, uid, task_stage_ids)[0]
                value.update({'user_id' : task_stage.user_id.id})
            else:
                value.update({'user_id' : False})

        return super(task, self).write(cr, uid, ids, value, context)

task()


"""
the pex formula is for generating a pex python executable. It's intended to
replace the existing eggscript formula, which has had issues with it's virtualenv.

https://github.com/pantsbuild/pex

[sprinter]
eggs = sprinter
entry_point = sprinter.install:main
executable_name = sprinter
"""
from __future__ import unicode_literals

import os
from sprinter.formula.base import FormulaBase
from pex.interpreter import PythonInterpreter
from pex.pex_builder import PEXBuilder
from pex.resolvable import Resolvable
from pex.resolver import Resolver
from pex.resolver_options import ResolverOptionsBuilder
from pex.bin.pex import interpreter_from_options, configure_clp

class PexFormula(FormulaBase):

    required_options = FormulaBase.required_options + [
        "eggs", "entry_point", "executable_name"
    ]

    def install(self):
        self._build()

    def update(self):
        self._build()

    @property
    def install_directory(self):
        return self.directory.install_directory(self.feature_name)

    def _build(self):
        target_path = os.path.join(self.install_directory,
                                   self.target.get("executable_name"))
        parser, resolver_options_builder = configure_clp()
        pex_builder = PEXBuilder(
            interpreter=self._get_interpreter(parser)
        )
        pex_builder.set_entry_point(self.target.get("entry_point"))
        for dist in self._get_distributions(resolver_options_builder):
            pex_builder.add_distribution(dist)
            pex_builder.add_requirement(dist.as_requirement())
        pex_builder.build(target_path)
        self.directory.symlink_to_bin(self.target.get("executable_name"),
                                      target_path)

    def _get_distributions(self, resolver_options_builder):
        eggs = self.target.get("eggs").splitlines()
        resolvables = [Resolvable.get(e, resolver_options_builder) for e in eggs]
        resolver = Resolver()
        return resolver.resolve(resolvables)

    def _get_interpreter(self, parser):
        options, reqs = parser.parse_args(args=[
            "-e", self.target.get("executable_name"),
            "--wheel",
        ])
        return interpreter_from_options(options)


import os
import torch
import numpy as np
from model import CNN
from dataset import DatasetFolder, npy_loader

from torch.nn import CrossEntropyLoss
import numpy as np
from argparse import ArgumentParser

def build_argparser():

    parser = ArgumentParser()

    parser.add_argument("--data_path", help="path to dataset *.npy", required=True, type=str)
    parser.add_argument("--save_model_path", help="path to saved model", default=None, type=str)
    parser.add_argument("--test_batch_size", help="num of testing batch size", default=1024, type=int)
    parser.add_argument("--val_ratio", help="validation data size to training data size ratio", default=0.2, type=float)
    parser.add_argument("--random_state", help="num of random_state", default=777, type=int)

    return parser

if __name__ == "__main__":

    print("+++ SignalNet Initialized +++")

    #data_path = "../../data/MAFAULDA_XX"
    #save_model_path = "../../data/SignalNet_demo.pth"
    #test_batch_size = 1024
    #val_ratio = 0.2

    args = build_argparser().parse_args()

    data_path = args.data_path
    save_model_path = args.save_model_path
    test_batch_size = args.test_batch_size
    val_ratio = args.val_ratio

    assert os.path.exists(data_path)
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

    #random_state = 777
    random_state = args.random_state
    torch.manual_seed(random_state)
    np.random.seed(random_state)
    torch.backends.cudnn.deterministic = True

    print("+++ Load Dataset +++")
    dataset = DatasetFolder(
              root=data_path,
              loader=npy_loader,
              extensions='.npy'
              )
    train_set, test_set = torch.utils.data.random_split(dataset, [len(dataset) - int(len(dataset)*val_ratio), int(len(dataset)*val_ratio)])

    test_loader = torch.utils.data.DataLoader(test_set, batch_size=test_batch_size, shuffle=False)

    print("+++ Build Model +++")
    cnn = CNN().to(device)
    cnn.load_state_dict(torch.load(save_model_path, device))
    total_step = len(test_loader)
    #print(total_step)
    #loss_list_test = []
    acc_list_test = []

    cnn.eval()
    criterion = CrossEntropyLoss()
    results = {}

    with torch.no_grad():
        for i, (signals, labels) in enumerate(test_loader):
            # Run the forward pass
            signals=signals.float().to(device)
            labels=labels.float().to(device)
            outputs = cnn(signals)
            loss = criterion(outputs, labels.long())
            #loss_list_test.append(loss.item())
            #if epoch%10 ==0:
            #    print(loss)
            total = labels.size(0)
            _, predicted = torch.max(outputs.data, 1)
            correct = (predicted == labels.long()).sum().item()
            acc_list_test.append(correct / total)

            labels_arr = labels.detach().cpu().numpy()
            predict_arr = predicted.detach().cpu().numpy()

            for pred, true in zip(predict_arr, labels_arr):
                if true not in results:
                    results[true] = {"n_total": 1}
                    if pred == true:
                        results[true]["n_correct"] = 1
                    else:
                        results[true]["n_correct"] = 0
                else:
                    results[true]["n_total"] += 1
                    if pred == true:
                        results[true]["n_correct"] += 1

            #print('Loss: {:.4f}, Accuracy: {:.2f}%'
            #      .format(loss.item(), (correct / total) * 100))

    n_total_correct = 0
    n_total_counts = 0
    for key, value in results.items():
        print("Lable: {}, Accuracy: {:.2f} %, Counts: {}".format(key, results[key]["n_correct"]/results[key]["n_total"]*100, results[key]["n_total"]))
        n_total_correct += results[key]["n_correct"]
        n_total_counts += results[key]["n_total"]
    print("Total Accuracy: {:.2f} %".format(n_total_correct/n_total_counts*100))


# routine to calculate the nth cenral moment of an input distribution

import numpy as np

def mystatmoments(xdist,ydist,x0,nth):

 a = xdist - x0
 ndist = xdist.shape[0]
 dx = (xdist[-1] - xdist[0])/(ndist-1)

 if (nth == 2):
  sum = np.sum(ydist*a*a)
 elif (nth == 3):
  aa = a*a
  sum = np.sum(ydist*aa*a)
 elif (nth ==4):
  aa= a*a
  sum = np.sum(ydist*aa*aa)

 return(sum*dx)

##!!!!!! test here
#x = np.arange(0,100,0.000001)
#
#x0 = 15.0
#sig = 2.0
#
#a = 1/(np.sqrt(2*np.pi*sig*sig))
#
#y = a * np.exp(-0.5*(x-x0)**2/(sig*sig))
#
#nth = 4
#op = mystatmoments(x,y,x0,nth)
#
#print op, x.shape[0]


import numpy as np
import fractions as fr
import itertools as it
import prettytable as pt

# Функции для ввода и вывода матрицы
def fr2str(f):
    return str(f).center(7)

def print_array(a, name = ''):
    s = ' ' + np.array2string(a, formatter = {'all':fr2str})[1:-1]
    if name != '':
        w = len(s)//a.shape[0]
        s += '\n{}'.format(name.center(w))
    print(s)

def input_array(n, m, message = ""):
    print(message, end = "")
    s = []
    while len(s) < m*n:
        s += input().replace(',', '.').split()
    l = list(map(fr.Fraction, s))
    return np.array(l).reshape((n,m))

def swap_rows(matr, n1, n2):
    matr[[n1,n2]] = matr[[n2,n1]]

def swap_columns(matr, m1, m2):
    matr[:,[m1,m2]] = matr[:,[m2,m1]]

def mult_row(matr, n, x):
    '''Умножить n-ю строку матрицы на x'''
    matr[n] *= x

def add_row(matr, n1, n2, x = 1):
    '''Умножить строку n1 матрицы на x и прибавить к n2 строке матрицы'''
    matr[n2] += matr[n1]*x

def gauss_step(matr, lead_i, lead_j):
    # Делим строку на разрешающий элемент (Если он не нулевой)
    if matr[lead_i, lead_j] == 0:
        return False
    else:
        mult_row(matr, lead_i, 1/matr[lead_i, lead_j])
        # К каждой из строк добавляем ведущую строку, умноженную на коэффициент
        for i in range(0, matr.shape[0]):
            if i != lead_i:
                mul = -matr[i, lead_j]
                add_row(matr, lead_i, i, mul)
        return True

def gauss(matr):
    n, m = matr.shape
    # Количество строк, вошедших в треугольную матрицу
    rows_eliminated = 0
    while rows_eliminated < n:
        # lead_i, lead_j - строка и столбец с ведущим элементом
        lead_j = rows_eliminated
        lead_i = rows_eliminated
        # Ищем строку/столбец с ненулевым разрешающим элементом
        while lead_j < m and matr[lead_i, lead_j] == 0:
            if lead_i < n-1:
                lead_i += 1
            else:
                lead_i = rows_eliminated
                lead_j += 1
        if lead_j == m:
            # Не найден ненулевой элемент, т.е., получили нулевую строку. Удалим все строки ниже
            for i in range(rows_eliminated, n):
                matr = np.delete(matr, i, axis=0)
            break
        if lead_i > rows_eliminated:
            swap_rows(matr, rows_eliminated, lead_i)
            lead_i = rows_eliminated
        gauss_step(matr, lead_i, lead_j)
        rows_eliminated += 1
    return matr

def can_replace(matr, j1, j2):
    i = 0
    while i < matr.shape[0] and matr[i, j1] == 0:
        i += 1
    return matr[i, j2] != 0

def replace(matr, j1, j2):
    '''Производит операцию замещения j1-й базисной переменной и \
        j2-й свободной переменной. Если операция замещения невозможна,
        т.е., разрешающий элемент равен 0, возвращает False'''
    if can_replace(matr, j1, j2):
        i = 0
        while i < matr.shape[0] and matr[i, j1] == 0:
            i += 1
        gauss_step(matr, i, j2)

n, m = map(int, input("Сколько в системе уравнений и переменных?\n").split())
ar = input_array(n, m)
print_array(ar, 'Матрица системы ограничений: \n')
k = int(input("Количество коэффициентов в целевой функции? "))
koef = list(map(fr.Fraction,
            input("Введите коэффициенты целевой функции x{}..x{} -> max\n".format(1, k)).split()))
# Если количество коэффициентов меньше, чем количество переменных,
# то остальные коэффициенты сочтем нулями
while len(koef) < m:
    koef.append(fr.Fraction(0))
s = "z = "
for i in range(len(koef)-1):
    s += "{}X{}+".format(str(koef[i]), i+1)
s += "{}X{}->max".format(str(koef[-1]), 5)
print("Целевая функция:", s, "\n")

print_array(otvet[0], "Оптимальный план, полученный перебором\n")
print("Значение целевой функции {}.\n".format(otvet[1]))


f = open('1.1.2_value.hex', 'r')
file_content = f.read().strip()
integer_parsed = int(file_content, 16)
binary_string = bin(integer_parsed)[2:]
b = str(binary_string)
f.close()

f = open('sol_1.1.2_binary.txt', 'w')
f.write(b)
f.close()