# -*- coding: utf-8 -*-
'''聊天相关服务: 聊天服<--->客户端
'''
from chatService.defines import *
import terminal_chat_pb2
import endPoint

class cService(terminal_chat_pb2.terminal2chat):

	@endPoint.result
	def rpcChatUp(self, ep, senderObj, reqMsg):return rpcChatUp(ep, senderObj, reqMsg)

	@endPoint.result
	def rpcBanChannelReq(self, ep, senderObj, reqMsg):return rpcBanChannelReq(ep, senderObj, reqMsg)

	@endPoint.result
	def rpcBanChannelSet(self, ep, senderObj, reqMsg):return rpcBanChannelSet(ep, senderObj, reqMsg)

	# @endPoint.result
	# def rpcAudioReq(self, ep, senderObj, reqMsg):return rpcAudioReq(ep, senderObj, reqMsg)

	@endPoint.result
	def rpcHeartbeat(self, ep, senderObj, reqMsg):return rpcHeartbeat(ep, senderObj, reqMsg)

def rpcHeartbeat(ep, senderObj, reqMsg):
	return True

def rpcChatUp(ep, senderObj, reqMsg):
	'''发送聊天信息
	'''
	channelId = reqMsg.channelId
	content = reqMsg.content
	isAudio = reqMsg.isAudio
	if not content:
		message.tips(senderObj, "请输入要说的话")
		return
	if channelId == CHANNEL_TEAM and not senderObj.teamId:
		message.tips(senderObj, "你没加入任何队伍，请加入后再发言")
		return
	if channelId == CHANNEL_GUILD:
		if not senderObj.guildId:
			message.tips(senderObj, "你没加入任何仙盟，请加入后再发言")
			return
		elif senderObj.guildBan:
			minute = (senderObj.guildBan - getSecond()) / 60
			if minute > 0:
				message.tips(senderObj, "你的禁言还剩#C04{}分钟#n".format(minute))
				return
	if channelId == CHANNEL_FIGHT and not senderObj.warId:
		message.tips(senderObj, "你不在战斗中，不可在战斗频道中发言")
		return
	if not doChannelCost(senderObj, channelId, content):
		return

	content	= trie.fliter(content)
	msg = {
		"channelId": channelId,
		"content": content,
		"isAudio": isAudio,
	}
	if isAudio:
		handleAudio(channelId, reqMsg.audioIdx, reqMsg.audioLen, msg)
	doChannel(senderObj, channelId, msg)

def doChannelCost(senderObj, channelId, content):
	'''发言检查和消耗
	'''
	limitData = getLimitData(senderObj, channelId)
	if not limitData:
		return 0

	limitLen = limitData.get("内容长度")
	if limitLen and calLenForWord(content) > limitLen:
		message.tips(senderObj, "发言内容太多啦，请分开两次发言")
		return 0

	limitInterval = limitData.get("时间间隔")
	if limitInterval:
		if not hasattr(senderObj, "lastTimeList"):
			senderObj.lastTimeList = {}
		if channelId not in senderObj.lastTimeList:
			senderObj.lastTimeList[channelId] = 0

		lastTime = senderObj.lastTimeList[channelId]
		leftTime = limitInterval - (getSecond() - lastTime)
		if leftTime > 0:
			message.tips(senderObj, "发言太频密，还有#C04%d秒#n可发言" % leftTime)
			return 0

	limitHuoli = limitData.get("活力")
	if limitHuoli:
		channelName = getChannelName(channelId)
		bFail, msgObj = chatService.getMainEP().rpcCostHuoli(senderObj.id, limitHuoli)
		if bFail or not msgObj.bValue:
			message.tips(senderObj, "活力不足，无法在#C04%s#n发言" % channelName)
			return 0

	if limitInterval:
		senderObj.lastTimeList[channelId] = getSecond()

	return 1

def getLimitData(senderObj, channelId):
	'''获取限制数据
	'''
	data = channelData.getData(channelId)
	if not data:
		return None

	limitData = {}
	level = senderObj.level
	for k,v in data.iteritems():
		if isinstance(v, str):
			if "lv" in v:
				v = v.replace("lv", str(level))
			v = int(eval(v))
		limitData[k] = v

	return limitData

def doChannel(senderObj, channelId, msg):
	msgObj = packReceiveMsg(senderObj, msg)
	if channelId in (CHANNEL_WORLD, CHANNEL_SYS_ANNOUNCE, CHANNEL_SYS_MESSAGE):
		broadcastAll(senderObj, msgObj, channelId)
	else:
		if channelId in roleIdsGetHandler:
			handler = roleIdsGetHandler[channelId]
			roleIdList = handler(senderObj)
		else:
			if msg.get("targetId"): # 系统发送的消息
				targetId = msg["targetId"]
			else:
				attrName = channelId2AttrName[channelId]
				targetId = getattr(senderObj, attrName)
			roleIdList = chatService.gChannelMgr.getRoleIdList(channelId, targetId)

		if roleIdList:
			broadcastByRoleIds(senderObj, msgObj, channelId, roleIdList)

def roleIdsGetCurrent(senderObj):
	'''当前频道
	'''
	# 暂时先这样
	roleIdList = []
	sceneId = senderObj.sceneId
	for targetSender in chatService.getAllRoleSenderList():
		if targetSender.sceneId != sceneId:
			continue
		if targetSender.warId: # 战斗中的不发
			continue
		roleIdList.append(targetSender.id)
	return roleIdList

def roleIdsGetTeamMake(senderObj):
	'''组队频道
	'''
	roleIdList = []
	level = senderObj.fastChatLevel
	for targetSender in chatService.getAllRoleSenderList():
		if targetSender.teamId:  #有队伍的不用发
			continue
		if targetSender.level < level:
			continue
		roleIdList.append(targetSender.id)
	return roleIdList

def roleIdsGetGuildAnnounce(senderObj):
	'''仙盟公告频道
	'''
	roleIdList = []
	for targetSender in chatService.getAllRoleSenderList():
		if targetSender.guildId: # 有仙盟的不发
			continue
		roleIdList.append(targetSender.id)
	return roleIdList

# 获取发送的角色id列表的处理列表
roleIdsGetHandler = {
	CHANNEL_CURRENT: roleIdsGetCurrent, # 当前频道
	CHANNEL_TEAM_MAKE:roleIdsGetTeamMake, # 组队频道
	CHANNEL_GUILD_ANNOUNCE:roleIdsGetGuildAnnounce, # 仙盟公告频道
}

def packReceiveMsg(senderObj, msg):
	msgObj = terminal_chat_pb2.receiveMsg()
	msgObj.sender.CopyFrom(senderObj.getMsg())
	msgObj.channelId = msg["channelId"]
	msgObj.content = msg["content"]
	msgObj.isAudio = msg.get("isAudio", 0)
	msgObj.roll = msg.get("roll", 0)
	audioInfo = msg.get("audioInfo", None)
	if audioInfo:
		msgObj.audio.CopyFrom(audioInfo)
	fastChatInfo = msg.get("fastChat")
	if fastChatInfo:
		msgObj.fastChat.CopyFrom(fastChatInfo)
	return msgObj

def broadcastAll(senderObj, msgObj, channelId):
	'''广播给所有角色
	'''
	packetData = endPoint.makePacket("rpcChatDown", msgObj)
	for targetSender in chatService.getAllRoleSenderList():
		ep = targetSender.endPoint
		if not ep: # 链接断开时，ep判断为False
			continue
		if targetSender.isChannelBaned(channelId, senderObj.id):
			continue
		ep.send(packetData)

def broadcastByRoleIds(senderObj, msgObj, channelId, roleIdList):
	'''根据角色id进行组播
	'''
	packetData = endPoint.makePacket("rpcChatDown", msgObj)
	for roleId in roleIdList:
		targetSender = chatService.getSender(roleId)
		if not targetSender:
			continue
		ep = targetSender.endPoint
		if not ep: # 链接断开时，ep判断为False
			continue
		if targetSender.isChannelBaned(channelId, senderObj.id):
			continue
		ep.send(packetData)

def rpcBanChannelReq(ep, senderObj, reqMsg):
	'''请求屏蔽的频道
	'''
	msg = terminal_chat_pb2.banChannelMsg()
	msg.channelIdList.extend(senderObj.channelBanList)
	senderObj.endPoint.rpcBanChannelRes(msg)

def rpcBanChannelSet(ep, senderObj, reqMsg):
	'''设置屏蔽的频道
	'''
	senderObj.channelBanList = reqMsg.channelIdList

# def rpcAudioReq(ep, senderObj, reqMsg):
# 	'''请求语音信息
# 	'''
# 	idx = reqMsg.audioIdx
# 	audioObj = chatService.gAudioMgr.getAudio(idx)
# 	if not audioObj:
# 		message.tips(senderObj, "不存在此语音信息")
# 		return
# 	senderObj.endPoint.rpcAudioRes(audioObj.getMsg())

def handleAudio(channelId, iIdx, iLen, dMsg):
	audioObj = chatService.gAudioMgr.addAudio(channelId, iIdx, iLen)
	if audioObj:
		dMsg["audioInfo"] = audioObj

from common import *
from chatService.defines import *
import chatService
import channelData
import message
import trie


############################################  NOTE  ########################################################
#
#           Creates NER training data in Spacy format from JSON downloaded from Dataturks.
#
#           Outputs the Spacy training data which can be used for Spacy training.
#
############################################################################################################
import json
import random
import logging
from sklearn.metrics import classification_report
from sklearn.metrics import precision_recall_fscore_support
from spacy.gold import GoldParse
from spacy.scorer import Scorer
from sklearn.metrics import accuracy_score
def convert_dataturks_to_spacy(dataturks_JSON_FilePath):
    try:
        training_data = []
        lines=[]
        #with open(dataturks_JSON_FilePath, 'r') as f:
        with open(dataturks_JSON_FilePath, 'r', encoding='utf-8') as f:
            lines = f.readlines()

        for line in lines:
            #data = json.loads(line)
            data = json.load(line)
            text = data['content']
            entities = []
            for annotation in data['annotation']:
                #only a single point in text annotation.
                point = annotation['points'][0]
                labels = annotation['label']
                # handle both list of labels or a single label.
                if not isinstance(labels, list):
                    labels = [labels]

                for label in labels:
                    #dataturks indices are both inclusive [start, end] but spacy is not [start, end)
                    entities.append((point['start'], point['end'] + 1 ,label))

            training_data.append((text, {"entities" : entities}))

        return training_data
    except Exception as e:
        logging.exception("Unable to process " + dataturks_JSON_FilePath + "\n" + "error = " + str(e))
        return None

import spacy
################### Train Spacy NER.###########
def train_spacy():

    TRAIN_DATA = convert_dataturks_to_spacy("traindata.json")
    nlp = spacy.blank('en')  # create blank Language class
    # create the built-in pipeline components and add them to the pipeline
    # nlp.create_pipe works for built-ins that are registered with spaCy
    if 'ner' not in nlp.pipe_names:
        ner = nlp.create_pipe('ner')
        nlp.add_pipe(ner, last=True)

    # add labels
    for _, annotations in TRAIN_DATA:
         for ent in annotations.get('entities'):
            ner.add_label(ent[2])

    # get names of other pipes to disable them during training
    other_pipes = [pipe for pipe in nlp.pipe_names if pipe != 'ner']
    with nlp.disable_pipes(*other_pipes):  # only train NER
        optimizer = nlp.begin_training()
        for itn in range(10):
            print("Statring iteration " + str(itn))
            random.shuffle(TRAIN_DATA)
            losses = {}
            for text, annotations in TRAIN_DATA:
                nlp.update(
                    [text],  # batch of texts
                    [annotations],  # batch of annotations
                    drop=0.2,  # dropout - make it harder to memorise data
                    sgd=optimizer,  # callable to update weights
                    losses=losses)
            print(losses)
    #test the model and evaluate it
    #examples = convert_dataturks_to_spacy("/home/abhishekn/dataturks/entityrecognition/testdata.json")
    examples = convert_dataturks_to_spacy("testdata.json")
    tp=0
    tr=0
    tf=0

    ta=0
    c=0
    for text,annot in examples:

        f=open("resume"+str(c)+".txt","w")
        doc_to_test=nlp(text)
        d={}
        for ent in doc_to_test.ents:
            d[ent.label_]=[]
        for ent in doc_to_test.ents:
            d[ent.label_].append(ent.text)

        for i in set(d.keys()):

            f.write("\n\n")
            f.write(i +":"+"\n")
            for j in set(d[i]):
                f.write(j.replace('\n','')+"\n")
        d={}
        for ent in doc_to_test.ents:
            d[ent.label_]=[0,0,0,0,0,0]
        for ent in doc_to_test.ents:
            doc_gold_text= nlp.make_doc(text)
            gold = GoldParse(doc_gold_text, entities=annot.get("entities"))
            y_true = [ent.label_ if ent.label_ in x else 'Not '+ent.label_ for x in gold.ner]
            y_pred = [x.ent_type_ if x.ent_type_ ==ent.label_ else 'Not '+ent.label_ for x in doc_to_test]
            if(d[ent.label_][0]==0):
                #f.write("For Entity "+ent.label_+"\n")
                #f.write(classification_report(y_true, y_pred)+"\n")
                (p,r,f,s)= precision_recall_fscore_support(y_true,y_pred,average='weighted')
                a=accuracy_score(y_true,y_pred)
                d[ent.label_][0]=1
                d[ent.label_][1]+=p
                d[ent.label_][2]+=r
                d[ent.label_][3]+=f
                d[ent.label_][4]+=a
                d[ent.label_][5]+=1
        c+=1
    for i in d:
        print("\n For Entity "+i+"\n")
        print("Accuracy : "+str((d[i][4]/d[i][5])*100)+"%")
        print("Precision : "+str(d[i][1]/d[i][5]))
        print("Recall : "+str(d[i][2]/d[i][5]))
        print("F-score : "+str(d[i][3]/d[i][5]))
train_spacy()

#BREAK UP ON JSON OBJECT
#{"content": "Govardhana K
#\nSenior Software Engineer
# \n\nBengaluru, Karnataka, Karnataka -
# Email me on Indeed: indeed.com/r/Govardhana-K/\nb2de315d95905b68\n
# \nTotal IT experience 5 Years 6 Months
# \nCloud Lending Solutions INC 4 Month • Salesforce Developer
# \nOracle 5 Years 2 Month • Core Java Developer
# \nLanguages Core Java, Go Lang\nOracle PL-SQL programming,\nSales Force Developer with APEX.
# \n\nDesignations & Promotions
# \n\nWilling to relocate: Anywhere
# \n\nWORK EXPERIENCE\n\nSenior Software Engineer
# \n\nCloud Lending Solutions -  Bangalore, Karnataka -\n\nJanuary 2018 to Present
# \n\nPresent\n\nSenior Consultant\n\nOracle -  Bangalore, Karnataka -\n
# \nNovember 2016 to December 2017\n\nStaff Consultant\n\nOracle -  Bangalore, Karnataka -\n
# \nJanuary 2014 to October 2016\n
# \nAssociate Consultant\n\nOracle -  Bangalore, Karnataka -\n
# \nNovember 2012 to December 2013\n
# \nEDUCATION\n\nB.E in Computer Science Engineering\n
# \nAdithya Institute of Technology -  Tamil Nadu\n\nSeptember 2008 to June 2012\n
# \nhttps://www.indeed.com/r/Govardhana-K/b2de315d95905b68?isid=rex-download&ikw=download-top&co=IN\nhttps://www.indeed.com/r/Govardhana-K/b2de315d95905b68?isid=rex-download&ikw=download-top&co=IN\n\n\nSKILLS\n\nAPEX.
# (Less than 1 year), Data Structures (3 years), FLEXCUBE (5 years), Oracle (5 years),\nAlgorithms (3 years)\n
# \nLINKS\n\nhttps://www.linkedin.com/in/govardhana-k-61024944/\n
# \nADDITIONAL INFORMATION\n\nTechnical Proficiency:\n\nLanguages: Core Java, Go Lang, Data Structures & Algorithms, Oracle\nPL-SQL programming, Sales Force with APEX.
# \nTools: RADTool, Jdeveloper, NetBeans, Eclipse, SQL developer,
# \nPL/SQL Developer, WinSCP, Putty
# \nWeb Technologies: JavaScript, XML, HTML, Webservice\n
# \nOperating Systems: Linux, Windows
# \nVersion control system SVN & Git-Hub
# \nDatabases: Oracle\nMiddleware: Web logic, OC4J
# \nProduct FLEXCUBE: Oracle FLEXCUBE Versions 10.x, 11.x and 12.x
# \n\nhttps://www.linkedin.com/in/govardhana-k-61024944/",
#
# "annotation":[
# {"label":["Companies worked at"],
# "points":[{"start":1749,"end":1754,"text":"Oracle"}]},
# {"label":["Companies worked at"],
# "points":[{"start":1696,"end":1701,"text":"Oracle"}]},
# {"label":["Companies worked at"],
# "points":[{"start":1417,"end":1422,"text":"Oracle"}]},
# {"label":["Skills"],
# "points":[{"start":1356,"end":1792,"text":"Languages: Core Java, Go Lang, Data Structures & Algorithms, Oracle
# \nPL-SQL programming, Sales Force with APEX.
# \nTools: RADTool, Jdeveloper, NetBeans, Eclipse, SQL developer,
# \nPL/SQL Developer, WinSCP, Putty
# \nWeb Technologies: JavaScript, XML, HTML, Webservice
# \n\nOperating Systems: Linux, Windows
# \nVersion control system SVN & Git-Hub
# \nDatabases: Oracle
# \nMiddleware: Web logic, OC4J
# \nProduct FLEXCUBE: Oracle FLEXCUBE Versions 10.x, 11.x and 12.x"}]},
# {"label":["Companies worked at"],
# "points":[{"start":1209,"end":1214,"text":"Oracle"}]},
# {"label":["Skills"],
# "points":[{"start":1136,"end":1247,"text":"APEX. (Less than 1 year), Data Structures (3 years), FLEXCUBE (5 years), Oracle (5 years),
# \nAlgorithms (3 years)\n"}]},
# {"label":["Graduation Year"],
# "points":[{"start":928,"end":931,"text":"2012"}]},
# {"label":["College Name"],
# "points":[{"start":858,"end":888,"text":"Adithya Institute of Technology"}]},
# {"label":["Degree"],
# "points":[{"start":821,"end":855,"text":"B.E in Computer Science Engineering"}]},
# {"label":["Graduation Year"],
# "points":[{"start":787,"end":790,"text":"2012"}]},
# {"label":["Companies worked at"],
# "points":[{"start":744,"end":749,"text":"Oracle"}]},
# {"label":["Designation"],
# "points":[{"start":722,"end":741,"text":"Associate Consultant"}]},
# {"label":["Companies worked at"],
# "points":[{"start":658,"end":663,"text":"Oracle"}]},
# {"label":["Designation"],
# "points":[{"start":640,"end":655,"text":"Staff Consultant"}]},
# {"label":["Companies worked at"],
# "points":[{"start":574,"end":579,"text":"Oracle"}]},
# {"label":["Designation"],
# "points":[{"start":555,"end":572,"text":"Senior Consultant\n"}]},
# {"label":["Companies worked at"],
# "points":[{"start":470,"end":492,"text":"Cloud Lending Solutions"}]},
# {"label":["Designation"],
# "points":[{"start":444,"end":468,"text":"Senior Software Engineer\n"}]},
# {"label":["Companies worked at"],
# "points":[{"start":308,"end":313,"text":"Oracle"}]},
# {"label":["Companies worked at"],
# "points":[{"start":234,"end":239,"text":"Oracle"}]},
# {"label":["Companies worked at"],
# "points":[{"start":175,"end":197,"text":"Cloud Lending Solutions"}]},
# {"label":["Email Address"],
# "points":[{"start":93,"end":136,"text":"indeed.com/r/Govardhana-K/\nb2de315d95905b68\n"}]},
# {"label":["Location"],
# "points":[{"start":39,"end":47,"text":"Bengaluru"}]},
# {"label":["Designation"],
# "points":[{"start":13,"end":37,"text":"Senior Software Engineer\n"}]},
# {"label":["Name"],"points":[{"start":0,"end":11,"text":"Govardhana K"}]}]}


"""Wrapper for pounders in the tao package."""
import sys
from functools import partial

import numpy as np

if sys.platform != "win32":
    from petsc4py import PETSc

def minimize_pounders_np(
    func,
    x0,
    bounds,
    gatol=1e-8,
    grtol=1e-8,
    gttol=1e-10,
    init_tr=None,
    max_iterations=None,
    n_errors=None,
):
    """Minimize a function using the Pounders algorithm.

    Pounders can be a useful tool for economists who estimate structural models using
    indirect inference, because unlike commonly used algorithms such as Nelder-Mead,
    Pounders is tailored for minimizing a non-linear sum of squares objective function,
    and therefore may require fewer iterations to arrive at a local optimum than
    Nelder-Mead.

    The criterion function :func:`func` should return an array of the errors NOT an
    array of the squared errors or anything else.

    Args:
        func (callable): Objective function.
        x0 (np.ndarray): Starting values of the parameters.
        bounds (Tuple[np.ndarray]): A tuple containing two NumPy arrays where the first
            corresponds to the lower and the second to the upper bound. Unbounded
            parameters are represented by infinite values. The arrays have the same
            length as the parameter vector.
        gatol (float): Stop if relative norm of gradient is less than this. If set to
            False the algorithm will not consider gatol. Default is 1e-8.
        grtol (float): Stop if norm of gradient is less than this. If set to False the
            algorithm will not consider grtol. Default is 1e-8.
        gttol (float): Stop if norm of gradient is reduced by this factor. If set to
            False the algorithm will not consider grtol. Default is 1e-10.
        init_tr (float): Sets the radius for the initial trust region that the optimizer
            employs. If `None` the algorithm uses 100 as initial  trust region radius.
            Default is `None`.
        max_iterations (int): Alternative Stopping criterion. If set the routine will
            stop after the number of specified iterations or after the step size is
            sufficiently small. If the variable is set the default criteria will all be
            ignored. Default is `None`.
        n_errors (int or None): The number of outputs of `func` are necessary to
            pre-allocate the results array. If the argument is ``None``, evaluate the
            function once. This might be undesirable during dashboard optimizations.

    Returns:
        results (dict): Dictionary with processed optimization results.

    .. _TAO Users Manual:
        https://www.mcs.anl.gov/petsc/petsc-current/docs/tao_manual.pdf
    .. _Solving Derivative-Free Nonlinear Least Squares Problems with POUNDERS:
        https://www.mcs.anl.gov/papers/P5120-0414.pdf

    """
    if sys.platform == "win32":
        raise NotImplementedError("The pounders algorithm is not available on Windows.")

    # We need to know the dimension of the output of the criterion function. Evaluate
    # plain `criterion` to prevent logging.
    if n_errors is None:
        n_errors = len(func(x0))

    # We want to get containers for the func vector and the paras.
    x0 = _initialise_petsc_array(x0)
    residuals_out = _initialise_petsc_array(n_errors)

    # Create the solver object.
    tao = PETSc.TAO().create(PETSc.COMM_WORLD)

    # Set the solver type.
    tao.setType("pounders")

    tao.setFromOptions()

    def func_tao(tao, x, resid_out):
        """Evaluate objective and attach result to an petsc object f.

        This is required to use the pounders solver from tao.

        Args:
             tao: The tao object we created for the optimization task.
             x (PETSc.array): Current parameter values.
             f: Petsc object in which we save the current function value.

        """
        resid_out.array = func(x.array)

    # Set the procedure for calculating the objective. This part has to be changed if we
    # want more than pounders.
    tao.setResidual(func_tao, residuals_out)

    # We try to set user defined convergence tests.
    if init_tr is not None:
        tao.setInitialTrustRegionRadius(init_tr)

    # Add bounds.
    n_params = len(x0.array)
    processed_bounds = []
    for bound in bounds:
        bound = np.full(n_params, bound) if isinstance(bound, (int, float)) else bound
        processed_bounds.append(_initialise_petsc_array(bound))
    tao.setVariableBounds(processed_bounds)

    # Put the starting values into the container and pass them to the optimizer.
    tao.setInitial(x0)

    # Obtain tolerances for the convergence criteria. Since we can not create gttol
    # manually we manually set gatol and or grtol to zero once a subset of these two is
    # turned off and gttol is still turned on.
    default_gatol = gatol if gatol else -1
    default_gttol = gttol if gttol else -1
    default_grtol = grtol if grtol else -1
    # Set tolerances for default convergence tests.
    tao.setTolerances(gatol=default_gatol, gttol=default_gttol, grtol=default_grtol)

    # Set user defined convergence tests. Beware that specifying multiple tests could
    # overwrite others or lead to unclear behavior.
    if max_iterations is not None:
        tao.setConvergenceTest(partial(_max_iters, max_iterations))
    elif gttol is False and gatol is False:
        tao.setConvergenceTest(partial(_grtol_conv, grtol))
    elif grtol is False and gttol is False:
        tao.setConvergenceTest(partial(_gatol_conv, gatol))
    elif gttol is False:
        tao.setConvergenceTest(partial(_grtol_gatol_conv, grtol, gatol))

    # Run the problem.
    tao.solve()

    results = _process_pounders_results(residuals_out, tao)

    # Destroy petsc objects for memory reasons.
    tao.destroy()
    x0.destroy()
    residuals_out.destroy()

    return results

def _initialise_petsc_array(len_or_array):
    """Initialize an empty array or fill in provided values.

    Args:
        len_or_array (int or np.ndarray): If the value is an integer, allocate an empty
            array with the given length. If the value is an array, allocate an array of
            equal length and fill in the values.

    """
    length = len_or_array if isinstance(len_or_array, int) else len(len_or_array)

    array = PETSc.Vec().create(PETSc.COMM_WORLD)
    array.setSizes(length)
    array.setFromOptions()

    if isinstance(len_or_array, np.ndarray):
        array.array = len_or_array

    return array

def _max_iters(max_iterations, tao):
    if tao.getSolutionStatus()[0] < max_iterations:
        return 0
    elif tao.getSolutionStatus()[0] >= max_iterations:
        tao.setConvergedReason(8)

def _gatol_conv(gatol, tao):
    if tao.getSolutionStatus()[2] >= gatol:
        return 0
    elif tao.getSolutionStatus()[2] < gatol:
        tao.setConvergedReason(3)

def _grtol_conv(grtol, tao):
    if tao.getSolutionStatus()[2] / tao.getSolutionStatus()[1] >= grtol:
        return 0
    elif tao.getSolutionStatus()[2] / tao.getSolutionStatus()[1] < grtol:
        tao.setConvergedReason(4)

def _grtol_gatol_conv(grtol, gatol, tao):
    if tao.getSolutionStatus()[2] / tao.getSolutionStatus()[1] >= grtol:
        return 0
    elif tao.getSolutionStatus()[2] / tao.getSolutionStatus()[1] < grtol:
        tao.setConvergedReason(4)

    elif tao.getSolutionStatus()[2] < gatol:
        tao.setConvergedReason(3)

def _translate_tao_convergence_reason(tao_resaon):
    mapping = {
        3: "gatol below critical value",
        4: "grtol below critical value",
        5: "gttol below critical value",
        6: "step size small",
        7: "objective below min value",
        8: "user defined",
        -2: "maxits reached",
        -4: "numerical porblems",
        -5: "max funcevals reached",
        -6: "line search failure",
        -7: "trust region failure",
        -8: "user defined",
    }
    return mapping[tao_resaon]

def _process_pounders_results(residuals_out, tao):
    results = {
        # Harmonized results.
        "status": "success",
        "fitness": tao.function,
        "x": tao.solution.array,
        "n_evaluations": tao.getIterationNumber(),
        # Other results.
        "fitness_values": residuals_out.array,
        "conv": _translate_tao_convergence_reason(tao.getConvergedReason()),
        "gnorm": tao.gnorm,
        "cnorm": tao.cnorm,
    }

    return results


# encoding=utf-8
from sklearn.metrics import mean_squared_error, mean_absolute_error
import numpy as np
from keras.metrics import mean_absolute_percentage_error
from sklearn.metrics import r2_score

# ¼ÆËãRMSE
def calcRMSE(pred, true):
    return np.sqrt(mean_squared_error(true, pred))

def calcMSE(pred, true):
    return mean_squared_error(true, pred)

# ¼ÆËãMAE
def calcMAE(pred, true):
    # pred = pred[:, 0]
    return mean_absolute_error(true, pred)

# ¼ÆËãMAPE
def calcMAPE(pred, true, epsion=0.0000000):
    # pred = pred[:,0] # ÁÐ×ªÐÐ£¬±ãÓÚ¹ã²¥¼ÆËãÎó²îÖ¸±ê
    # print (true-pred).shape
    # print true.shape
    # print pred.shape
    # true += epsion
    return np.sum(np.abs((true - pred) / true)) * 100 / len(true)
    # return mean_absolute_percentage_error(true, pred)

# ¼ÆËãSMAPE
def calcSMAPE(pred, true):
    delim = (np.abs(true) + np.abs(pred)) / 2.0
    return np.sum(np.abs((true - pred) / delim)) / len(true) * 100

def mape(predicted, test):
    temps = 0.0
    instances = 0
    for i in range(len(predicted)):
        temps += abs(predicted[i] - test[i]) * 100.0 / test[i]
        instances += 1

    return temps / instances

def mpe(predicted, test):
    if not len(predicted) == len(test):
        print("Predicted values and output test instances do not match.")

    temps = 0.0
    instances = 0
    for i in range(len(predicted)):
        temps += (predicted[i] - test[i]) * 100.0 / test[i]
        instances += 1

    return temps / instances

def mse(predicted, test):
    if not len(predicted) == len(test):
        print("Predicted values and output test instances do not match.")

    temps = 0.0
    instances = 0
    for i in range(len(predicted)):
        temps += (predicted[i] - test[i]) ** 2
        instances += 1

    return temps / instances

def rmse(predicted, test):
    return (mse(predicted, test)) ** 0.5

def mae(predicted, test):
    if not len(predicted) != len(test):
        print("Predicted values and output test instances do not match.")

    temps = 0.0
    instances = 0
    for i in range(len(predicted)):
        temps += abs(predicted[i] - test[i])
        instances += 1

    return temps / instances

def r2(predicted, test):
    if not len(predicted) == len(test):
        print("Predicted values and output test instances do not match.")

    return r2_score(test, predicted)

def pinballLoss(y_true, y_pred):
    '''
    r 邻域范围
    location 记录真实值和预测值相比那个地方大，那个地方小
    re 邻域的张量
    e 真实值与预测值差距的绝对值
    position 真实值与预测值差距的绝对值 与 邻域 的大小比较
    h1 l2规范
    h2 l1规范
    h标准规范化后的数据
    p损失值
    '''
    p=0
    q1=tf.cast(tf.linspace(0.1,0.1,1),tf.float64)
    q2=tf.cast(1-q1,tf.float64)
    r=0.01
    location=tf.less(y_true,y_pred)
    e = tf.abs(y_true - y_pred)

    re=tf.ones_like(e)*r
    position=tf.less(e,re)

    h1=tf.square(e)/(2*r)
    h2=e-r/2
    h=tf.where(position,h1,h2)

    p1=tf.multiply(q2,h)
    p2=tf.multiply(q1,h)
    p=tf.reduce_mean(tf.where(location,p1,p2),axis=0)
    p=tf.reduce_mean(p)

    return p


"""
lab_utils_common.py
    functions common to all optional labs, Course 1, Week 2
"""

import numpy as np
import matplotlib.pyplot as plt

plt.style.use('./deeplearning.mplstyle')
dlblue = '#0096ff'; dlorange = '#FF9300'; dldarkred='#C00000'; dlmagenta='#FF40FF'; dlpurple='#7030A0';
dlcolors = [dlblue, dlorange, dldarkred, dlmagenta, dlpurple]
dlc = dict(dlblue = '#0096ff', dlorange = '#FF9300', dldarkred='#C00000', dlmagenta='#FF40FF', dlpurple='#7030A0')

##########################################################
# Regression Routines
##########################################################

#Function to calculate the cost
def compute_cost_matrix(X, y, w, b, verbose=False):
    """
    Computes the gradient for linear regression
     Args:
      X (ndarray (m,n)): Data, m examples with n features
      y (ndarray (m,)) : target values
      w (ndarray (n,)) : model parameters
      b (scalar)       : model parameter
      verbose : (Boolean) If true, print out intermediate value f_wb
    Returns
      cost: (scalar)
    """
    m = X.shape[0]

    # calculate f_wb for all examples.
    f_wb = X @ w + b
    # calculate cost
    total_cost = (1/(2*m)) * np.sum((f_wb-y)**2)

    if verbose: print("f_wb:")
    if verbose: print(f_wb)

    return total_cost

def compute_gradient_matrix(X, y, w, b):
    """
    Computes the gradient for linear regression

    Args:
      X (ndarray (m,n)): Data, m examples with n features
      y (ndarray (m,)) : target values
      w (ndarray (n,)) : model parameters
      b (scalar)       : model parameter
    Returns
      dj_dw (ndarray (n,1)): The gradient of the cost w.r.t. the parameters w.
      dj_db (scalar):        The gradient of the cost w.r.t. the parameter b.

    """
    m,n = X.shape
    f_wb = X @ w + b
    e   = f_wb - y # ndarray(m,)
    dj_dw  = (1/m) * (X.T @ e) # X.T 为转置矩阵(n,m)；dj_dw最终为(n,1)
    dj_db  = (1/m) * np.sum(e) # b的偏导数刚好为error

    return dj_db,dj_dw

# Loop version of multi-variable compute_cost
def compute_cost(X, y, w, b):
    """
    compute cost
    Args:
      X (ndarray (m,n)): Data, m examples with n features
      y (ndarray (m,)) : target values
      w (ndarray (n,)) : model parameters
      b (scalar)       : model parameter
    Returns
      cost (scalar)    : cost
    """
    m = X.shape[0]
    cost = 0.0
    for i in range(m):
        f_wb_i = np.dot(X[i],w) + b           #(n,)(n,)=scalar
        cost = cost + (f_wb_i - y[i])**2
    cost = cost/(2*m)
    return cost

def compute_gradient(X, y, w, b):
    """
    Computes the gradient for linear regression
    Args:
      X (ndarray (m,n)): Data, m examples with n features
      y (ndarray (m,)) : target values
      w (ndarray (n,)) : model parameters
      b (scalar)       : model parameter
    Returns
      dj_dw (ndarray Shape (n,)): The gradient of the cost w.r.t. the parameters w.
      dj_db (scalar):             The gradient of the cost w.r.t. the parameter b.
    """
    m,n = X.shape           #(number of examples, number of features)
    dj_dw = np.zeros((n,))
    dj_db = 0.

    for i in range(m):
        err = (np.dot(X[i], w) + b) - y[i]
        for j in range(n):
            dj_dw[j] = dj_dw[j] + err * X[i,j]
        dj_db = dj_db + err
    dj_dw = dj_dw/m
    dj_db = dj_db/m

    return dj_db,dj_dw


"""
This is an upgraded version of Ceshine's LGBM starter script, simply adding
more average features and weekly average features on it.
"""
from datetime import date, timedelta

import pandas as pd
import numpy as np
import sys

##########################################################################

df_sub = pd.read_csv('../input/zero_promo_testset.csv')

df_sub['unit_sales'] = 0

df_prev = pd.read_csv('../submit/T300.csv')

t_new = pd.merge(df_prev, df_sub, on=['id'], how='left')
t_new['unit_sales'] = t_new.unit_sales_y.combine_first(t_new.unit_sales_x)

submission = t_new[['id', 'unit_sales']]
del t_new

print("Merged  SUM =",  submission.unit_sales.sum())
print("Merged  MEAN =",  submission.unit_sales.mean())

submission.to_csv('../submit/T300_V2.csv.gz',
                  float_format='%.4f', index=None, compression='gzip')


# The MIT License (MIT)
#
# Copyright (c) 2020 Aibolit
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from itertools import zip_longest
from pathlib import Path
from unittest import TestCase

from aibolit.ast_framework import AST, ASTNodeType
from aibolit.utils.ast_builder import build_ast
from aibolit.extract_method_baseline.extract_semantic import (
    extract_method_statements_semantic,
    StatementSemantic,
)

def variables_semantic(*variables_names: str) -> StatementSemantic:
    return StatementSemantic(used_variables=set(variables_names))

class PaperExampleTestCase(TestCase):
    def test_semantic_extraction(self):
        semantic = extract_method_statements_semantic(self.method_ast)
        for comparison_index, (statement, actual_semantic, expected_semantic) in enumerate(
            zip_longest(semantic.keys(), semantic.values(), self.method_semantic)
        ):
            self.assertEqual(
                actual_semantic,
                expected_semantic,
                f"Failed on {statement.node_type} statement on line {statement.line}. "
                f"Comparison index is {comparison_index}.",
            )

    @classmethod
    def setUpClass(cls):
        current_directory = Path(__file__).absolute().parent
        filepath = current_directory / "ExampleFromPaper.java"

        ast = AST.build_from_javalang(build_ast(filepath))
        try:
            class_declaration = next(
                node
                for node in ast.get_root().types
                if node.node_type == ASTNodeType.CLASS_DECLARATION and node.name == "ExampleFromPaper"
            )

            method_declaration = next(
                node for node in class_declaration.methods if node.name == "grabManifests"
            )
        except StopIteration as e:
            raise RuntimeError(
                f"Failed to find method grabManifests in class ExampleFromPaper in file {filepath}."
            ) from e

        cls.method_ast = ast.get_subtree(method_declaration)

    method_semantic = [
        StatementSemantic(used_variables={"manifests", "length"}, used_objects={"rcs"}),
        StatementSemantic(used_variables={"i", "length"}, used_objects={"rcs"}),
        variables_semantic("rec"),
        variables_semantic("i", "rcs"),
        StatementSemantic(used_variables={"rec", "rcs", "i"}, used_methods={"grabRes"}),
        StatementSemantic(used_variables={"rec", "rcs", "i"}, used_methods={"grabNonFileSetRes"}),
        variables_semantic("rec", "j", "length"),
        StatementSemantic(used_variables={"name", "rec", "j"}, used_methods={"getName", "replace"}),
        variables_semantic("rcs", "i"),
        variables_semantic("afs", "rcs", "i"),
        StatementSemantic(used_objects={"afs"}, used_methods={"equals", "getFullpath", "getProj"}),
        StatementSemantic(used_objects={"name.afs"}, used_methods={"getFullpath", "getProj"}),
        StatementSemantic(used_objects={"afs"}, used_methods={"equals", "getPref", "getProj"}),
        StatementSemantic(used_variables={"pr"}, used_objects={"afs"}, used_methods={"getPref", "getProj"}),
        StatementSemantic(used_objects={"pr"}, used_methods={"endsWith"}),
        variables_semantic("pr"),
        variables_semantic("pr", "name"),
        StatementSemantic(
            used_variables={"MANIFEST_NAME"}, used_objects={"name"}, used_methods={"equalsIgnoreCase"}
        ),
        variables_semantic("manifests", "i", "rec", "j"),
        variables_semantic("manifests", "i"),
        variables_semantic("manifests", "i"),
        variables_semantic("manifests"),
    ]


import logging

import dateutil.parser
from datetime import datetime

import six

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

def to_datetime(timestamp):
    """
    Converts ISO 8601 timestamp to datetime object.
    """
    if isinstance(timestamp, datetime) or timestamp is None:
        return timestamp

    return dateutil.parser.parse(timestamp)

def to_iso(obj):
    """
    Converts datetime object to an ISO 8601 timestamp.
    """
    if isinstance(obj, six.string_types) or obj is None:
        return obj
    elif isinstance(obj, datetime):
        timestamp = obj.isoformat()
        if ('+' not in timestamp and len(timestamp) == 19 and
            not timestamp.endswith('Z')):
            timestamp += 'Z' # UTC
        return timestamp
    else:
        raise Exception("Unable to convert %s to an ISO 8601 timestampe." %
                        obj)


# setup.py
# Copyright (c) 2013-2019 Pablo Acosta-Serafini
# See LICENSE for details
# pylint: disable=C0111,E0401,E0601,E1111,R0904,W0122,W0201,W0621

# Taken in large part from:
#    http://www.jeffknupp.com/blog/2013/08/16/
#    open-sourcing-a-python-project-the-right-way/
# With additional hints from:
#     http://oddbird.net/set-your-code-free-preso/
# The function to get the version number from __init__.py is from:
#     https://python-packaging-user-guide.readthedocs.org/
#     en/latest/single_source_version/
# Standard library imports
from __future__ import print_function
import io
import glob
import os
import sys

# PyPI imports
from setuptools import setup
from setuptools.command.test import test as TestCommand

# Intra-package imports
from pypkg.functions import (
    get_entry_points,
    get_pkg_data_files,
    get_pkg_submodules,
    load_requirements,
    python_version,
)

###
# Supported interpreter check
###
# When installing from tarball/zip/wheel, path is temporary one and setup.py
# is not in a directory where its name is the package name, have to find
# package name by finding location of pkgdata file
STEM = "pkgdata"
FOUND = False
START_DIR = os.path.dirname(os.path.abspath(__file__))
for (DIRPATH, _, FNAMES) in os.walk(START_DIR):
    # Ignore .tox, .git and other directories
    if DIRPATH[len(START_DIR) + 1 :].startswith("."):
        continue
    #
    for FNAME in FNAMES:
        if os.path.splitext(os.path.basename(FNAME))[0] == STEM:
            FNAME = os.path.join(DIRPATH, FNAME)
            sys.path.append(DIRPATH)
            import pkgdata

            PKG_NAME = os.path.basename(
                os.path.dirname(os.path.abspath(sys.modules["pkgdata"].__file__))
            )
            FOUND = True
            break
if not FOUND:
    raise RuntimeError("Supported Python interpreter versions cold not be found")
PYTHON_VER = python_version("{0:0x}".format(sys.hexversion & 0xFFFF0000)[:-4])
SUPPORTED_INTERPS = sorted(pkgdata.SUPPORTED_INTERPS)
if PYTHON_VER not in SUPPORTED_INTERPS:
    sys.exit("Supported interpreter versions: {0}".format(", ".join(SUPPORTED_INTERPS)))

###
# Functions
###
def get_short_desc(long_desc):
    """Get first sentence of first paragraph of long description."""
    found = False
    olines = []
    for line in [item.rstrip() for item in long_desc.split("\n")]:
        if found and (((not line) and (not olines)) or (line and olines)):
            olines.append(line)
        elif found and olines and (not line):
            return (" ".join(olines).split(".")[0]).strip()
        found = line == ".. [[[end]]]" if not found else found
    return ""

def read(*filenames, **kwargs):
    """Read plain text file(s)."""
    encoding = kwargs.get("encoding", "utf-8")
    sep = kwargs.get("sep", "\n")
    buf = []
    for filename in filenames:
        with io.open(filename, encoding=encoding) as fobj:
            buf.append(fobj.read())
    return sep.join(buf)

###
# Global variables
###
REPO = "http://github.com/pmacosta/{pkg_name}/".format(pkg_name=PKG_NAME)
AUTHOR = "Pablo Acosta-Serafini"
AUTHOR_EMAIL = "pmasdev@gmail.com"
LICENSE = "MIT"
PKG_DIR = os.path.abspath(os.path.dirname(__file__))
LONG_DESCRIPTION = read(
    os.path.join(PKG_DIR, "README.rst"), os.path.join(PKG_DIR, "CHANGELOG.rst")
)
SHORT_DESC = get_short_desc(LONG_DESCRIPTION)
# Actual directory is os.join(sys.prefix, 'share', PKG_NAME)
SHARE_DIR = os.path.join("share", PKG_NAME)
INSTALL_REQUIRES = load_requirements(PKG_DIR, PYTHON_VER, "source")
TESTING_REQUIRES = load_requirements(PKG_DIR, PYTHON_VER, "testing")
if os.environ.get("MERGE_REQUIREMENTS", False):
    INSTALL_REQUIRES = INSTALL_REQUIRES + TESTING_REQUIRES
try:
    DATA_FILES = get_pkg_data_files(SHARE_DIR)
except IOError:
    print("PKG_DIR: {0}".format(PKG_DIR))
    print("Contents:")
    print(glob.glob(os.path.join(PKG_DIR, "*")))
    print("PKG_DIR/data")
    print("Contents:")
    print(glob.glob(os.path.join(PKG_DIR, "data", "*")))
    raise

###
# Extract version (from coveragepy)
###
VERSION_PY = os.path.join(PKG_DIR, PKG_NAME, "pkgdata.py")
with open(VERSION_PY) as fobj:
    __version__ = VERSION_INFO = ""
    # Execute the code in pkgdata.py.
    exec(compile(fobj.read(), VERSION_PY, "exec"))
if VERSION_INFO[3] == "alpha":
    DEVSTAT = "3 - Alpha"
elif VERSION_INFO[3] in ["beta", "candidate"]:
    DEVSTAT = "4 - Beta"
else:
    assert VERSION_INFO[3] == "final"
    DEVSTAT = "5 - Production/Stable"

###
# Classes
###
class Tox(TestCommand):  # noqa
    user_options = [("tox-args=", "a", "Arguments to pass to tox")]

    def initialize_options(self):  # noqa
        TestCommand.initialize_options(self)
        self.tox_args = None

    def finalize_options(self):  # noqa
        TestCommand.finalize_options(self)
        self.test_args = []
        self.test_suite = True

    def run_tests(self):  # noqa
        import shlex
        import tox

        args = self.tox_args
        if args:
            args = shlex.split(self.tox_args)
        errno = tox.cmdline(args=args)
        sys.exit(errno)

###
# Processing
###
# package_data is used only for binary packages, i.e.
# $ python setup.py bdist ...
# but NOT when building source packages, i.e.
# $ python setup.py sdist ...
setup(
    name=PKG_NAME,
    version=__version__,
    url=REPO,
    license=LICENSE,
    author=AUTHOR,
    tests_require=TESTING_REQUIRES,
    install_requires=INSTALL_REQUIRES,
    cmdclass={"tests": Tox},
    author_email=AUTHOR_EMAIL,
    description=SHORT_DESC,
    long_description=LONG_DESCRIPTION,
    packages=[PKG_NAME] + [PKG_NAME + "." + item for item in get_pkg_submodules()],
    entry_points=get_entry_points(),
    data_files=DATA_FILES,
    zip_safe=False,
    platforms="any",
    classifiers=[
        "Programming Language :: Python :: {0}".format(pyver)
        for pyver in SUPPORTED_INTERPS
    ]
    + [
        "Development Status :: " + DEVSTAT,
        "Natural Language :: English",
        "Environment :: Console",
        "Intended Audience :: Developers",
        "License :: OSI Approved :: " + LICENSE + " License",
        "Operating System :: OS Independent",
        "Topic :: Software Development :: Libraries :: Python Modules",
    ],
)


from __future__ import absolute_import, division, print_function, unicode_literals
import tensorflow as tf
from helpers import save_img, load_img
from model import StyleTransferModel
import config

# layers
CONTENT_L = ['block5_conv2']

STYLE_L = ['block1_conv1',
           'block2_conv1',
           'block3_conv1',
           'block4_conv1',
           'block5_conv1']

def pre_process_image(path):
    img = load_img(path)
    img = tf.keras.applications.vgg19.preprocess_input(img)
    return img

# Content Loss
def content_loss_f(content, generated):
    return tf.losses.mean_squared_error(content, generated)

##
content_img = load_img(config.CONTENT_IMG)
style_img = load_img(config.STYLE_IMG)

model = StyleTransferModel(CONTENT_L, STYLE_L)

style_targets = model(style_img)['style']
content_target = model(content_img)['content']

num_style_layers = len(STYLE_L)
num_content_layers = len(CONTENT_L)

# Style Loss
def loss_f(outputs):
    style_outputs = outputs['style']
    content_outputs = outputs['content']

    style_loss = tf.add_n([tf.reduce_mean((style_outputs[name] - style_targets[name]) ** 2)
                           for name in style_outputs.keys()])

    style_loss *= config.style_weight / num_style_layers

    content_loss = tf.add_n([content_loss_f(content_target[name], content_outputs[name])
                             for name in content_outputs.keys()])
    content_loss *= config.content_weight / num_content_layers
    total_loss = style_loss + content_loss
    return total_loss

image = tf.Variable(content_img)
optimizer = tf.optimizers.Adam(learning_rate=0.2, beta_1=0.99, epsilon=1e-1)

def high_pass_x_y(tensor):
    x_var = tensor[:, :, 1:, :] - tensor[:, :, :-1, :]
    y_var = tensor[:, 1:, :, :] - tensor[:, :-1, :, :]

    return x_var, y_var

def total_variation_loss(tensor):
    x_deltas, y_deltas = high_pass_x_y(tensor)
    return tf.reduce_mean(x_deltas ** 2) + tf.reduce_mean(y_deltas ** 2)

total_variation_weight = 1e8

@tf.function()
def train_step():
    with tf.GradientTape() as tape:
        outputs = model(image)
        loss = loss_f(outputs)
        loss += total_variation_weight * total_variation_loss(image)

    grad = tape.gradient(loss, image)
    optimizer.apply_gradients([(grad, image)])
    image.assign(tf.clip_by_value(image, clip_value_min=0.0, clip_value_max=1.0))

def train():
    import time
    start = time.time()

    step = 0
    for epoch in range(config.EPOCHS):
        for i in range(config.steps_per_epoch):
            step += 1
            train_step()
            print('.', end="")
        print()
        print("Epoch", epoch, "done")
        save_img(image, config.SAVE_PATH, str(epoch) + " " + config.FILE_NAME)
    end = time.time()

    print("Total time {:.1f}".format(end - start))


import paho.mqtt.client as mqtt
import operator
import numpy as np

import tellopy
import collections

from emg.emglimbo import EMG_Preprocessor
from emg.emglimbo import EMG_Classifier

def handler(event, sender, data, **args):
    global drone
    drone = sender
    if event is drone.EVENT_FLIGHT_DATA:
        print(data)

def stop_drone():
    global drone
    drone.forward(0)
    drone.up(0)
    drone.down(0)
    drone.counter_clockwise(0)
    drone.clockwise(0)

def on_message(client, userdata, message):
    """Callback function
    """
    global memory
    global in_the_air
    global drone
    global preprocessor
    global classifier

    # Decode a message
    message_as_string = str(message.payload.decode("utf-8"))
    message_as_string = ''.join(message_as_string.split())  # remove all white characters

    # Process message
    packets = preprocessor.process(message_as_string)

    # Add to buffer checking if there's enough data to predict
    for packet in packets:
        preprocessor.append_to_buffer(packet)
        if preprocessor.check_buffered_data_size():
            features = preprocessor.get_features()

            # Predict a gesture and print
            prob_dist = classifier.classify(features)

            # Current gesture
            gesture = max(prob_dist.items(), key=operator.itemgetter(1))[0]
            print(f"****** {gesture} ******")

            # Save in memory
            memory.rotate(-1)
            memory[-1] = gesture

            # Take off / land
            if memory.count('pinch thumb-small') == 5:
                if not in_the_air:
                    drone.takeoff()
                    in_the_air = True
                else:
                    drone.land()
                    in_the_air = False

            # Palm landing
            '''
            if memory.count('pinch thumb-ring') == 5:
                if in_the_air:
                    drone.palm_land()
                    in_the_air = False
            '''

            # Move
            dynamism = 30
            if memory[-1] == memory[-2]:
                if memory[-1] == 'fist':
                    drone.forward(dynamism)
                #elif memory[-1] == 'pinch thumb-index':
                #    drone.up(dynamism)
                #elif memory[-1] == 'pinch thumb-middle':
                #    drone.down(dynamism)
                elif memory[-1] == 'flexion':
                    drone.counter_clockwise(dynamism)
                elif memory[-1] == 'extension':
                    drone.clockwise(dynamism)
                else:
                    stop_drone()

# MQTT data
broker_data = {
  "broker_address": "192.168.9.119"
}
topic = "sensors/data/emg"

# File created flag
file_created = False

# Preprocessor
window = 500000
stride = 500000
preprocessor = EMG_Preprocessor(window, stride)

# Create classifier model
model_path = "emg/model.tar"
classifier = EMG_Classifier(model_path, fake=False)

# Create drone
drone = tellopy.Tello()
drone.connect()
drone.wait_for_connection(60.0)
#drone.subscribe(drone.EVENT_FLIGHT_DATA, handler=handler)

# Create memory of last 5 gestures
memory = collections.deque(['idle'] * 5)

# In the air flag
in_the_air = False

# Create a client
print("Creating new subscriber instance")
client = mqtt.Client("emg_sub")

# Type username and password before connecting to broker
# client.username_pw_set(broker_data['username'], broker_data['password'])

# Connect with broker
print("Connecting to broker")
client.connect(broker_data['broker_address'])

# Subscribe to topic
print("Subscribing to topic")
client.subscribe(topic)

# Attach function to callback
client.on_message = on_message

try:
    # Start the loop
    client.loop_forever()

except KeyboardInterrupt:
    # Stop subscriber loop
    client.loop_stop()

    # Disconnect with drone
    drone.quit()


def search_txt_file():

        txt_filename = input("Enter file name: ")
        txt_file = open(txt_filename, 'r')

        the_dic = dict()

        for line in txt_file:
                line = line.strip()
                line = line.lower()
                for word in line.split():
                        the_dic[word] = the_dic.get(word, 0) + 1

        word_to_search = input("Enter the word that you would like to search: ")
        if word_to_search in the_dic:
                return True
        return False


import json
from rest_framework import serializers

from .models import (Task)
from projects.models import Project
from projects.serializers import ProjectSerializer
from patents.models import (Publication, Claim, Element)
from patents.serializers import (PublicationSerializer, ClaimSerializer, ElementSerializer)

class TaskRelatedField(serializers.RelatedField):
    def to_representation(self, value):
        if isinstance(value, Project):
            related_model = ProjectSerializer(value)
            model_type = 'project'

        elif isinstance(value, Publication):
            related_model = PublicationSerializer(value)
            model_type = 'publication'

        elif isinstance(value, Claim):
            related_model = ClaimSerializer(value)
            model_type = 'claim'

        elif isinstance(value, Element):
            related_model = ElementSerializer(value)
            model_type = 'element'

        else:
            raise Exception('Unexpected task type')

        return {'id': related_model.data['id'], 'type': model_type}

class TaskSerializer(serializers.ModelSerializer):
    #related_model = TaskRelatedField(source='content_object', read_only=True)
    class Meta:
        model = Task
        #fields = ('id', 'name', 'description', 'related_model')
        fields = ('id', 'name', 'description')
        read_only_fields = ('id',)


# Title: Lecture 10 Individual Script#
# Lesson: Biot-Savart's Law     #
# Filename: lecture_10.py         #
# Original Author: Joe Griffin  #
# Most Recent Edit: 3/2/2014   #
# Last Editor: PR      #
#################################

# This script calculates the magnetic field of a current-carrying ring at a distance
#   z above the ring, on the central axis of the ring.

import numpy as np
from math import sin, cos, pi
from const import mu0
dtheta = 1e-4                                       # Integral Precision

I = 10.0                                            # Ring Properties I in amp, R in m
R = 0.5

def B(z):                                           # For the moment, we only vary z
    P = np.array((0.0, 0.0, z))
    B = np.array((0.0, 0.0, 0.0))
    for i in xrange(int(2 * pi / dtheta)):
            theta = i * dtheta
            pos = R * np.array((cos(theta),         # Location of differential current
                                sin(theta),
                                0.0))
            r = P - pos
            r_mag = np.linalg.norm(r)
            dl = R * dtheta * np.array((-sin(theta),# Differential wire vector, a positive current flows CCW from the top
                                        cos(theta),
                                        0.0))
            dB = (mu0 * I * np.cross(dl, r) /       # Invoke Biot-Savart Law
                  (4.0 * pi * r_mag**3))
            B = B + dB
    return B

def analyticalB(z):
    B_z = ((mu0 * I * R**2) /
           (2.0 * (z**2 + R**2)**(3 / 2.0)))
    return np.array((0, 0, B_z))


import piexif
import os
import pytesseract
from PIL import Image, ImageFilter
import stat
import re
import glob

def split(list_of_strings, split_on_num):
    out = []
    for element in list_of_strings:
        while len(element) > split_on_num:
            out.append(element[:split_on_num])
            element = element[split_on_num:]
        if len(element) > 0:
            out.append(element)
    return out

def trim_list(list_of_strings, max_length):
    out = []
    for i in list_of_strings:
        if len(i) > max_length:
            i = i[:(len(i) // max_length) * max_length]
        out.append(i)
    return out

# setting Path
pytesseract.pytesseract.tesseract_cmd = r'D:\Coding\ImgPy\Tesseract\tesseract.exe'

# user variables
folder_path = r""
blur = False
grayscale = False
text_to_find = ''

os.chdir(folder_path)
file_list = glob.glob('*.jpg')
for image_path in file_list:
    print(image_path)
    exif_dict = piexif.load(image_path)
    comment_asci = []
    img = Image.open(image_path)
    if grayscale:
        img = img.convert("L")
    if blur:
        img = img.filter(ImageFilter.GaussianBlur(radius=1))
    os.chmod(image_path, stat.S_IWRITE)
    img.save(image_path)
    # ocr
    ocr_text = pytesseract.image_to_string(Image.open(image_path), lang='eng', config="-c tessedit_char_whitelist"
                                                                                "=NO.no1234567890()")
    # process text given by ocr
    ocr_text = ocr_text.lower()
    ocr_text = ocr_text[:-1]
    ocr_text = ocr_text.replace(" ", "")
    ocr_text = ocr_text.strip()
    ocr_text = ocr_text.splitlines()
    comment = ''
    # iterate through lines
    for line in ocr_text:
        # check if text_to_find is in the line
        if text_to_find in line:
            index = line.index(text_to_find[0])
            line = re.sub(r'\([^)]*\)', '', line)
            line = line[index:]
            comment += line + "\n"
    comment = re.sub('[()a-zA-Z.]', '', comment)
    comment = comment.strip()
    comment = comment.splitlines()
    comment = split(comment, 6)
    comment = trim_list(comment, 6)
    comment = "\n".join(comment)
    print(comment)
    # convert into ascii
    for char in comment:
        ascii_char = ord(char)
        comment_asci.append(ascii_char)
        comment_asci.append(0)
    comment_asci.append(0)
    comment_asci.append(0)
    comment_asci = tuple(comment_asci)
    # set it as property in file
    exif_dict['0th'][piexif.ImageIFD.XPComment] = comment_asci
    exif_bytes = piexif.dump(exif_dict)
    piexif.insert(exif_bytes, image_path)


from bottle import (
    route, run, template, request, redirect
)

import string
from scraputils import get_news
from db import News, session
from bayes import NaiveBayesClassifier

@route("/")
@route("/news")
def news_list():
    s = session()
    rows = s.query(News).filter(News.label == None).all()
    return template('news_template', rows=rows)

@route("/add_label/")
def add_label():
    this_label = request.query.label
    this_id = request.query.id
    s = session()
    changing_news = s.query(News).get(this_id)
    changing_news.label = this_label
    s.commit()
    redirect("/news")

@route("/update")
def update_news():
    news_list = get_news("https://news.ycombinator.com/newest", n_pages=4)
    s = session()
    prev_news_list = s.query(News).all()
    for new in news_list:
        for prev_new in prev_news_list:
            if (new['title'] == prev_new.title and
                    new['author'] == prev_new.author):
                break
        else:
            this_new = News(title=new['title'],
                            author=new['author'],
                            url=new['url'],
                            comments=new['comments'],
                            points=new['points'])
            s.add(this_new)
            s.commit()
        prev_news_list = s.query(News).all()
    redirect("/news")

def clean(s):
    translator = str.maketrans("", "", string.punctuation)
    return s.translate(translator)

@route('/recomendations')
def recomendations():
    s = session()
    all_data = []
    data = [[], []]
    length = s.query(News).count()
    for ID in range(1, length + 1):
        post = s.query(News).get(ID)
        all_data.append(post)
    for i in range(1, length):
        data[0].append(all_data[i].title)
        data[1].append(all_data[i].label)
    X, y = [], []
    for i in range(len(data[0])):
        X.append(data[0][i])
        y.append(data[1][i])
    X = [clean(x).lower() for x in X]
    model = NaiveBayesClassifier()
    part = len(X)*7 // 10
    X_train, y_train, X_test, y_test = X[:part], y[:part], X[part:], y[part:]
    model.fit(X_train, y_train)
    score=model.score(X_test, y_test)
    rec = NaiveBayesClassifier(alpha=0.01)
    rec.fit(X, y)
    rows = s.query(News).filter(News.label == None).all()
    X_new = []
    for row in rows:
        X_new.append(row.title)
    classified_news = rec.predict(X_new)
    return template('news_recomendations', rows=classified_news, score=score)

if __name__ == "__main__":
    run(host="localhost", port=8080)


# %load q04_encoding/build.py
# Default imports
import pandas as pd
from sklearn.preprocessing import LabelEncoder

ny_housing = pd.read_csv('data/train.csv')
housing_data = ny_housing[['MasVnrArea', 'GrLivArea', 'LotShape', 'GarageType', 'SalePrice']]

# Write your code here:
def encoding(housing_data):
    columnsToEncode = list(housing_data.select_dtypes(include=['category','object']))
    le=LabelEncoder()
    housing_data['newLotShape']=le.fit_transform(housing_data[columnsToEncode[0]])
    ld=pd.get_dummies(housing_data[columnsToEncode[1]], drop_first=True)
    frames = [housing_data,ld]
    result = pd.concat(frames,axis=1)
    return result


from __future__ import print_function
from ortools.constraint_solver import routing_enums_pb2
from ortools.constraint_solver import pywrapcp
import json
import requests
import time

API_KEY = ""

# load destination data for today's deliveries
# currently load from a text file
def get_destination_list():
    with open("destinationlist.json") as file:
        data = json.load(file)
        return data

# use openrouteservice.org to get distance between all point of insterest
# currently we are getting geocode by sending each location one by one. This is very slow
# need to improve
def request_distance_data(locations):
    # make request to obtain the geocode of a address
    def get_geocode(address):
        #prepare request
        template = 'https://api.openrouteservice.org/geocode/search?api_key=' + API_KEY
        text = '&text=' + address

        #send request
        r = requests.get(template + text)
        # print("test: status code is " + str(r.status_code))

        # read results
        respondJson = json.loads(r.text)

        # we are picking the first result when there's mulitple search result
        # need to improve in future
        return respondJson['features'][0]["geometry"]["coordinates"]

    def get_travel_time(geocode1, geocode2):
        # prepare request
        template = 'https://api.openrouteservice.org/v2/directions/driving-car?api_key='  + API_KEY
        start = '&start=' + str(geocode1[0]) +','+ str(geocode1[1])
        end = '&end=' + str(geocode2[0]) +',' + str(geocode2[1])

        # print(template+start+end)
        #send request
        r = requests.get(template + start + end)
        print("test: status code is " + str(r.status_code))
        if(r.status_code!=200):
            # print(r.text)
            return 10000000
        #read results
        respondJson = json.loads(r.text)
        # print(json.dumps(respondJson))

        if(respondJson):
            if(respondJson['features'][0]['properties']['summary'] == {}):
                # this happens when the starting location is the end location
                return 0
            return int(respondJson['features'][0]['properties']['summary']['duration'])
        else:
            print(json.dumps(respondJson))
            return 10000000

    print("requesting geocodes...")
    geocodeList = []
    for location in locations:
        geocodeList.append(get_geocode(location['pick-up-location']))
        geocodeList.append(get_geocode(location['drop-off-location']))

    data = {}

    # construct pickups-deliveries table
    data['pickups_deliveries'] = []

    for i in range(0,len(geocodeList),2):
        data['pickups_deliveries'] .append([i,i+1])

    print("requesting distances...")
    # construct adjacency matrix
    data['distance_matrix'] = []
    for geocode1 in geocodeList:
        data['distance_matrix'].append([])
        for geocode2 in geocodeList:
            data['distance_matrix'][-1].append(get_travel_time(geocode1,geocode2))
            time.sleep(1.5) # open route service has a 40 time/min request cap

    data['num_vehicles'] = 1
    data['depot'] = 0

    return data

def print_solution(data, manager, routing, solution):
    """Prints solution on console."""
    total_distance = 0
    for vehicle_id in range(data['num_vehicles']):
        index = routing.Start(vehicle_id)
        plan_output = 'Route for vehicle {}:\n'.format(vehicle_id)
        route_distance = 0
        while not routing.IsEnd(index):
            plan_output += ' {} -> '.format(manager.IndexToNode(index))
            previous_index = index
            index = solution.Value(routing.NextVar(index))
            route_distance += routing.GetArcCostForVehicle(
                previous_index, index, vehicle_id)
        plan_output += '{}\n'.format(manager.IndexToNode(index))
        plan_output += 'Distance of the route: {}m\n'.format(route_distance)
        print(plan_output)
        total_distance += route_distance
    print('Total Distance of all routes: {}m'.format(total_distance))

def main():
    with open("cred.json") as file:
        API_KEY = json.load(file)['API_KEY']

    # destinationlist = get_destination_list()
    # data = request_distance_data(destinationlist)

    with open('temp.json') as file:
        data = json.load(file)

    # print(json.dumps(data))
    # exit()

     # Create the routing index manager.
    manager = pywrapcp.RoutingIndexManager(len(data['distance_matrix']),
                                           data['num_vehicles'], data['depot'])

    # Create Routing Model.
    routing = pywrapcp.RoutingModel(manager)

    # Define cost of each arc.
    def distance_callback(from_index, to_index):
        """Returns the manhattan distance between the two nodes."""
        # Convert from routing variable Index to distance matrix NodeIndex.
        from_node = manager.IndexToNode(from_index)
        to_node = manager.IndexToNode(to_index)
        # return data['distance_matrix'][from_node][to_node]
        return data['distance_matrix'][from_index][to_index]

    transit_callback_index = routing.RegisterTransitCallback(distance_callback)
    routing.SetArcCostEvaluatorOfAllVehicles(transit_callback_index)

    # Add Distance constraint.
    dimension_name = 'Distance'
    routing.AddDimension(
        transit_callback_index,
        0,  # no slack
        300000,  # vehicle maximum travel distance
        True,  # start cumul to zero
        dimension_name)
    distance_dimension = routing.GetDimensionOrDie(dimension_name)
    distance_dimension.SetGlobalSpanCostCoefficient(100)

    # Define Transportation Requests.
    for request in data['pickups_deliveries']:
        pickup_index = manager.NodeToIndex(request[0])
        delivery_index = manager.NodeToIndex(request[1])
        routing.AddPickupAndDelivery(pickup_index, delivery_index)
        routing.solver().Add(
            routing.VehicleVar(pickup_index) == routing.VehicleVar(
                delivery_index))
        routing.solver().Add(
            distance_dimension.CumulVar(pickup_index) <=
            distance_dimension.CumulVar(delivery_index))

    # Setting first solution heuristic.
    search_parameters = pywrapcp.DefaultRoutingSearchParameters()
    search_parameters.first_solution_strategy = (
        routing_enums_pb2.FirstSolutionStrategy.PARALLEL_CHEAPEST_INSERTION)

    # Solve the problem.
    solution = routing.SolveWithParameters(search_parameters)

    # Print solution on console.
    if solution:
        print_solution(data, manager, routing, solution)

if __name__ == '__main__':
    main()


from __future__ import absolute_import
import datetime
import logging
from celery.schedules import crontab
from celery.task import periodic_task
from django.conf import settings
from django.db.models import Q
from eums.celery import app
from eums.models import Run, RunQueue, Runnable, DistributionPlan, DistributionPlanNode, Flow
from eums.rapid_pro.rapid_pro_service import rapid_pro_service
from eums.services.delivery_run_message import DeliveryRunMessage
from eums.models.alert import Alert

logger = logging.getLogger(__name__)

def schedule_run_for(runnable):
    if _should_schedule(runnable):
        _cancel_current_run(runnable)
        run_delay = _calculate_delay(runnable)
        schedule_run_directly_for(runnable, run_delay)

def schedule_run_directly_for(runnable, run_delay):
    if Run.has_scheduled_run(runnable.contact_person_id):
        RunQueue.enqueue(runnable, run_delay)
    elif settings.RAPIDPRO_LIVE:
        task = _schedule_run.apply_async(args=[runnable.id], countdown=run_delay)
        Run.objects.create(scheduled_message_task_id=task.id, runnable=runnable,
                           status=Run.STATUS.scheduled, phone=runnable.contact.phone)

def _should_schedule(runnable):
    return runnable.completed_run() is None

@app.task
def _schedule_run(runnable_id):
    runnable = Runnable.objects.get(id=runnable_id)
    message = DeliveryRunMessage(runnable)
    rapid_pro_service.create_run(runnable.build_contact(), runnable.flow(),
                                 message.description(), message.sender_name())

def _calculate_delay(runnable):
    expected_delivery_date = datetime.datetime.combine(runnable.delivery_date,
                                                       datetime.datetime.min.time())
    when_to_send_message = expected_delivery_date + datetime.timedelta(days=settings.DELIVERY_STATUS_CHECK_DELAY)
    delay_in_seconds = (when_to_send_message - datetime.datetime.now()).total_seconds()
    return delay_in_seconds if delay_in_seconds > 0 else settings.DELIVERY_BUFFER_IN_SECONDS

def _cancel_current_run(runnable):
    current_run = runnable.current_run()
    if current_run:
        _cancel(current_run)

def _cancel(run):
    app.control.revoke(run.scheduled_message_task_id)
    run.update_status(Run.STATUS.cancelled)

@periodic_task(run_every=crontab(minute=0, hour=0))
def expire_overdue_runs():
    overdue_runs = Run.overdue_runs()
    for overdue_run in overdue_runs:
        overdue_run.update_status(Run.STATUS.expired)
        next_run = RunQueue.dequeue(overdue_run.runnable.contact_person_id)
        if next_run:
            schedule_run_for(next_run.runnable)
            next_run.update_status(RunQueue.STATUS.started)

@periodic_task(run_every=crontab(minute=0, hour=0))
def distribution_alert_raise():
    distribution_plans = DistributionPlan.objects.all()
    for distribution_plan in distribution_plans:
        runnable = Runnable.objects.get(id=distribution_plan.runnable_ptr_id)
        if is_distribution_expired_alert_not_raised(runnable) \
                and is_shipment_received_but_not_distributed(distribution_plan) \
                and is_distribution_expired(distribution_plan):
            logger.info('Raise alert')
            runnable.create_alert(Alert.ISSUE_TYPES.distribution_expired)

def is_distribution_expired_alert_not_raised(runnable):
    logger.info('is_distribution_expired_alert_not_raised:' + str(
        not Alert.objects.filter(issue=Alert.ISSUE_TYPES.distribution_expired,
                                 runnable=runnable)))
    return not Alert.objects.filter(issue=Alert.ISSUE_TYPES.distribution_expired,
                                    runnable=runnable)

def is_shipment_received_but_not_distributed(distribution_plan):
    runnable_id = distribution_plan.runnable_ptr_id
    not_distributed = DistributionPlanNode.objects.filter(Q(distribution_plan_id=runnable_id) & (
        Q(tree_position=Flow.Label.MIDDLE_MAN) | Q(tree_position=Flow.Label.END_USER))).count() == 0
    logger.info(
        'is_shipment_received_but_not_distributed:' + str(
            distribution_plan.shipment_received() and not_distributed))
    return distribution_plan.shipment_received() and not_distributed

def is_distribution_expired(distribution_plan):
    time_limitation = distribution_plan.time_limitation_on_distribution
    date_received_str = distribution_plan.received_date()
    if time_limitation and date_received_str:
        date_received = datetime.datetime.strptime(date_received_str.split('T')[0], '%Y-%m-%d').date()
        logger.info(
            'is_distribution_expired:' + str((datetime.date.today() - date_received).days - 2 > time_limitation))
        if (datetime.date.today() - date_received).days - 2 >= time_limitation:
            return True
    return False


from telegram.ext import MessageHandler, Filters

import topics

def gram_dist(a, b): # Расстояние Дамерау-Левенштайна для анализа грам.ошибок
    n, m = len(a), len(b)
    if n > m:
        a, b = b, a
        n, m = m, n

    current_column = range(n+1)
    for i in range(1, m+1):
        previous_column, current_column = current_column, [i]+[0]*n
        for j in range(1, n + 1):
            add, delete, change = previous_column[j] + 1, current_column[j-1] + 1, previous_column[j-1]
            if a[j-1] != b[i-1]:
                change += 1
            current_column[j] = min(add, delete, change)

    return current_column[n] / n

def text_reply(update, bot):
    text_message = update.message.text
    for greeting in topics.greet_list:
        if gram_dist(greeting, text_message.lower()) <= 0.3:
            return greetings(update, bot)
    return bot.bot.sendMessage(chat_id=update.message.chat_id, text=text_message)

def greetings(update, bot):
    bot.bot.sendMessage(
        chat_id=update.message.chat_id,
        text=f"Здравствуй, { update.message.from_user.first_name }"
        )

text_handler = MessageHandler(Filters.text, text_reply)


#!/usr/bin/env python
import pygame

class Button(pygame.sprite.Sprite):
    def __init__(self, normal, above, onclick, rect):
        pygame.sprite.Sprite.__init__(self)
        self.normal = normal
        self.above = above
        self.onclick = onclick
        self.rect = rect
        self.image = self.normal

    def update(self):
        cur_pos = pygame.mouse.get_pos()
        mouse_pressed = pygame.mouse.get_pressed()
        if self.rect.collidepoint(cur_pos):
            if mouse_pressed[0]:
                self.image = self.onclick
            else:
                self.image = self.above
        else:
            self.image = self.normal

    def draw(self, screen):
        screen.blit(self.image, self.rect)

    def isclicked(self):
        cur_pos = pygame.mouse.get_pos()
        mouse_pressed=pygame.mouse.get_pressed()
        if self.rect.collidepoint(cur_pos) and mouse_pressed[0]:
            return True
        else:
            return False

    def contains(self, x, y):
        cur_pos = (x, y)
        if self.rect.collidepoint(cur_pos):
            return True
        else:
            return False


# Copyright 2017 The TensorFlow Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Utilities for defining TensorFlow Bazel dependencies."""

load("@bazel_tools//tools/build_defs/repo:utils.bzl", "patch")

def _http_archive_impl(ctx):
    override = ctx.attr.override
    if override != None and override in ctx.os.environ:
        repo_path = ctx.os.environ[override]
        print("using override: {}={}".format(override, repo_path))
        ctx.symlink(repo_path, ctx.path("."))
        for src, tgt in ctx.attr.link_files.items():
            ctx.delete(ctx.path(tgt))
    else:
        ctx.download_and_extract(
            ctx.attr.url,
            "",
            ctx.attr.sha256,
            ctx.attr.type,
            ctx.attr.strip_prefix,
        )
        if ctx.attr.delete:
            for path in ctx.attr.delete:
                ctx.delete(ctx.path(path))
        patch(ctx)
    if ctx.attr.build_file != None:
        # Use "BUILD.bazel" to avoid conflict with third party projects that contain a
        # file or directory called "BUILD"
        buildfile_path = ctx.path("BUILD.bazel")
        ctx.symlink(ctx.attr.build_file, buildfile_path)
    for src, tgt in ctx.attr.link_files.items():
        ctx.symlink(Label(src), ctx.path(tgt))

# Downloads and creates Bazel repos for dependencies.
#
# For link_files, specify each dict entry as:
# "//path/to/source:file": "localfile"
http_archive = repository_rule(
    implementation = _http_archive_impl,
    attrs = {
        "sha256": attr.string(mandatory = True),
        "url": attr.string(mandatory = True),
        "strip_prefix": attr.string(),
        "type": attr.string(),
        "delete": attr.string_list(),
        "build_file": attr.label(),
        "link_files": attr.string_dict(),
        "override": attr.string(),
        "patches": attr.label_list(default = []),
        "patch_tool": attr.string(default = ""),
        "patch_args": attr.string_list(default = ["-p1"]),
        "patch_cmds": attr.string_list(default = []),
        "patch_cmds_win": attr.string_list(default = []),
    },
)


# Submitter: mmwong2 (Wong, Michelle)
# Partner  : vcphung(Phung, Victor)
# We certify that we worked cooperatively on this programming
#   assignment, according to the rules for pair programming

from collections import defaultdict
from goody import type_as_str  # Useful in some exceptions

class DictList:
    def __init__(self, *args):
        l = []
        if args == ():
            raise AssertionError('There are no dictionaries')
        for element in args:
            if type(element) != dict:
                raise AssertionError('Some arguments are not dictionaries')
            elif element == dict():
                raise AssertionError('Some dictionaries are empty')
            l.append(element)
        self.dl = list(l)

    def __len__(self)->int:
        d = dict()
        for dictionary in self.dl:
            d.update(dictionary)
        return len(list(d))

    def __bool__(self)->bool:
        #returns true if self.dl has more than 1 dictionary
        if len(self.dl) == 1:
            return False
        return True

    def __repr__(self)->str:
        s = 'DictList('
        for i in self.dl:
            s += str(i) + ','
        s = s[:-1]
        s += ')'
        return s

    def __contains__(self, key:str)->bool:
        for d in self.dl:
            for element in d:
                if element == key:
                    return True
        return False

    def __getitem__(self, key:str)->int:
        temp = self.dl
        for d in reversed(temp):
            for element in d:
                if key == element:
                    return d[key]
        raise KeyError("'"+str(key)+ "' appears in no dictionaries")

    def __setitem__(self, key:str, newvalue:str):
        temp = self.dl
        for d in reversed(temp):
            for element in d:
                if key == element:
                    d[key]=newvalue
                    return
        self.dl.append({key:newvalue})

    def __delitem__(self,key:str):
        flag = 0
        if self.__contains__(key):
            newdl = []
            temp = self.dl
            for d in reversed(temp):
                newd = dict()
                for element in d:
                    if key != element and flag == 0:
                        newd[element]= d[element]
                    elif key == element and flag == 0:
                        flag = 1
                    else:
                        newd[element] = d[element]
                newdl.insert(0,newd)
            for d in newdl:
                if d == {}:
                    newdl.remove(d)
            self.dl = newdl
        else:
            raise KeyError("'"+str(key)+ "' appears in no dictionaries")

    def __call__(self,key:str):
        list1 = []
        for d in self.dl:
            for element in d:
                if key == element:
                    list1.append((self.dl.index(d),d[key]))
        return list1

    def collapse(self):
        newd = {}
        for d in self.dl:
            newd.update(d.items())
        return newd

    def __iter__(self):
        called_keys = []
        for d in reversed(self.dl):
            inner = []
            for key in d:
                if key not in called_keys:
                    inner.append(key)
            inner.sort()
            called_keys.extend(inner)
        for i in called_keys:
            yield i

    def items(self):
        for i in self:
            yield (i, self[i])

    def __eq__(self,right):
        left = {}
        right1 = {}
        if type(right) != dict and type(right) != DictList:
            raise TypeError('Its not a list or dict')

        for d in self.dl:
            for element in d:
                left[element] = d.get(element)
        if type(right) == DictList:
            for d2 in right.dl:
                for element2 in d2:
                    right1[element2] = d2.get(element2)
            if len(left) != len(right1):
                return False
            for k in left:
                if k not in right1:
                    return False
                if left[k] != right1[k]:
                    return False
        else:

            for k in left:
                if k not in right:
                    return False
                if left[k] != right[k]:
                    return False
        return True

    def __lt__(self,right):
        left = {}
        right1 = {}
        if type(right) != dict and type(right) != DictList:
            raise TypeError('Its not a list or dict')

        for d in self.dl:
            for element in d:
                left[element] = d.get(element)
        if type(right) == DictList:
            for d3 in right.dl:
                for element2 in d3:
                    right1[element2] = d3.get(element2)
            if len(left) >= len(right1):
                return False

            for k in left:
                if k not in right1:
                    return False
                if left[k] != right1[k]:
                    return False
        else:
            if len(left) >= len(right):
                return False
            for k in left:
                if k not in right:
                    return False
                if left[k] != right[k]:
                    return False

        return True

    def __gt__(self,right):
        left = {}
        right1 = {}
        if type(right) != dict and type(right) != DictList:
            raise TypeError('Its not a list or dict')
        for d in self.dl:
            for element in d:
                left[element] = d.get(element)
        if type(right) == DictList:
            for d3 in right.dl:
                for element2 in d3:
                    right1[element2] = d3.get(element2)
            if len(left) <= len(right1):
                return False
            for k in right1:
                if k not in left:
                    return False
                if left[k] != right1[k]:
                    return False
        else:
            if len(left) <= len(right):
                return False

            for k in right1:
                if k not in left:
                    return False
                if left[k] != right[k]:
                    return False

        return True

    def __add__(self,right):
        newDict = []
        left = {}
        right1 = {}
        if type(right) != dict and type(right) != DictList:
            raise TypeError('Its not a list or dict')
        for d in self.dl:
            newDict.append(d)
        if type(right) == DictList:
            for d2 in right:
                newDict.append(d2)
        else:
            newDict.append(right)
        return newDict
'''
    def __setattr__(self):
        pass
'''
if __name__ == '__main__':
    #Simple tests before running driver
    #Put your own test code here to test DictList before doing bsc tests

    #d = DictList(dict(a=1,b=2), dict(b=12,c=13))
    '''
    print('len(d): ', len(d))
    print('bool(d):', bool(d))
    print('repr(d):', repr(d))
    print(', '.join("'"+x+"'" + ' in d = '+str(x in d) for x in 'abcx'))
    print("d['a']:", d['a'])
    print("d['b']:", d['b'])
    print("d('b'):", d('b'))
    print('iter results:', ', '.join(i for i in d))
    print('item iter results:', ', '.join(str(i) for i in d.items()))
    print('d.collapse():', d.collapse())
    print('d==d:', d==d)
    print('d+d:', d+d)
    print('(d+d).collapse():', (d+d).collapse())

    print()
    '''
    import driver
    driver.default_file_name = 'bsc1.txt'
#     driver.default_show_exception=True
#     driver.default_show_exception_message=True
#     driver.default_show_traceback=True
    driver.driver()


from web3 import Web3, HTTPProvider
import sys
import os
import json
import time
from pprint import pprint

w3 = Web3(HTTPProvider("http://37.187.153.186:8545",request_kwargs={'timeout':60})) #Geh Mainnet node

with open('./private-key.txt') as keyfile:
    metaKey = keyfile.read()
metaAccount = w3.eth.account.from_key(metaKey)

with open("./eth/build/contracts/Pixel.json") as f:
    pixelContract = json.load(f)

#gas value to set

Pixel = w3.eth.contract(abi=pixelContract['abi'], bytecode=pixelContract['bytecode'])
tx_dict = Pixel.constructor().buildTransaction({
    'from' : metaAccount.address,
    'nonce' : w3.eth.getTransactionCount(metaAccount.address)
})
tx = w3.eth.account.signTransaction(tx_dict, metaAccount.key)
result = w3.eth.sendRawTransaction(tx.rawTransaction)
print(result.hex())
txReceipt = w3.eth.waitForTransactionReceipt(result)
pprint(txReceipt)
print(txReceipt['contractAddress'])


import os
import sys
from datetime import datetime

from blessings import Terminal

from config import loadConfig
from utils import run, create_virtualenv, install_requirements
from test_data import TEST_TRIANGLES

term = Terminal()

BANDIT_EXECUTABLE = os.path.join(os.path.dirname(sys.executable), 'bandit')

class StopExecution(Exception):
    pass

class Result(object):
    def __init__(self,
                 name,
                 time=None,
                 failures=None,
                 solution_directory=None):
        self.name = name
        self.time = time
        self.failures = failures or []
        self.solution_directory = solution_directory

    @property
    def has_failures(self):
        return bool(self.failures)

    def __str__(self):
        if not self.has_failures:
            return '{name} completed successfully in {time}'.format(name=self.name,
                                                                    time=self.time)
        else:
            return '{name} failed for {failures}'.format(name=self.name,
                                                         failures='\n'.join(self.failures))

    def write_failures_to_file(self):
        if not self.failures:
            print(term.yellow('No failure to write'))
        elif not self.solution_directory:
            print(term.yellow('Cannot write error file. No directory specified'))
        else:
            full_path = os.path.join(self.solution_directory, 'error.txt')
            with open(full_path, 'w') as f:
                f.write('\n'.join(self.failures))

    def add_failure(self, failure):
        self.failures.append(failure)

def run_all():
    config = loadConfig('settings.conf')
    LOCAL_DIRECTORY = config.get('ATTACHMENTS', 'LOCAL_DIRECTORY')
    EXECUTABLE = config.get('EXECUTION', 'EXECUTABLE')
    STOP_ON_FIRST_FAILURE = config.getboolean('EXECUTION', 'STOP_ON_FIRST_FAILURE')

    solutions = os.listdir(LOCAL_DIRECTORY)

    results = []

    for solution in solutions:
        print('Running {}'.format(term.blue(solution)))

        path = os.path.join(LOCAL_DIRECTORY, solution)

        result = Result(solution,
                        solution_directory=path)

        try:
            check_bandit(path)
        except StopExecution:
            raise
        except Exception as e:
            print(term.red('Got exception running {}'.format(solution)))
            print(term.red(str(e)))
            failure_message = str(e)

            if hasattr(e, 'stdout') and e.stdout:
                print(term.red(e.stdout))
                failure_message = failure_message + '\nFrom stdout:\n{stdout}'.format(stdout=e.stdout)

            result.add_failure(failure_message)
            results.append(result)
            continue

        venv_path = create_virtualenv(path)
        install_requirements(venv_path, path)

        start_time = datetime.now()
        for test in TEST_TRIANGLES:
            try:
                failure_message = check_triangle(path,
                                                 python_executable=os.path.join(venv_path, 'bin', 'python3'),
                                                 executable=EXECUTABLE,
                                                 **test)

                if failure_message:
                    result.add_failure(failure_message)

                    if STOP_ON_FIRST_FAILURE:
                        break
            except StopExecution:
                raise
            except Exception as e:
                print(term.red('ERROR:'))
                print(term.red('Got exception running {}'.format(solution)))
                print(term.red(str(e)))
                failure_message = str(e)

                if hasattr(e, 'stdout') and e.stdout:
                    print(term.red(e.stdout))
                    failure_message = failure_message + '\nFrom stdout:\n{stdout}'.format(stdout=e.stdout)

                result.add_failure(failure_message)

                if STOP_ON_FIRST_FAILURE:
                    break

        finish_time = datetime.now()
        result.time = finish_time - start_time
        results.append(result)

        print()

    return results

def check_bandit(path):
    run(path,
        '{} -r {}'.format(BANDIT_EXECUTABLE, path),
        suppress_output=True,
        )

def check_triangle(path,
        python_executable='python3',
        executable='traversal.py',
        filename=None,
        expected=None,
        conversion_func=None):
    # filename and expected are actually required args for the function but I define them
    # as kwargs to allow ** unpacking of the test values
    if not filename:
        raise StopExecution('Test solutions have been improperly defined: filename is missing')
    if not expected:
        raise StopExecution('Test solutions have been improperly defined: expected is missing')

    data_directory = os.path.join(
            os.path.abspath(os.path.dirname(__file__)),
            'data')

    return run(path,
               '{} {} {}'.format(python_executable,
                                 executable,
                                 os.path.join(data_directory, filename)),
               executable=executable,
               expected=expected,
               conversion_func=conversion_func)

if __name__ == '__main__':
    run_all()


#!/anaconda3/bin/python

import sys

class ListNode:
    def __init__(self, x):
        self.val = x
        self.next = None

class Solution:
    def mergeTwoLists(self, l1, l2):
        if not l1 and not l2:
            return None
        elif not l1:
            return l2
        elif not l2:
            return l1
        else:
            if l1.val > l2.val:
                l1, l2 = l2, l1
            l1.next = self.mergeTwoLists(l1.next,l2)
        return l1

if __name__ == "__main__":
    a = Solution()
    l1a = ListNode(1)
    l2a = ListNode(1)

    l1a.next = ListNode(2)
    l1a.next.next = ListNode(4)

    l2a.next = ListNode(3)
    l2a.next.next = ListNode(4)

    a.mergeTwoLists(l1a,l2a)


import numpy as np

from util import param_type

class TestParameterCreation(object):

    def test_transform_array_to_paramstype(self):
        test_array = np.ones(53)
        test_params = param_type.transform_array_to_paramstype(test_array)
        necessary_attributes = ['beta0', 'ften0', 'oten0',
                                'lambda_e', 'lambda_u', 'fire_rate',
                                'offer_probs', 'fixed_gamma', 'mincer_gamma',
                                'thetatypes', 'mincer_rates']
        for attribute in necessary_attributes:
            assert hasattr(test_params, attribute)

    def test_transform_fixed_gangles_to_gammas(self):
        g_angles = np.linspace(0,np.pi/2, num=9)[1:-1]
        gamma_frontier_curvature = 2.0
        test = param_type.transform_fixed_gangles_to_gammas(g_angles,
                                                            gamma_frontier_curvature)
        test = np.delete(test, [1,6], axis=1)
        assert np.allclose(np.sum(test**2,axis=0), 1.0)
        assert np.allclose(np.arccos(test[0, :]), g_angles)
        assert np.allclose(np.arcsin(test[1, :]), g_angles)


import random

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.multiprocessing import Pool
from torch.nn.init import xavier_normal_, xavier_uniform_
from torchsummary import summary

from preprocess import print_table
from utils import flatten_1_deg, loadpkl, savepkl

torch.multiprocessing.set_start_method('spawn', force=True)

class Table2Vec(nn.Module):

    def __init__(self, vocab_size, embedding_dim, device, config):
        super(Table2Vec, self).__init__()
        self.device = device
        self.table_prep_params = config['table_prep_params']
        self.embeddings = nn.Embedding(vocab_size, embedding_dim)
        self.embeddings.weight.requires_grad = True
        xavier_normal_(self.embeddings.weight.data)

        self.conv1 = nn.Conv2d(embedding_dim, 128, (5, 4), 2, 0)
        self.mp1 = nn.MaxPool2d((2, 2), 2)
        self.bn1 = nn.BatchNorm2d(128)
        self.drop1 = nn.Dropout(0.3)
        self.drop2 = nn.Dropout(0.2)
        self.fc1 = nn.Linear(128*11*2, 128)
        self.fc2 = nn.Linear(128, 1)

    def get_rand_table(self):
        rand_table_name = random.choice(range(len(self.all_tables)))
        t_d = self.all_tables[rand_table_name].tolist()
        # print(type(self.all_tables), type(t_d))

        # numDataRows, numCols = np.array(t_d).shape[:2]
        # while numDataRows == 0 or numCols == 0:
        #     # print('sample table again coz 0')
        #     t_d, rand_table_name = self.get_rand_table()
        return t_d, rand_table_name

    def generate_rand_cell(self, table, table_lst):
        table_data, table_name = self.get_rand_table()
        # print(type(table_data))
        numDataRows, numCols = np.array(table_data).shape[:2]
        rand_row_ix = random.choice(list(range(numDataRows)))
        rand_col_ix = random.choice(list(range(numCols)))
        rand_cell = table_data[rand_row_ix][rand_col_ix]
        # print(type(rand_cell))

        # while (
        #     # True
        #     # rand_cell in flatten_1_deg(table) or
        #     # table_name in table_lst or
        #     rand_cell.count(1) == len(rand_cell)
        # ):
        #     # if table_name in table_lst:
        #     #     return self.generate_rand_cell(table, table_lst)
        #     # elif rand_cell.count(1) == len(rand_cell):
        #     #     rand_row_ix = random.choice(list(range(numDataRows)))
        #     #     rand_col_ix = random.choice(list(range(numCols)))
        #     #     rand_cell = table_data[rand_row_ix][rand_col_ix]
        #     # else:
        #     #     break
        #     # print('sample table again coz repeat')
        #     rand_cell, table_name = self.generate_rand_cell(table, table_lst)

        return rand_cell, table_name

    def generate_neg_table(self, _):
        t_l = []
        t = []

        rand_rows = random.choice(
            list(range(3, self.table_prep_params['MAX_ROW_LEN'])))
        rand_cols = random.choice(
            list(range(2, self.table_prep_params['MAX_COL_LEN'])))
        # rand_rows = self.table_prep_params['MAX_ROW_LEN']
        # rand_cols = self.table_prep_params['MAX_COL_LEN']

        for i in range(rand_rows):
            r = []
            for j in range(rand_cols):
                c, t_name = self.generate_rand_cell(t, t_l)
                r.append(c)
                t_l.append(t_name)
            t.append(r)
        t = self.pad_table(t)
        # print(_)
        return t

    def pad_table(self, table):
        for row in table:
            for j in range(0, self.table_prep_params['MAX_COL_LEN']-len(row)):
                row.append([1]*self.table_prep_params['LENGTH_PER_CELL'])
        for i in range(0, self.table_prep_params['MAX_ROW_LEN']-len(table)):
            table.append([[1]*self.table_prep_params['LENGTH_PER_CELL']]
                         * self.table_prep_params['MAX_COL_LEN'])
        return table

    def generate_neg(self, size):
        Xn = [self.generate_neg_table(i) for i in range(size)]
        # with Pool(40) as p:
        #     Xn = [tqdm(p.imap(self.generate_neg_table, range(size)), total=size)]
        # p = Pool(processes=20)
        # Xn = p.map(self.generate_neg_table, range(size))
        # p.close()
        # p.join()

        yn = [0]*size
        Xn = torch.tensor(Xn, device=self.device)
        yn = torch.tensor(yn, dtype=torch.float, device=self.device)
        return Xn, yn.reshape((-1, 1))

    def forward(self, inputs, y_true):
        self.all_tables = inputs
        Xn, yn = self.generate_neg(len(inputs))
        total_inputs = torch.cat((inputs, Xn), dim=0)
        total_y = torch.cat((y_true, yn), dim=0)

        emb = self.embeddings(total_inputs)
        emb = emb.mean(3)
        emb = emb.permute(0, 3, 1, 2)

        x = self.conv1(emb)
        # x = self.bn1(x)
        x = self.mp1(x)
        x = self.drop1(x)
        x = F.relu(x)
        x = x.view(x.shape[0], -1)
        x = self.fc1(x)
        # x = self.drop2(x)
        x = F.relu(x)
        x = self.fc2(x)
        pred = torch.sigmoid(x)
        return pred, total_y


import csv
import matplotlib.pyplot as plt

times = []
vel = []
ang = []
with open('inputs.csv') as csvfile:
    readCSV = csv.reader(csvfile, delimiter=',')
    for row in readCSV:
        if times == []:
            t0 = float(row[0])
        times.append(float(row[0]) - t0)
        vel.append(float(row[1]))
        ang.append(float(row[2]))

plt.subplot(1, 2, 1)
plt.scatter(times, vel, color = 'r')
plt.plot(times, vel, color = 'r')
plt.title('Velocity Input')
plt.xlabel('time')
plt.ylabel('velocity')
plt.subplot(1, 2, 2)
plt.scatter(times, ang, color = 'r')
plt.plot(times, ang, color = 'r')
plt.title('Angular Velocity Input')
plt.xlabel('time')
plt.ylabel('angular velocity')
plt.show()