# -*- coding: utf-8 -*-
"""
Created on Tue Nov 28 19:25:11 2017

@author: Bird
这个文件完成模型的训练过程
"""
#%%包文件导入区
import pickle
import matplotlib.pyplot as plt
import numpy as np
import sklearn
from sklearn.model_selection import train_test_split

from keras.models import Sequential
from keras.layers import Dense, Activation, Flatten, Lambda
from keras.layers import Conv2D, Cropping2D

#%%函数定义区
def GetFromPickle(FileName):
    '''
    调入存储的数据文件
    '''
    pickle_file = './Data/Pretreatment' + FileName
    with open(pickle_file, 'rb') as f:
        pickle_data = pickle.load(f)
        Images = pickle_data['Images']
        steering_angle = pickle_data['steering_angle']

    return Images, steering_angle

def RandomZeroData(angle_In):
    '''
    针对训练数据里的零数据，通过增加噪声额的方法避免为0
    '''
    angle_Out = []
    for Counter in range(len(angle_In)):
        A_steering_angle = angle_In[Counter]
        if A_steering_angle == 0:
            angle_Out.append(A_steering_angle + 0.01 * np.random.randn())
        else:
            angle_Out.append(A_steering_angle)
    return angle_Out

def generator(Images, steering_angle, batch_size=128):
    #利用生成器为每一个训练集增加不同的噪声
    num_samples = len(steering_angle)
    while 1: # Loop forever so the generator never terminates
        Images, steering_angle = sklearn.utils.shuffle(Images, steering_angle)
        for offset in range(0, num_samples, batch_size):
            X_train = Images[offset:offset+batch_size]
            y_train = steering_angle[offset:offset+batch_size]

            y_train = y_train + 0.01 * np.random.randn(len(y_train))

            yield X_train, y_train

#%%调入训练数据
Left_Images, Left_steering_angle = GetFromPickle('Left')
Left_Images_Flip, Left_steering_angle_Flip = GetFromPickle('Left_Flip')
Right_Images, Right_steering_angle = GetFromPickle('Right')
Right_Images_Flip, Right_steering_angle_Flip = GetFromPickle('Right_Flip')

Images = np.concatenate((Left_Images,Left_Images_Flip,Right_Images,Right_Images_Flip),axis=0)
steering_angle = np.concatenate((Left_steering_angle,Left_steering_angle_Flip,Right_steering_angle,Right_steering_angle_Flip),axis=0)

del Left_Images,Left_Images_Flip,Right_Images,Right_Images_Flip
del Left_steering_angle,Left_steering_angle_Flip,Right_steering_angle,Right_steering_angle_Flip

#%%生成训练网络
model = Sequential()
#标准化层(接受浮点输入)
model.add(Lambda(lambda x: x/255.0 - 0.5, input_shape = (160,320,3)))
#model.add(Lambda(lambda x: x/255.0, input_shape = (160,320,3)))
#图像裁剪层
model.add(Cropping2D(cropping=((70,25),(0,0))))
#卷积1
model.add(Conv2D(24, 5, strides=(2, 2)))
model.add(Activation('relu'))
#卷积2
model.add(Conv2D(36, 5, strides=(2, 2)))
model.add(Activation('relu'))
#卷积3
model.add(Conv2D(48, 5, strides=(2, 2)))
model.add(Activation('relu'))
#卷积4
#model.add(Conv2D(64, 3, strides=(1, 1)))
#model.add(Activation('relu'))
#卷积4
#model.add(Conv2D(64, 3, strides=(1, 1)))
#model.add(Activation('relu'))
#进入分类层
model.add(Flatten())
#model.add(Dense(1164))
#model.add(Activation('sigmoid'))
#model.add(Dense(100))
#model.add(Activation('sigmoid'))
model.add(Dense(50))
model.add(Activation('sigmoid'))
model.add(Dense(10))
model.add(Activation('sigmoid'))
model.add(Dense(1))

#模型生成
model.compile(optimizer='Adam',loss='mean_squared_error')

#%%训练模型
#model.fit(x = Images, y = steering_angle, batch_size = 128, epochs=1,validation_split=0.2,shuffle=True)
X_train, X_test, y_train, y_test = train_test_split(Images,steering_angle, test_size=0.2, shuffle = True)
train_generator = generator(X_train,y_train)
validation_generator = generator(X_test,y_test)

model.fit_generator(train_generator, steps_per_epoch=len(y_train)//128,epochs=3, validation_data=validation_generator, validation_steps=len(y_test)//128)

#%%保存模型
#model.save(filepath='./Model/Accept3.h5',overwrite=True,include_optimizer = True)


import os
import re
import docplex.cp.utils_visu as visu
from docplex.cp.model import CpoModel, CpoStepFunction, INTERVAL_MIN, INTERVAL_MAX
import functools
import math
minutes_in_day = 480
number_of_employers = 3
numdetails = 12
WorkerNames = ["worker1", "worker2", "worker3"]
TaskNames = ["obtain_parts", "repair", "delivery"]
# minutes_on_repair - число минут, которые тратит работник на починку девайса
minutes_on_repair = [140, 80, 160, 120, 160, 120, 130, 100, 40, 140, 160, 60]
# minutes_on_obtain_parts - число минут, которые тратит работник на поиск запчастей
minutes_on_obtain_parts = [20, 30, 40, 50, 100, 10, 20, 40, 100, 10, 50, 20]
# minutes_on_delivery - число минут, которое уходит на доставку can be done in parallel
minutes_on_delivery = [120, 150, 170, 100, 140, 100, 150, 230, 100, 90, 180, 280]
# euro_penalty - число евро, составляющие штраф за откладку на день
euro_penalty = [100, 80, 120, 100, 80, 120, 120, 90, 100, 80, 150, 13]
mdl = CpoModel()
details = [mdl.interval_var(name="Detail{}".format(i + 1)) for i in range(numdetails)]
TaskNames_ids = {}
itvs_1 = {}
itvs_2 = {}
itvs_3 = {}
itvs = {}
#zadadim treh
for d in range(numdetails):
        _name = "obtain_parts1_{}".format(d + 1)
        itvs_1[(d, 0)] = mdl.interval_var(size=minutes_on_obtain_parts[d], name=_name, optional=True)
for d in range(numdetails):
        _name = "repair1_{}".format(d + 1)
        itvs_1[(d, 1)] = mdl.interval_var(size=minutes_on_repair[d], name=_name, optional=True)

for d in range(numdetails):
        _name = "obtain_parts2_{}".format(d + 1)
        itvs_2[(d, 0)] = mdl.interval_var(size=minutes_on_obtain_parts[d], name=_name, optional=True)
for d in range(numdetails):
        _name = "repair2_{}".format(d + 1)
        itvs_2[(d, 1)] = mdl.interval_var(size=minutes_on_repair[d], name=_name, optional=True)

for d in range(numdetails):
        _name = "obtain_parts3_{}".format(d + 1)
        itvs_3[(d, 0)] = mdl.interval_var(size=minutes_on_obtain_parts[d], name=_name, optional=True)
for d in range(numdetails):
        _name = "repair3_{}".format(d + 1)
        itvs_3[(d, 1)] = mdl.interval_var(size=minutes_on_repair[d], name=_name, optional=True)
#zadadim obshie
for d in range(numdetails):
        _name = "obtain_parts{}".format(d + 1)
        itvs[(d, 0)] = mdl.interval_var(size=minutes_on_obtain_parts[d], name=_name)
for d in range(numdetails):
        _name = "repair{}".format(d + 1)
        itvs[(d, 1)] = mdl.interval_var(size=minutes_on_repair[d], name=_name)
for d in range(numdetails):
        _name = "delivery{}".format(d + 1)
        itvs[(d, 2)] = mdl.interval_var(size=minutes_on_delivery[d], name=_name)
 #альтернатива между работниками
for i in range(numdetails):
    for j in range(2):
        mdl.add(mdl.alternative(itvs[(i, j)], [itvs_3[(i, j)], itvs_2[(i, j)], itvs_1[(i, j)]]))
#навесим последовательность TO DO для 1,2,3
#for h in range(numdetails):
#        mdl.add(mdl.end_before_start(itvs[(h, 1)], itvs[(h, 2)]))
#for h in range(numdetails):
#        mdl.add(mdl.end_before_start(itvs[(h, 0)], itvs[(h, 1)]))
#навесим последовательность жестко
for h in range(numdetails):
        mdl.add(mdl.end_before_start(itvs[(h, 1)], itvs[(h, 2)]))
for h in range(numdetails):
       mdl.add(mdl.end_before_start(itvs[(h, 0)], itvs[(h, 1)]))
for h in range(numdetails):
        mdl.add(mdl.start_at_end(itvs[(h, 1)], itvs[(h, 0)]))
#навесим принадлежность
for h in range(numdetails):
    mdl.add(mdl.span(details[h], [itvs[(h, t)] for t in range(3)]))
workers = {}
workers1 = {"worker1": mdl.sequence_var([itvs_1[(h, t)] for h in range(numdetails) for t in range(2)], types=[t for h in range(numdetails) for t in range(2)], name="worker1")}
workers.update(workers1)
workers2 = {"worker2": mdl.sequence_var([itvs_2[(h, t)] for h in range(numdetails) for t in range(2)], types=[t for h in range(numdetails) for t in range(2)], name="worker2")}
workers.update(workers2)
workers3 = {"worker3": mdl.sequence_var([itvs_3[(h, t)] for h in range(numdetails) for t in range(2)], types=[t for h in range(numdetails) for t in range(2)], name="worker3")}
workers.update(workers3)
deliveryworker = {"deliveryworker": mdl.sequence_var([itvs[(h, 2)] for h in range(numdetails)], types=[h for h in range(numdetails)], name="deliveryworker")}
workers.update(deliveryworker)
for w in WorkerNames:
    mdl.add(mdl.no_overlap(workers[w]))
mdl.add(mdl.minimize(mdl.sum(euro_penalty[i] * mdl.float_div(mdl.max([mdl.end_of(itvs[(i, 2)]) - 480, 0]), mdl.end_of(itvs[(i, 2)]) - 480 + mdl.equal(mdl.end_of(itvs[(i, 2)]) - 480, 0) )  for i in range(numdetails))))
#mdl.add(mdl.minimize(mdl.sum(euro_penalty[i] * mdl.float_div(mdl.max([mdl.end_of(itvs[(i, 2)]) - 480, 0]),mdl.max([mdl.end_of(itvs[(i, 2)]) - 480)) for i in range(numdetails)))))

#mdl.add(mdl.minimize(mdl.sum(euro_penalty[i] * mdl.max([mdl.step_at(mdl.end_of(itvs[(i, 2)]) - 480,1), 0]) for i in range(numdetails))))

#mdl.add(mdl.minimize(mdl.sum(euro_penalty[i] * mdl.float_div(mdl.max([mdl.exponent(mdl.end_of(itvs[(i, 2)]) - 480)-1, 0]), mdl.exponent(mdl.end_of(itvs[(i, 2)]) - 480)) for i in range(numdetails))))
#mdl.add(mdl.minimize(mdl.sum(euro_penalty[i] * mdl.float_div(mdl.max([mdl.exponent(mdl.power(mdl.end_of(itvs[(i, 2)]) - 480, 0.33))-1, 0]), mdl.exponent(mdl.power(mdl.end_of(itvs[(i, 2)]) - 480, 0.33))) for i in range(numdetails))))

#mdl.add(mdl.minimize(mdl.sum(euro_penalty[i] * mdl.presence_of(itvs[(i, 2)]) for i in range(numdetails))))
print("Solving model....")
msol = mdl.solve(FailLimit=100000, TimeLimit=10)
print("Solution: ")
msol.print_solution()

def compact(name):
    # Example: A31_M1_TP1 -> 31
    task, foo = name.split('_', 1)
    return task[1:]

def showsequence(s,name):
    vs = s.get_value()
    starts=[]
    ends=[]
    nms=[]
    for i,v in enumerate(vs):
        if v.end<minutes_in_day:
            starts.append(v.start)
            ends.append(v.end)
            nm=v.get_name().replace("1_","").replace("2_","").replace("3_","")
            nms.append(nm)

    for i,s in enumerate(starts):
        if msol.get_var_solution("delivery"+re.findall('(\d+)', nms[len(starts)-1])[0]).get_end() > 480:
            nms.pop()
            ends.pop()
            starts.pop()
    visu.sequence(name=name,intervals=[(s,ends[i],i,nms[i]) for i,s in enumerate(starts) ])

def showdelivery():
    starts=[]
    ends=[]
    nms=[]
    color = []
    intervals=[(msol.get_var_solution("delivery"+str(i+1)).get_start(), msol.get_var_solution("delivery"+str(i+1)).get_end(), i, msol.get_var_solution("delivery"+str(i+1)).get_name()) for i in range(numdetails) if msol.get_var_solution("delivery"+str(i+1)).get_end()<minutes_in_day]
    for i,interval in enumerate(intervals):
        starts.append(interval[0])
        ends.append(interval[1])
        color.append(interval[2])
        nms.append(interval[3])
    n = 1
    while n < len(starts):
      for i in range(len(starts)-n):
          if starts[i] > starts[i+1]:
               starts[i],starts[i+1] = starts[i+1],starts[i]
               ends[i],ends[i+1] = ends[i+1],ends[i]
               color[i],color[i+1] = color[i+1],color[i]
               nms[i],nms[i+1] = nms[i+1],nms[i]
      n += 1
    starts_res=[starts[0]]
    ends_res=[ends[0]]
    nms_res=[nms[0]]
    color_res = [color[0]]
    used = [starts[0]]
    p = 0
    for y,t in enumerate(starts):
        for i,s in enumerate(starts):
            k = len(ends_res)
            for j,r in enumerate(ends_res):
                if s >= r:
                    p = p + 1
            if  (p == k) & (not (i in used)):
                starts_res.append(s)
                ends_res.append(ends[i])
                nms_res.append(nms[i])
                color_res.append(color[i])
                used.append(s)
                p = 0
        print(starts_res, ends_res, nms_res)
        print('----')
        visu.sequence(name='delivery',intervals=[(starts_res[i],ends_res[i],color_res[i],nms_res[i]) for i,s in enumerate(starts_res) ])
        if len(used) == len(starts):
            break

        for e,w in enumerate(starts):
              if not (w in used):
                  print(used)
                  starts_res=[starts[e]]
                  ends_res=[ends[e]]
                  nms_res=[nms[e]]
                  color_res = [color[e]]
                  used.append(w)
                  break

    #visu.sequence(name = 'Delivery1',intervals = intervals ) #if msol.get_var_solution("delivery"+str(i+1)).get_end() <= 480])
#if msol.get_var_solution("delivery"+str(i+1)).get_end() <= 480])
    #print(sum([euro_penalty[i] * max([msol.get_var_solution("delivery"+str(i+1)).get_end() - 480, 0]) for i in range(numdetails)]))
    #print(sum([euro_penalty[i] * max([math.exp(msol.get_var_solution("delivery"+str(i+1)).get_end() - 480) - 1, 0])/math.exp(msol.get_var_solution("delivery"+str(i+1)).get_end() - 480) for i in range(numdetails)]))
    #for i in range(12):
        #print(1*(msol.get_var_solution("delivery"+str(i+1)).get_end() - 480))

if msol and visu.is_visu_enabled():

    #visu.sequence(name='Machine1',intervals=[(0,10,1,'Job1'),(15,35,2,'Job2')],transitions=[(10,13)])
    showdelivery()
    showsequence(msol.get_var_solution(workers["worker1"]),"worker1")
    showsequence(msol.get_var_solution(workers["worker2"]),"worker2")
    showsequence(msol.get_var_solution(workers["worker3"]),"worker3")
    #showsequence(msol.get_var_solution(workers["deliveryworker"]),"deliveryworker")
    #visu.interval(starts)
    visu.show()


from django.urls import path
from . import views

urlpatterns = [
    path('', views.index),
    path('register', views.register),
    path('login', views.login),
    path('books', views.books),
    path('logout', views.logout),
    path('books/add', views.addBookPage),
    path('addbook', views.addBook),
    path('books/<int:bookid>', views.bookInfo),
    path('users/<int:uID>', views.userPage),
]


from flask import Flask, request, jsonify

from pram.data   import GroupSizeProbe, ProbeMsgMode
from pram.entity import Group, GroupQry, GroupSplitSpec, Site
from pram.rule   import GoToRule, DiscreteInvMarkovChain, TimeInt, TimePoint
from pram.sim    import Simulation

import json

from pramity_rules import SimpleFluProgress, SimpleGoTo, MallMovement, MallFlu, PlayableMallFlu

app = Flask(__name__)
rules = {}
sites = []
probe_grp_size_site = None

def add_initial_rules(time_offset):
	"Imports rules built into pram and inserts them into the rules dictionary"
	global sites
	sites = {s:Site(s) for s in ['home', 'work-a', 'work-b', 'work-c', 'store-a', 'store-b']}
	mall_sites_names = ["big_theater", "down_store1", "down_store2", "down_store3", "down_store4", "down_store5", "down_store6", "down_store7", "down_store8", "down_store9", "big_down_store10", "big_down_store11", "big_down_store12", "up_store1", "up_store2", "up_store3", "up_store4", "up_store5", "up_store6", "up_store7", "up_store8", "up_store9", "big_down_courtyard_1", "big_down_courtyard_2", "big_down_courtyard_3", "big_down_courtyard_4"]
	mall_sites = [Site(s) for s in mall_sites_names]
	for i in range(len(mall_sites)):
		sites[mall_sites_names[i]] = mall_sites[i]

	#rules["Mall Movement"] = [MallMovement(0.2, mall_sites)]
	rules["Mall Flu"] = [MallFlu(1, 0.1, mall_sites)]
	rules["Playable Mall Flu"] = [PlayableMallFlu(1)]

	rules["Simple Flu Progress Rule"] = [SimpleFluProgress('flu-status', { 's': [0.95, 0.05, 0.00], 'i': [0.00, 0.50, 0.50], 'r': [0.10, 0.00, 0.90] }, sites['home'])]
	rules["Home-Work-School Rules"] = [SimpleGoTo(TimeInt( (8 - time_offset)%24,(12 - time_offset)%24), 0.4, 'home',  'work',  'Some agents leave home to go to work'),
		SimpleGoTo(TimeInt((16- time_offset)%24,(20- time_offset)%24), 0.4, 'work',  'home',  'Some agents return home from work'),
		SimpleGoTo(TimeInt((16- time_offset)%24,(21- time_offset)%24), 0.2, 'home',  'store', 'Some agents go to a store after getting back home'),
		SimpleGoTo(TimeInt((17- time_offset)%24,(23- time_offset)%24), 0.3, 'store', 'home',  'Some shopping agents return home from a store'),
		SimpleGoTo(TimePoint((24- time_offset)%25),  1.0, 'store', 'home',  'All shopping agents return home after stores close'),
		SimpleGoTo(TimePoint( (2- time_offset)%25),  1.0, None, 'home',  'All still-working agents return home')]

	global probe_grp_size_site
	probe_grp_size_site = GroupSizeProbe.by_rel('site', Site.AT, sites.values(), msg_mode=ProbeMsgMode.DISP, memo='Mass distribution across sites')
	return

@app.route('/')
def home():
	return 'Hello World!'

def make_relations_serializable(attr):
	keys = []
	vals = []
	for k, v in attr.items():
		if not k == Site.AT:
			keys.append(k)
			vals.append(v.name)
	return keys, vals

def make_attributes_serializable(attr):
	keys = []
	vals = []
	for k, v in attr.items():
		keys.append(k)
		vals.append(v)
	return keys, vals

def get_mass_flow(s):
    redistributions = []
    if s.pop.last_iter.mass_flow_specs == None:
    	return redistributions
    for mfs in s.pop.last_iter.mass_flow_specs:
        for g_dst in mfs.dst:
            redistribution = {}

            source_dict = {}
            source_dict['attributeKeys'], source_dict['attributeValues'] = make_attributes_serializable(mfs.src.attr)
            if Site.AT in mfs.src.rel.keys():
                source_dict['site'] = mfs.src.rel[Site.AT].name
            else:
                source_dict['site'] = None
            source_dict['n'] = 0
            source_dict['relationKeys'], source_dict['relationValues'] = make_relations_serializable(mfs.src.rel)

            destination_dict = {}
            destination_dict['attributeKeys'], destination_dict['attributeValues'] = make_attributes_serializable(g_dst.attr)
            if Site.AT in g_dst.rel.keys():
                destination_dict['site'] = g_dst.rel[Site.AT].name
            else:
                destination_dict['site'] = None
            destination_dict['relationKeys'], destination_dict['relationValues'] = make_relations_serializable(g_dst.rel)

            destination_dict['n'] = 0

            redistribution['source'] = source_dict
            redistribution['destination'] = destination_dict
            redistribution['mass'] = g_dst.m

            redistributions.append(redistribution)
    return {"redistributions": redistributions}

def run_and_get_mass_flow(s, number_of_iterations):
    simSteps = []
    for i in range(number_of_iterations):
        s.run(1)
        simSteps.append(get_mass_flow(s))
    return {"simSteps": simSteps}

def get_attribute_dictionary(k, v):
	a_dict = {}
	for i in range(len(k)):
		a_dict[k[i]] = v[i]

	return a_dict

@app.route('/run_simulation', methods=['POST'])
def run_simulation():
	sim_info = json.loads(request.form['runInfo'])

	initial_groups = sim_info['groups']
	included_rules = sim_info['rules']
	runs = sim_info['runs']
	time_offset = sim_info['start_time']

	add_initial_rules(time_offset)

	s = Simulation(do_keep_mass_flow_specs=True)
	#s.add_probe(probe_grp_size_site)

	for rule_name in included_rules:
		for r in rules[rule_name]:
			s.add_rule(r)

	g_index = 0
	for group in initial_groups:
		g_name = "g" + str(g_index)
		g_index = g_index + 1
		g_mass = group['n']
		g_attributes = get_attribute_dictionary(group['attributeKeys'], group['attributeValues'])
		g_relations = get_attribute_dictionary(group['relationKeys'], group['relationValues'])
		g = Group(g_name, g_mass, g_attributes)

		if not group['site'] == "":
			g.set_rel(Site.AT, sites[group['site']])

		for k,v in g_relations.items():
			g.set_rel(k, sites[v])
		s.add_group(g)

	flows = run_and_get_mass_flow(s, runs)

	return jsonify(flows)


'''28/09/2018 Calculate onset dates from daily NCEP-NCAR reanalysis, and save'''

import numpy as np
import xarray as xr
import matplotlib.pyplot as plt
import sh
from pylab import rcParams
import pandas as pd
from climatology import bob_onset, scsm_onset, ism_onset

filename = '/disca/share/rg419/ncep_ncar_uwnd_daily_850.nc'   # Have taken 850hPa level from Stephen's datafile and saved seperately

data = xr.open_dataset('/disca/share/rg419/ncep_ncar_uwnd_daily_850_1948_2016.nc')

#data_1 = xr.open_dataset('/disca/share/rg419/ncep_ncar_uwnd_daily_850_1948_1987.nc')
#data_2 = xr.open_dataset('/disca/share/rg419/ncep_ncar_uwnd_daily_850_1988_2017.nc')
#data = xr.open_mfdataset(['/disca/share/rg419/ncep_ncar_uwnd_daily_850_1948_1987.nc',
#                          '/disca/share/rg419/ncep_ncar_uwnd_daily_850_1988_2017.nc'])
#data = xr.concat([data_1, data_2], dim='time')

def day_means_of_year(data, year):   # Function to get day of year
    data_year = data.sel(time=str(year))
    if len(data_year.time)==366:
        day = np.arange(1.,367.)
    else:
        day = np.arange(1.,366.)
    data_year = data_year.assign_coords(day = ('time', day))
    data_year = data_year.groupby('day').mean(('time'))
    return data_year

def pentad_means_of_year(data, year):  # Function to get pentad of year
        data_year = data.sel(time=str(year))
        if len(data_year.time)==366:
            pentad = np.repeat(np.arange(1., 74.), 5)
            pentad = np.insert(pentad, 10, 2)
        else:
            pentad = np.repeat(np.arange(1., 74.), 5)
        data_year = data_year.assign_coords(pentad = ('time', pentad))
        data_year = data_year.groupby('pentad').mean(('time'))
        return data_year

# Get SCSM onset date
onset_ncep=[]
for year in range(1948,2017):
    print(year)
    u_year = pentad_means_of_year(data.uwnd, year)
    u_ncep = scsm_onset(u_year, pdim='lev', pentaddim='pentad', latdim='lat', londim='lon')
    onset_ncep.append(u_ncep[1])

print(onset_ncep)
np.save('ncep_onsets_scsm',np.array(onset_ncep))

# Get BoB onset date
onset_ncep=[]
for year in range(1948,2017):
    print(year)
    u_year = day_means_of_year(data.uwnd, year)
    u_ncep = bob_onset(u_year, pdim='lev', daydim='day', latdim='lat', londim='lon')
    onset_ncep.append(u_ncep[1])

print(onset_ncep)
np.save('ncep_onsets_bob_days',np.array(onset_ncep))

# Get ISM onset date
onset_ncep=[]
for year in range(1948,2017):
    print(year)
    u_year = day_means_of_year(data.uwnd, year)
    u_ncep = ism_onset(u_year, pdim='lev', daydim='day', latdim='lat', londim='lon')
    onset_ncep.append(u_ncep[1])

print(onset_ncep)
np.save('ncep_onsets_ism_days',np.array(onset_ncep))


# -*- coding: utf-8 -*-
# Copyright 2019 the HERA Project
# Licensed under the MIT License

import pytest
import os
import numpy as np
from collections import OrderedDict as odict
import copy
import glob
import scipy.stats as stats
from pyuvdata import UVCal, UVData

from .. import io, lstbin, utils
from ..datacontainer import DataContainer
from ..data import DATA_PATH

@pytest.mark.filterwarnings("ignore:The default for the `center` keyword has changed")
@pytest.mark.filterwarnings("ignore:Degrees of freedom <= 0 for slice")
@pytest.mark.filterwarnings("ignore:Mean of empty slice")
@pytest.mark.filterwarnings("ignore:invalid value encountered in true_divide")
@pytest.mark.filterwarnings("ignore:invalid value encountered in greater")
class Test_lstbin(object):
    def setup_method(self):
        # load data
        np.random.seed(0)
        self.data_files = [sorted(glob.glob(DATA_PATH + '/zen.2458043.4*XRAA.uvh5')),
                           sorted(glob.glob(DATA_PATH + '/zen.2458044.4*XRAA.uvh5')),
                           sorted(glob.glob(DATA_PATH + '/zen.2458045.4*XRAA.uvh5'))]

        (self.data1, self.flgs1, self.ap1, a, self.freqs1, t, self.lsts1,
         p) = io.load_vis(self.data_files[0], return_meta=True, filetype="uvh5")
        (self.data2, self.flgs2, ap, a, self.freqs2, t, self.lsts2,
         p) = io.load_vis(self.data_files[1], return_meta=True, filetype="uvh5")
        (self.data3, self.flgs3, ap, a, self.freqs3, t, self.lsts3,
         p) = io.load_vis(self.data_files[2], return_meta=True, filetype="uvh5")
        self.data_list = [self.data1, self.data2, self.data3]
        self.flgs_list = [self.flgs1, self.flgs2, self.flgs3]
        self.lst_list = [self.lsts1, self.lsts2, self.lsts3]

    def test_make_lst_grid(self):
        lst_grid = lstbin.make_lst_grid(0.01, begin_lst=None, verbose=False)
        assert len(lst_grid) == 628
        assert np.isclose(lst_grid[0], 0.0050025360725952121)
        lst_grid = lstbin.make_lst_grid(0.01, begin_lst=np.pi, verbose=False)
        assert len(lst_grid) == 628
        assert np.isclose(lst_grid[0], 3.1365901175171982)
        lst_grid = lstbin.make_lst_grid(0.01, begin_lst=-np.pi, verbose=False)
        assert len(lst_grid) == 628
        assert np.isclose(lst_grid[0], 3.1365901175171982)

    @pytest.mark.filterwarnings("ignore:All-NaN slice encountered")
    @pytest.mark.filterwarnings("ignore:divide by zero encountered in true_divide")
    def test_lstbin(self):
        dlst = 0.0007830490163484
        # test basic execution
        output = lstbin.lst_bin(self.data_list, self.lst_list, flags_list=self.flgs_list, dlst=None,
                                median=True, lst_low=0, lst_hi=np.pi, verbose=False)
        output = lstbin.lst_bin(self.data_list, self.lst_list, flags_list=None, dlst=0.01,
                                verbose=False)
        output = lstbin.lst_bin(self.data_list, self.lst_list, flags_list=self.flgs_list, dlst=dlst,
                                verbose=False)
        # check shape and dtype
        assert output[1][(24, 25, 'ee')].dtype == np.complex
        assert output[1][(24, 25, 'ee')].shape == (224, 64)
        # check number of points in each bin
        assert np.allclose(output[-1][(24, 25, 'ee')].real[0, 30], 1)
        assert np.allclose(output[-1][(24, 25, 'ee')].real[30, 30], 2)
        assert np.allclose(output[-1][(24, 25, 'ee')].real[100, 30], 3)
        assert np.allclose(output[-1][(24, 25, 'ee')].real[190, 30], 2)
        assert np.allclose(output[-1][(24, 25, 'ee')].real[220, 30], 1)
        # check with large spacing lst_grid
        output = lstbin.lst_bin(self.data_list, self.lst_list, dlst=.01, verbose=False)
        assert np.allclose(output[-1][(24, 25, 'ee')].real[10, 30], 38)
        # check flgs are propagated
        flgs1 = copy.deepcopy(self.flgs1)
        flgs1[(24, 25, 'ee')][:, 32] = True
        flgs2 = copy.deepcopy(self.flgs2)
        flgs2[(24, 25, 'ee')][:, 32] = True
        flgs3 = copy.deepcopy(self.flgs3)
        flgs_list = [flgs1, flgs2, flgs3]
        output = lstbin.lst_bin(self.data_list, self.lst_list, dlst=dlst, flags_list=flgs_list)
        assert np.allclose(output[2][(24, 25, 'ee')][0, 32], True)
        assert np.allclose(output[2][(24, 25, 'ee')][180, 32], False)
        assert np.allclose(output[2][(24, 25, 'ee')][210, 32], False)
        # test return no avg
        output = lstbin.lst_bin(self.data_list, self.lst_list, dlst=dlst, flags_list=self.flgs_list, return_no_avg=True)
        assert len(output[2][list(output[2].keys())[0]][100]) == 3
        assert len(output[2][list(output[2].keys())[0]][100][0]) == 64
        # test switch bl
        conj_data3 = DataContainer(odict(list(map(lambda k: (utils.reverse_bl(k), np.conj(self.data3[k])), self.data3.keys()))))
        data_list = [self.data1, self.data2, conj_data3]
        output = lstbin.lst_bin(data_list, self.lst_list, dlst=dlst)
        assert output[1][(24, 25, 'ee')].shape == (224, 64)
        # test sigma clip
        output = lstbin.lst_bin(self.data_list, self.lst_list, flags_list=None, dlst=0.01,
                                verbose=False, sig_clip=True, min_N=5, sigma=2)
        # test wrapping
        lst_list = list(map(lambda l: (copy.deepcopy(l) + 6) % (2 * np.pi), self.lst_list))
        output = lstbin.lst_bin(self.data_list, lst_list, dlst=0.001, begin_lst=np.pi)
        assert output[0][0] > output[0][-1]
        assert len(output[0]) == 175
        # test appropriate data_count
        output = lstbin.lst_bin(self.data_list, self.lst_list, flags_list=None, dlst=dlst, lst_low=0.25, lst_hi=0.3,
                                verbose=False)
        assert np.allclose(output[4][(24, 25, 'ee')], 3.0)

    def test_lst_align(self):
        # test basic execution
        output = lstbin.lst_align(self.data1, self.lsts1, dlst=None, flags=self.flgs1, flag_extrapolate=True, verbose=False)
        assert output[0][(24, 25, 'ee')].shape == (180, 64)
        assert len(output[2]) == 180
        assert np.allclose(output[2][0], 0.20163512170971379)
        # test flag extrapolate
        assert np.all(output[1][(24, 25, 'ee')][-1])
        # test no dlst
        output = lstbin.lst_align(self.data1, self.lsts1, dlst=None, flags=self.flgs1, flag_extrapolate=True, verbose=False)
        # test wrapped lsts
        lsts = (self.lsts1 + 6) % (2 * np.pi)
        output = lstbin.lst_align(self.data1, lsts, dlst=None, flags=self.flgs1, flag_extrapolate=True, verbose=False)
        assert np.allclose(output[2][150], 0.035628730243852047)

    @pytest.mark.filterwarnings("ignore:The expected shape of the ENU array")
    @pytest.mark.filterwarnings("ignore:antenna_diameters is not set")
    def test_lst_bin_files(self):
        # basic execution
        file_ext = "{pol}.{type}.{time:7.5f}.uvh5"
        lstbin.lst_bin_files(self.data_files, ntimes_per_file=250, outdir="./", overwrite=True,
                             verbose=False, file_ext=file_ext, ignore_flags=True)
        output_lst_file = "./zen.ee.LST.0.20124.uvh5"
        output_std_file = "./zen.ee.STD.0.20124.uvh5"
        assert os.path.exists(output_lst_file)
        assert os.path.exists(output_std_file)
        uv1 = UVData()
        uv1.read(output_lst_file)
        # assert nsample w.r.t time follows 1-2-3-2-1 pattern
        nsamps = np.mean(uv1.get_nsamples(52, 52, 'ee'), axis=1)
        expectation = np.concatenate([np.ones(22), np.ones(22) * 2, np.ones(136) * 3, np.ones(22) * 2, np.ones(21)]).astype(np.float)
        assert np.allclose(nsamps, expectation)
        os.remove(output_lst_file)
        os.remove(output_std_file)

        # test with multiple blgroups
        lstbin.lst_bin_files(self.data_files, ntimes_per_file=250, outdir="./", overwrite=True,
                             verbose=False, file_ext=file_ext, Nblgroups=3, ignore_flags=True)
        assert os.path.exists(output_lst_file)
        assert os.path.exists(output_std_file)
        uv2 = UVData()
        uv2.read(output_lst_file)
        assert uv1 == uv2
        os.remove(output_lst_file)
        os.remove(output_std_file)

        # test rephase
        lstbin.lst_bin_files(self.data_files, ntimes_per_file=250, outdir="./", overwrite=True,
                             verbose=False, rephase=True, file_ext=file_ext)
        output_lst_file = "./zen.ee.LST.0.20124.uvh5"
        output_std_file = "./zen.ee.STD.0.20124.uvh5"
        assert os.path.exists(output_lst_file)
        assert os.path.exists(output_std_file)
        os.remove(output_lst_file)
        os.remove(output_std_file)

        # test data_list is empty
        data_files = [[sorted(glob.glob(DATA_PATH + '/zen.2458043.*XRAA.uvh5'))[0]],
                      [sorted(glob.glob(DATA_PATH + '/zen.2458045.*XRAA.uvh5'))[-1]]]
        lstbin.lst_bin_files(data_files, ntimes_per_file=30, outdir="./", overwrite=True,
                             verbose=False, file_ext=file_ext)
        output_lst_files = ['./zen.ee.LST.0.20124.uvh5', './zen.ee.LST.0.31870.uvh5', './zen.ee.LST.0.36568.uvh5']
        assert os.path.exists(output_lst_files[0])
        assert os.path.exists(output_lst_files[1])
        assert os.path.exists(output_lst_files[2])
        output_files = np.concatenate([glob.glob("./zen.ee.LST*"),
                                       glob.glob("./zen.ee.STD*")])
        for of in output_files:
            if os.path.exists(of):
                os.remove(of)

        # test smaller ntimes file output, sweeping through f_select
        lstbin.lst_bin_files(self.data_files, ntimes_per_file=80, outdir="./", overwrite=True,
                             verbose=False, vis_units='Jy', file_ext=file_ext)
        output_files = sorted(glob.glob("./zen.ee.LST*") + glob.glob("./zen.ee.STD*"))
        # load a file
        uvd1 = UVData()
        uvd1.read(output_files[1])
        assert uvd1.vis_units == 'Jy'
        assert 'Thisfilewasproducedbythefunction' in uvd1.history.replace('\n', '').replace(' ', '')
        assert uvd1.Ntimes == 80
        assert np.isclose(uvd1.nsample_array.max(), 3.0)
        # remove files
        for of in output_files:
            if os.path.exists(of):
                os.remove(of)

        # test output_file_select
        lstbin.lst_bin_files(self.data_files, ntimes_per_file=80, outdir="./", overwrite=True, output_file_select=1,
                             verbose=False, vis_units='Jy', file_ext=file_ext)
        output_files = sorted(glob.glob("./zen.ee.LST*") + glob.glob("./zen.ee.STD*"))
        # load a file
        uvd2 = UVData()
        uvd2.read(output_files[0])
        # assert equivalence with previous run
        assert uvd1 == uvd2
        # remove files
        for of in output_files:
            if os.path.exists(of):
                os.remove(of)

        # assert bad output_file_select produces no files
        output_files = sorted(glob.glob("./zen.ee.LST*") + glob.glob("./zen.ee.STD*"))
        for of in output_files:
            if os.path.exists(of):
                os.remove(of)
        lstbin.lst_bin_files(self.data_files, ntimes_per_file=80, outdir="./", overwrite=True, output_file_select=100,
                             verbose=False, file_ext=file_ext)
        output_files = sorted(glob.glob("./zen.ee.LST*") + glob.glob("./zen.ee.STD*"))
        assert len(output_files) == 0

        # test fixed start
        lstbin.lst_bin_files(self.data_files, ntimes_per_file=250, outdir="./", overwrite=True,
                             verbose=False, lst_start=0.18, fixed_lst_start=True, file_ext=file_ext)
        output_lst_file = "./zen.ee.LST.0.17932.uvh5"
        output_std_file = "./zen.ee.STD.0.17932.uvh5"
        assert os.path.exists(output_lst_file)
        assert os.path.exists(output_std_file)
        os.remove(output_lst_file)
        os.remove(output_std_file)
        extra_files = ["zen.ee.LST.0.37508.uvh5", "zen.ee.STD.0.37508.uvh5"]
        for of in extra_files:
            if os.path.exists(of):
                os.remove(of)

        # test input_cal
        uvc = UVCal()
        uvc.read_calfits(os.path.join(DATA_PATH, 'zen.2458043.12552.xx.HH.uvORA.abs.calfits'))
        uvc.flag_array[uvc.ant_array.tolist().index(24)] = True
        uvc.gain_array[uvc.ant_array.tolist().index(25)] = 1e10
        input_cals = []
        for dfiles in self.data_files:
            input_cals.append([uvc for df in dfiles])

        lstbin.lst_bin_files(self.data_files, ntimes_per_file=250, outdir="./", overwrite=True,
                             verbose=False, input_cals=input_cals, file_ext=file_ext)

        output_lst_file = "./zen.ee.LST.0.20124.uvh5"
        output_std_file = "./zen.ee.STD.0.20124.uvh5"
        assert os.path.exists(output_lst_file)
        assert os.path.exists(output_std_file)
        os.remove(output_lst_file)
        os.remove(output_std_file)

        # test input_cal with only one Ntimes
        input_cals = []
        for dfiles in self.data_files:
            input_cals.append([uvc.select(times=uvc.time_array[:1], inplace=False) for df in dfiles])
        lstbin.lst_bin_files(self.data_files, ntimes_per_file=250, outdir="./", overwrite=True,
                             verbose=False, input_cals=input_cals, file_ext=file_ext)
        assert os.path.exists(output_lst_file)
        assert os.path.exists(output_std_file)

        # assert gains and flags were propagated
        lstb = UVData()
        lstb.read(output_lst_file)
        assert np.allclose(np.abs(lstb.get_data(25, 37)[~lstb.get_flags(25, 37)]), 0.0)
        assert np.all(lstb.get_flags(24, 25))

        os.remove(output_lst_file)
        os.remove(output_std_file)

    def test_lst_bin_arg_parser(self):
        a = lstbin.lst_bin_arg_parser()
        args = a.parse_args(["--dlst", "0.1", "--input_cals", "zen.2458043.12552.HH.uvA.omni.calfits", "zen.2458043.12552.xx.HH.uvORA.abs.calfits",
                             "--overwrite", "zen.2458042.12552.xx.HH.uvXA", "zen.2458043.12552.xx.HH.uvXA"])

        assert np.isclose(args.dlst, 0.1)
        assert len(args.input_cals) == 2
        assert len(args.data_files) == 2

    def test_sigma_clip(self):
        # test basic execution
        np.random.seed(0)
        x = stats.norm.rvs(0, 1, 1000)
        x[10] = 4
        x[11] = -4
        arr = lstbin.sigma_clip(x, sigma=2.0)
        assert np.all(arr[10])
        assert np.all(arr[11])
        # test array performance
        x = np.array(list(map(lambda s: stats.norm.rvs(0, s, 100), np.arange(1, 5.1, 1))))
        x[0, 50] = 100
        x[4, 50] = 5
        arr = lstbin.sigma_clip(x, sigma=2.0)
        assert np.all(arr[0, 50])
        assert not np.any(arr[4, 50])
        # test flags
        arr = stats.norm.rvs(0, 1, 10).reshape(2, 5)
        flg = np.zeros_like(arr, np.bool)
        flg[0, 3] = True
        out = lstbin.sigma_clip(arr, flags=flg, min_N=5)
        assert not np.any(out[0, 3])
        out = lstbin.sigma_clip(arr, flags=flg, min_N=1)
        assert np.all(out[0, 3])

    def tearDown(self):
        output_files = sorted(glob.glob("./zen.ee.LST*") + glob.glob("./zen.ee.STD*"))
        for of in output_files:
            if os.path.exists(of):
                os.remove(of)


import numpy
import pandas

class Score(object):
    def __init__(self, a: numpy.ndarray, p: numpy.ndarray):
        self.a = numpy.array(a)  # actual/observation values
        self.p = numpy.array(p)  # predicted/forecast values
        assert len(self.a) == len(self.p)

    def mae(self) -> numpy.float:
        return numpy.abs((self.a - self.p)).mean()

    def mse(self) -> numpy.float:
        return ((self.a - self.p) ** 2).mean()

    def rmse(self) -> numpy.float:
        return self.mse() ** 0.5

    def mape(self) -> numpy.float:
        a = self._adjusted_values(self.a)
        return numpy.abs(((self.a - self.p) / a)).mean()

    def mspe(self) -> numpy.float:
        a = self._adjusted_values(self.a)
        return (((self.a - self.p) / a) ** 2).mean()

    def rmspe(self) -> numpy.float:
        return self.mspe() ** 0.5

    def _adjusted_values(self, v) -> numpy.ndarray:
        v = v.copy()
        c = 1.0
        if (v != 0).any():
            c = v[v != 0].mean()
        v[v == 0] = c
        return v

    def sqmrpa(self) -> numpy.float:  # square measurement sqmr of prediction/actual
        a = self._adjusted_values(self.a)
        return self.p.sum() / a.sum()

    def sqmrap(self) -> numpy.float:  # square measurement sqmr of actual/prediction
        p = self._adjusted_values(self.p)
        return self.a.sum() / p.sum()

    def gmpa(self) -> numpy.float:
        a = self._adjusted_values(self.a)
        r = numpy.abs(self.p / a)
        if (r == 0).all():
            return 0.0
        return r.prod() ** (1 / len(r))

    def gmap(self) -> numpy.float:
        p = self._adjusted_values(self.p)
        r = numpy.abs(self.a / p)
        if (r == 0).all():
            return 0.0
        return r.prod() ** (1 / len(r))

    def rsq(self) -> numpy.float:  # R square
        srs = ((self.a - self.p) ** 2).sum()  # sum of residuals squared
        srm = ((self.a - self.a.mean()) ** 2).sum()  # sum of residual from mean
        if srm == 0:
            srm = 1.0
        return 1 - srs / srm

    @property
    def dic(self) -> dict:
        return dict(
            mae=self.mae(),
            mse=self.mse(),
            rmse=self.rmse(),
            mape=self.mape(),
            mspe=self.mspe(),
            rmspe=self.rmspe(),
            sqmrpa=self.sqmrpa(),
            sqmrap=self.sqmrap(),
            gmpa=self.gmpa(),
            gmap=self.gmap(),
            rsq=self.rsq(),
        )

    @property
    def df(self) -> pandas.DataFrame:
        s = self.dic
        df = pandas.DataFrame(s, index=["scores"]).T
        return df

    def to_csv(self, tsv_file, sep=","):
        self.df.to_csv(tsv_file, sep=sep, header=False)

    def to_json(self, json_file):
        self.df.to_json(json_file)

    @property
    def json(self) -> str:
        return self.df.to_json()

if __name__ == "__main__":
    y = numpy.array([1.1, 1.2, 1.3])
    p = numpy.array([1.0, 0.9, 1.4])
    score = Score(y, p)
    print(score.df)
    print(score.json)
    score.to_csv("score.tsv")
    score.to_json("score.json")

    y = numpy.array([0.1, 0.2, -0.1])
    p = numpy.array([0.0, 0.0, 0.0])
    score = Score(y, p)
    print(score.df)

    y = numpy.array([0.0, 0.0, 0.0])
    p = numpy.array([0.1, 0.2, -0.1])
    score = Score(y, p)
    print(score.df)


from pyfirmata import Arduino,util
import paho.mqtt.client as mqtt
import time
import sys

#Definimos las variables
placa = Arduino("COM6")
rele1=13;
rele2=12;
rele3=11;
rele4=10;
topic="undefined"
valor=1

#Definimos las funciones
def on_connect(client,userdata,flags,rc):
    print("Conectado - codigo de resultado:" +str(rc))

def on_message(client,userdata,msg):
    #print(msg.topic + " " + str(msg.qos) + " " + str(msg.payload))
    global topic
    global valor
    topic = msg.topic
    valor=int(msg.payload)

def on_publish(client,obj,mid):
    print("mid: " + str(mid))

def on_subscribe(client,obj,mid,granted_qos):
    print("Subscribed: " + str(mid) + " " + str(granted_qos))

def on_log(client,obj,level,string):
    print(string)

client = mqtt.Client()
client.on_message = on_message
client.on_connect = on_connect
client.on_publish = on_publish
client.on_subscribe = on_subscribe
client.username_pw_set('xrxavxlq', 'hrZH1Htfj73s')

#Conectamos con el servidor
try:
    client.connect("m10.cloudmqtt.com",11129,60)
except:
    print("No se pudo conectar con el MQTT Broker...")
    print("Cerrando...")
    sys.exit()

print("Inicio")
placa.digital[rele1].write(valor)
placa.digital[rele2].write(valor)
placa.digital[rele3].write(valor)
placa.digital[rele4].write(valor)
client.subscribe("sala")
client.subscribe("dormitorio")
client.subscribe("patio")
client.subscribe("calle")

while True:
    rc = client.loop()
    if(topic=="sala"):
        placa.digital[rele1].write(valor)
    elif(topic=="dormitorio"):
        placa.digital[rele2].write(valor)
    elif(topic=="patio"):
        placa.digital[rele3].write(valor)
    elif(topic=="calle"):
        placa.digital[rele4].write(valor)
    time.sleep(0.5)
    topic=""
print("Fin")


__author__ = "David Chaves-Fraga"
__credits__ = ["David Chaves-Fraga"]

__license__ = "Apache-2.0"
__maintainer__ = "David Chaves-Fraga"
__email__ = "david.chaves@upm.es"
__name__ = "INVERSETC-0003 based on YARRRMLTC-0005"

import os
from ruamel.yaml import YAML
import yatter
from rdflib.graph import Graph
from deepdiff import DeepDiff

def test_inversetc0006():
    yaml = YAML(typ='safe', pure=True)
    with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'mapping.yml')) as file:
        expected_mapping = yaml.load(file)

    input_mapping = Graph()
    mapping_path = input_mapping.parse(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'mapping.ttl'), format="ttl")
    translated_mapping = yaml.load(str(yatter.inverse_translation(mapping_path)))

    ddiff = DeepDiff(expected_mapping['mappings'], translated_mapping['mappings'], ignore_order=True)

    if ddiff:
        assert False
    else:
        assert True


# !/usr/bin/env python
# -*- coding:utf-8 -*-
import os
import sys
from functools import reduce
from time import sleep

import numpy as np
import tensorflow as tf

ROOT_DIR = os.path.join(os.path.dirname(__file__), '..')
sys.path.append(ROOT_DIR)

from PKUBASE.utils import file_handle
from PKUBASE.utils import trie
from PKUBASE.utils import ques_idf
from PKUBASE.utils import prev_post_words
from word2vec import word2vec
from ltp import ltp
from PKUBASE.kb import knowledge_base

class NER(object):
    def __init__(self, myltp, myword2vec):
        # 参数
        self.sentence_max_len = 45

        # 加载前后词概率统计
        self.myprev_post_counter = prev_post_words.loadPrevPostCounterByFile()

        # TrieTree
        mentions_path = os.path.join(ROOT_DIR, 'PKUBASE', 'mention.mentions')
        mentions = file_handle.pickle2data(mentions_path)
        self.myTrieTree = trie.TrieTree(mentions)
        # LTP
        self.myltp = myltp

        # 加载IDF_dict
        idf_pickle_path = os.path.join(ROOT_DIR, 'PKUBASE', 'question_idf.pickle')
        self.idf_dict = ques_idf.IDFDict(idf_pickle_path)

        # word2vec
        self.myword2vec = myword2vec

        # 加载lstm_LM模型,获取输出结果

        lstm_lm_graph = tf.Graph()
        with lstm_lm_graph.as_default():
            self.lstm_lm_sess = tf.Session(config=config)
            lstm_lm_checkpoint_dir = os.path.join(ROOT_DIR, 'PKUBASE', 'ner', '20180620184339')
            lstm_lm_checkpoint_file = tf.train.latest_checkpoint(lstm_lm_checkpoint_dir)
            lstm_lm_saver = tf.train.import_meta_graph('%s.meta' % lstm_lm_checkpoint_file)
            lstm_lm_saver.restore(self.lstm_lm_sess, lstm_lm_checkpoint_file)
            self.lstm_lm_inputx = lstm_lm_graph.get_operation_by_name('input_x').outputs[0]
            self.lstm_lm_lens = lstm_lm_graph.get_operation_by_name('lens').outputs[0]
            self.lstm_lm_log_probs = lstm_lm_graph.get_operation_by_name('Predictions/log_probs').outputs[0]

        # 加载ner_lr模型,获取输出结果
        ner_lr_graph = tf.Graph()
        with ner_lr_graph.as_default():
            self.ner_lr_sess = tf.Session(config=config)
            ner_lr_checkpoint_dir = os.path.join(ROOT_DIR, 'PKUBASE', 'ner', '20180628200736')
            ner_lr_checkpoint_file = tf.train.latest_checkpoint(ner_lr_checkpoint_dir)
            ner_lr_saver = tf.train.import_meta_graph('%s.meta' % ner_lr_checkpoint_file)
            ner_lr_saver.restore(self.ner_lr_sess, ner_lr_checkpoint_file)
            self.ner_lr_input = ner_lr_graph.get_operation_by_name('input_x').outputs[0]
            self.ner_lr_outputs = ner_lr_graph.get_operation_by_name('outputs').outputs[0]

    def find_ne(self, id, question, entity_rerank=False, entitys=None):
        # print(question)
        # 计算各种特征
        current_ques = []
        current_mentions = []
        prev_features, post_features, prev_post_features, idf_features, ne_lens = [], [], [], [], []
        triedict_items = self.myTrieTree.count(question).items()
        for mention, count in triedict_items:
            ques = question.replace(mention, ' NER ')
            # 因为传的参数必须为list,所以需要用[ques]; 因为返回值也为list[list[]],所以只取第一个[0]
            cws_word = self.myltp.ltp_cws([ques])[0]
            question_word = [mention if w == 'NER' else w for w in cws_word]

            prev_feature, post_feature, prev_post_feature = self.myprev_post_counter.cal_pos_feature(question_word,
                                                                                mention)
            idf = self.idf_dict.get(mention)
            c_len = len(mention)

            # print('prev:%s\tpost:%s\tprev_post:%s\tidf:%s\tlen:%s' % (
            #     prev_feature, post_feature, prev_post_feature, idf, c_len))

            current_ques.append(cws_word)
            current_mentions.append(mention)
            prev_features.append(prev_feature)
            post_features.append(post_feature)
            prev_post_features.append(prev_post_feature)
            idf_features.append(idf)
            ne_lens.append(c_len)

        data_ids, data_lens = self.myword2vec.texts2ids(current_ques, sequence_length=self.sentence_max_len,
                                                        actual_len=True)

        print('Start ner lm model ......')
        split_by_batch_num = 16  # 这是在输入样例数太多时，为了避免显存不够而设置的最大样例数目，大于此数就分批处理
        lstm_lm_log_probs = []
        for ith in range((len(data_ids) // split_by_batch_num) + 1):
            # 判断目前的batch长度
            if (split_by_batch_num <= (len(data_ids) - split_by_batch_num * ith)):
                current_len = split_by_batch_num
            else:
                current_len = len(data_ids) % split_by_batch_num
            lstm_lm_log_probs.extend(self.lstm_lm_sess.run(self.lstm_lm_log_probs, feed_dict={
                self.lstm_lm_inputx: data_ids[ith * split_by_batch_num:ith * split_by_batch_num + current_len],
                self.lstm_lm_lens: data_lens[ith * split_by_batch_num:ith * split_by_batch_num + current_len]}))

            # try:
            #     log_probs.extend(self.lstm_lm_sess.run(self.lstm_lm_log_probs, feed_dict={
            #         self.lstm_lm_inputx: data_ids[ith * split_by_batch_num:ith * split_by_batch_num + current_len],
            #         self.lstm_lm_lens: data_lens[ith * split_by_batch_num:ith * split_by_batch_num + current_len]}))
            # except:
            #     print('%s \t total:%d' % (current_ques[0], len(data_ids)))
            #     exit(-1)

        lstm_lm_results = lstm_lm_log_probs

        print('Start ner lr model ......')
        # 合并特征
        final_features = list(
            zip(lstm_lm_results, idf_features, ne_lens, prev_features, post_features, prev_post_features))
        # print(final_features)

        scores = self.ner_lr_sess.run(self.ner_lr_outputs, feed_dict={self.ner_lr_input: final_features})
        idx_list = np.argsort(-scores).tolist()

        return [(current_ques[i], current_mentions[i], scores[i]) for i in idx_list]

class CrossBigramCounter(object):
    def __init__(self, path='cb.dict', thresh=2):
        fr = open(path, 'r')
        self.cb_dict = {}
        total = 0.0
        for line in fr:
            line = line.strip()
            cb, count_str = line.split()
            count = int(count_str)
            if count < thresh:
                break
            self.cb_dict[cb] = count
            total += count

        self.cb_dict = {cb: count / total for cb, count in self.cb_dict.items()}

    def get_freq(self, cb):
        return self.cb_dict.get(cb)

    def cal_cb_freq(self, sent1, sent2):
        assert isinstance(sent1, list)
        assert isinstance(sent2, list)

        freq_list = []
        # 修改NER替换符
        ne_idx = sent1.index('NER')
        for w1 in sent1[ne_idx + 1:]:
            if w1 in ['<S>', '_NE_', '</S>']:
                continue
            for w2 in sent2:
                cb = '%s_%s' % (w1, w2)
                if cb in self.cb_dict:
                    freq_list.append(self.get_freq(cb))
        if len(freq_list) > 0:
            return np.sum(freq_list)
        else:
            return 0.0

class PROP(object):
    def __init__(self, kb, myltp, myword2vec):
        self.max_ques_seq_len = 45
        self.max_prop_seq_len = 6

        self.kb = kb
        self.myltp = myltp
        self.myword2vec = myword2vec

        self.cb_counter = CrossBigramCounter(path=os.path.join(ROOT_DIR, 'PKUBASE', 'prop', 'cb.dict'))

        prop_graph = tf.Graph()
        with prop_graph.as_default():
            self.prop_sess = tf.Session(config=config)
            prop_checkpoint_dir = os.path.join(ROOT_DIR, 'PKUBASE', 'prop', 'runs', '1531728864', 'checkpoints')
            prop_checkpoint_file = tf.train.latest_checkpoint(prop_checkpoint_dir)
            prop_saver = tf.train.import_meta_graph('%s.meta' % prop_checkpoint_file)
            prop_saver.restore(self.prop_sess, prop_checkpoint_file)
            self.prop_ques_ids = prop_graph.get_operation_by_name('ques_ids').outputs[0]
            self.prop_ques_lens = prop_graph.get_operation_by_name('ques_lens').outputs[0]
            self.prop_prop_ids = prop_graph.get_operation_by_name('prop_ids').outputs[0]
            self.prop_prop_lens = prop_graph.get_operation_by_name('prop_lens').outputs[0]
            self.prop_lstm_input_keep_prob = prop_graph.get_operation_by_name('lstm_input_keep_prob').outputs[0]
            self.prop_lstm_output_keep_prob = prop_graph.get_operation_by_name('lstm_output_keep_prob').outputs[0]
            # Get outputs of graph
            try:
                self.prop_sem_sim_var = prop_graph.get_operation_by_name('similarity/sem_sim').outputs[0]
            except:
                self.prop_sem_sim_var = prop_graph.get_operation_by_name('mlp/sem_sim').outputs[0]

        prop_l2_graph = tf.Graph()
        with prop_l2_graph.as_default():
            self.prop_l2_sess = tf.Session(config=config)
            prop_l2_checkpoint_dir = os.path.join(ROOT_DIR, 'PKUBASE', 'prop', 'runs', '1531737377l2', 'checkpoints')
            prop_l2_checkpoint_file = tf.train.latest_checkpoint(prop_l2_checkpoint_dir)
            prop_l2_saver = tf.train.import_meta_graph('%s.meta' % prop_l2_checkpoint_file)
            prop_l2_saver.restore(self.prop_l2_sess, prop_l2_checkpoint_file)
            self.prop_l2_X = prop_l2_graph.get_operation_by_name('X').outputs[0]
            self.prop_l2_y = prop_l2_graph.get_operation_by_name('y').outputs[0]

    def get_other_features(self, cutted_rmne_ques, cutted_prop_list):
        def cal_prop_cov_rate(rmne_question, prop):
            return sum([int(ch in rmne_question) for ch in prop]) / float(len(prop))

        def cal_cov(rmne_question, prop):
            return sum([int(ch in rmne_question) for ch in prop])

        def cal_ques_cov_rate(rmne_question, prop):
            return sum([int(ch in rmne_question) for ch in prop]) / float(len(rmne_question))

        def cal_prop_wcov_rate(rmne_question, prop, stopwords=[]):
            rmne_question = [w for w in rmne_question if w not in stopwords]
            return sum([int(w in rmne_question) for w in prop]) / float(len(prop))

        def cal_ques_wcov_rate(rmne_question, prop, stopwords=[]):
            rmne_question = [w for w in rmne_question if w not in stopwords]
            return sum([int(w in prop) for w in rmne_question]) / float(len(rmne_question))

        stopwords = ['的', '是', '知道', '吗']

        features_list = []
        ques_str = ''.join(cutted_rmne_ques)
        for cutted_prop in cutted_prop_list:
            prop_str = ''.join(cutted_prop)
            features = []
            features.append(cal_ques_cov_rate(ques_str, prop_str))
            features.append(cal_prop_cov_rate(ques_str, prop_str))
            features.append(cal_ques_wcov_rate(cutted_rmne_ques, cutted_prop, stopwords=stopwords))
            features.append(cal_prop_wcov_rate(cutted_rmne_ques, cutted_prop, stopwords=stopwords))
            features.append(self.cb_counter.cal_cb_freq(cutted_rmne_ques, cutted_prop))

            features_list.append(features)

        return np.array(features_list)

    def get_scores(self, question, ne, IsEntity=False):
        try:
            if (IsEntity):
                prop_list = list(self.kb.query_by_name(ne).keys())
            else:
                prop_list = list(reduce(set.union, (set(entity.keys()) for entity in self.kb.query_by_alias(ne))))
                prop_list = [prop for prop in prop_list if len(prop) > 0]
        except Exception as e:
            print(e)
            print('Error ne is %s' % (ne))
            print(self.kb.alias_dict.get(ne, []))
            print(self.kb.query_by_alias(ne))
            return None

        prop_list_removed = [prop[1:-1] for prop in prop_list]
        prop_words = self.myltp.ltp_cws(prop_list_removed)
        ques = question.replace(ne, ' NER ')

        print(prop_words)
        questions = [ques for i in range(len(prop_words))]
        question_words = self.myltp.ltp_cws(questions)

        ques_ids, ques_lens = self.myword2vec.texts2ids(question_words, sequence_length=self.max_ques_seq_len,
                                                        actual_len=True)
        prop_ids, prop_lens = self.myword2vec.texts2ids(prop_words, sequence_length=self.max_prop_seq_len,
                                                        actual_len=True)

        sem_sims = self.prop_sess.run(self.prop_sem_sim_var, feed_dict={
            self.prop_ques_ids: ques_ids,
            self.prop_ques_lens: ques_lens,
            self.prop_prop_ids: prop_ids,
            self.prop_prop_lens: prop_lens,
            self.prop_lstm_input_keep_prob: 1.0,
            self.prop_lstm_output_keep_prob: 1.0,
        })
        other_features = self.get_other_features(question_words[0], prop_words)
        all_features = np.concatenate((sem_sims.reshape((-1, 1)), other_features), 1)
        scores = self.prop_l2_sess.run([self.prop_l2_y], feed_dict={self.prop_l2_X: all_features})
        scores = np.squeeze(scores)

        # 这是从小到大排序
        idx_list = np.argsort(scores).tolist()
        # 所以需要逆序,从大到小
        idx_list.reverse()
        scores = scores.tolist()
        print(idx_list)
        if (not (isinstance(scores, list))):
            scores = [scores]
        print(scores)

        return [(question, ne, prop_list[i], scores[i]) for i in idx_list]

if __name__ == '__main__':

    # 默认运行最终测试集
    if(len(sys.argv)==2 and sys.argv[1]=='test'):
        print('Test...')
        test_question_path = 'test_questions.pickle'
        ids, questions = file_handle.pickle2data(test_question_path)
    else:
        print('Final Test...')
        final_test_question_path = 'final_test_questions.pickle'
        ids, questions = file_handle.pickle2data(final_test_question_path)

    middle_pro_path = 'middle_data.txt'
    mfw = open(middle_pro_path, 'w', encoding='utf-8')

    final_answer_path = 'final_answer.txt'
    fw = open(final_answer_path, 'w', encoding='utf-8')

    kb_path = os.path.join(ROOT_DIR, 'PKUBASE', 'pkubase-triples.txt.prep')
    kb = knowledge_base.KnowledgeBase(kb_path)

    myword2vec = word2vec.MyWord2Vec('wiki300.pickle', whether_append_special_character=True)

    myltp = ltp.LTP()

    # gensim_word2vec_model = gensim_word2vec.get_gensim_word2vec_model('GoogleNews-vectors-negative300.bin',
    #                                                                   InWord2vecDIR=True)

    # tensorflow GPU settings
    config = tf.ConfigProto()
    config.gpu_options.allow_growth = True
    config.allow_soft_placement = True

    ner_model = NER(myltp, myword2vec)

    prop_model = PROP(kb, myltp, myword2vec)

    for id, question in zip(ids, questions):
        ne_idx = 0
        prop_idx = 0
        ner_result = ner_model.find_ne(id, question)
        for ques, ne, score in ner_result:
            print('%s\t%s\t%s' % (ques, ne, score))
            mfw.write('%s\t%s\t%s\n' % (ques, ne, score))
        right_ne = ner_result[ne_idx][1]
        print('The result of first ner is %s' % (right_ne))

        frist_prop_result = prop_model.get_scores(question, right_ne, IsEntity=False)
        if (frist_prop_result == None):
            fw.write('\n')
            mfw.write('\n')
            continue

        while (frist_prop_result == None or len(frist_prop_result) <= 3):
            print('%s : num of prop is less than thresh' % (right_ne))
            sleep(0.5)
            ne_idx += 1
            right_ne = ner_result[ne_idx][1]
            frist_prop_result = prop_model.get_scores(question, right_ne, IsEntity=False)

        first_right_prop = frist_prop_result[prop_idx][2]

        final_right_prop = first_right_prop
        final_prop_result = frist_prop_result

        for ques, ne, prop, score in frist_prop_result:
            print('%s\t%s\t%s\t%s' % (ques, ne, prop, score))
            mfw.write('%s\t%s\t%s\n' % (ne, prop, score))

        print('The result of final ner is %s' % (right_ne))
        print('The result of first prop is %s' % (first_right_prop))

        # 第2跳判断
        first_answer_set = kb.query_by_alias(right_ne)
        for entity in first_answer_set:
            if first_right_prop in entity:
                first_answer = entity[first_right_prop][0]

        print('The first answer is %s ' % (first_answer))

        if (first_answer[0] == '<'):
            question = question.replace(right_ne, '|||')

            for entity in first_answer_set:
                if first_right_prop in entity:
                    replace_list=[]
                    replace_list.append(first_right_prop)
                    replace_list.extend(entity[first_right_prop])
                    for ans in replace_list:
                        ans_w=set(ans[1:-1])
                        for w in ans_w:
                            question=question.replace(w,'')

            question = question.replace('|||', ' NER ')

            # cws_word = myltp.ltp_cws([question])[0]
            # first_right_prop_word=myltp.ltp_cws([first_right_prop])[0]
            # prop_wv=sum([myword2vec.get(w) for w in first_right_prop_word])
            # min_word=''
            # min_sim=float('inf')
            # for word in cws_word:
            #     word_wv=myword2vec.get(word)
            #     o_dis=1/(1+np.linalg.norm(prop_wv-word_wv))
            #     print('cos_sim: %s\t%s\t' % (word,o_dis))
            #     if(min_sim>o_dis):
            #         min_sim=o_dis
            #         min_word=word
            # question=question.replace(min_word,'')
            # print('After caluacte: %s' % (question))

            second_prop_result = prop_model.get_scores(question, first_answer, IsEntity=True)

            if (second_prop_result == None):
                print('second is None')
            else:
                if (second_prop_result[0][2] != first_right_prop and second_prop_result[0][3] >= 0.8):
                    # 如果选择第2跳
                    final_right_prop = second_prop_result[0][2]
                    right_ne = first_answer

                for ques, ne, prop, score in second_prop_result:
                    print('%s\t%s\t%s\t%s' % (ques, ne, prop, score))
                    mfw.write('%s\t%s\t%s\n' % (ne, prop, score))

        candi_answer_set = set()
        for entity in kb.query_by_alias(right_ne):
            if final_right_prop in entity:
                candi_answer_set.add('\t'.join(entity[final_right_prop]))

        # 用于保证尽量没有空行答案
        while (prop_idx<len(final_prop_result)-1 and len(candi_answer_set) == 0 and (
            final_prop_result[prop_idx][3] - final_prop_result[prop_idx + 1][3]) <= 0.15):
            prop_idx += 1

            final_right_prop = final_prop_result[prop_idx][2]

            candi_answer_set = set()
            for entity in kb.query_by_alias(right_ne):
                if final_right_prop in entity:
                    candi_answer_set.add('\t'.join(entity[final_right_prop]))

        print('final ne is %s,  final prop is %s' % (right_ne, final_right_prop))
        mfw.write('final ne is %s,  final prop is %s\n' % (right_ne, final_right_prop))

        answer_list = list(candi_answer_set)
        print(answer_list)
        fw.write('%s\n' % ('\t'.join(answer_list)))
        mfw.write('%s\t%s\t%s\n\n' % (question, final_right_prop, '\t'.join(answer_list)))


from test.LinearRegression.LegoTest.LegoFunction import *
from ml.Regress import RidgeRegress

X = []
Y = []

setDataCollect(X, Y)
# crossValid(X, Y, 10)
#
X = np.array(X)
Y = np.array(Y)

Xmean = X.mean(axis=0)
Ymean = Y.mean()
Xvar = X.var(axis=0)
X = (X - Xmean) / Xvar
# Y = Y - Ymean

theta, pred = RidgeRegress().regress(X, Y, np.exp(0))
# pred = LR(X, Y).LWR()
print('err is: ', (abs(pred - Y)).sum())


# This is a file of modified loss
import torch
from torch.autograd import Function
from torch import nn
import math

class MLOSS(nn.Module):
	def __init__(self, id_num, feature_size, batch_ave):
		super(MLOSS, self).__init__()
		self.id_num = id_num
		self.feature_size = feature_size
		self.batch_ave = batch_ave

	def forward(self, x, id, T=0.07):
		batch_size = x.size(0)
		feature_size = self.feature_size

		out = torch.mm(x, self.batch_ave.t())
		out.div_(T).exp_()

		normalizer = torch.sum(out, dim=1)
		out = (out.t()/normalizer).t()

		out = out.log_()
		loss = -out[:, id].sum()
		loss = loss / batch_size

		return loss


from unittest.mock import call

import pytest
from requests.exceptions import ConnectTimeout, HTTPError, ReadTimeout
from urllib3.exceptions import ReadTimeoutError

from qcodes import Parameter

class AnyStringWith(str):
    def __eq__(self, other):
        return self in other

@pytest.fixture(name='mock_webclient', autouse=True)
def setup_webclient(mocker):
    mock_slack_sdk_module = mocker.MagicMock(name='slack_sdk_module')
    mock_webclient = mocker.MagicMock(name='WebclientMock')
    mock_slack_sdk_module.WebClient = mocker.MagicMock()
    mock_slack_sdk_module.WebClient.return_value = mock_webclient
    mocker.patch.dict('sys.modules', slack_sdk=mock_slack_sdk_module)

    response = {'members': [{'name': 'dummyuser', 'id': 'DU123'}]}
    mock_webclient.users_list.return_value = response

    def mock_conversations_list(types):
        if 'im' in types.split(','):
            return {'channels': [{'user': 'DU123', 'id': 'CH234'}]}
        else:
            return None

    mock_webclient.conversations_list.side_effect = mock_conversations_list

    return mock_webclient

@pytest.fixture(name='slack')
def slack_fixture():
    return setup_slack()

def setup_slack():
    slack_config = {
        'bot_name': 'bot',
        'token': '123',
        'names': ['dummyuser']
    }
    import qcodes.utils.slack  # pylint: disable=import-outside-toplevel
    slack = qcodes.utils.slack.Slack(config=slack_config, auto_start=False)

    return slack

def test_convert_command_should_convert_floats():
    import qcodes.utils.slack  # pylint: disable=import-outside-toplevel
    cmd, arg, kwarg = qcodes.utils.slack.convert_command('comm 0.234 key=0.1')
    assert cmd == 'comm'
    assert arg == [pytest.approx(0.234)]
    assert kwarg == {'key': pytest.approx(0.1)}

def test_slack_instance_should_contain_supplied_usernames(slack):
    assert 'dummyuser' in slack.users.keys()

def test_slack_instance_should_get_config_from_qc_config():
    from qcodes import config as cf  # pylint: disable=import-outside-toplevel
    slack_config = {
        'bot_name': 'bot',
        'token': '123',
        'names': ['dummyuser']
    }
    cf.add(key='slack', value=slack_config)
    import qcodes.utils.slack  # pylint: disable=import-outside-toplevel
    slack = qcodes.utils.slack.Slack(config=None, auto_start=False)
    assert 'dummyuser' in slack.users.keys()

def test_slack_instance_should_start(mocker):
    slack_config = {
        'bot_name': 'bot',
        'token': '123',
        'names': ['dummyuser']
    }
    mock_thread_start = mocker.patch('threading.Thread.start')
    import qcodes.utils.slack  # pylint: disable=import-outside-toplevel
    _ = qcodes.utils.slack.Slack(config=slack_config)

    mock_thread_start.assert_called()

def test_slack_instance_should_not_start_when_already_started(mocker):
    slack_config = {
        'bot_name': 'bot',
        'token': '123',
        'names': ['dummyuser']
    }
    mock_thread_start = mocker.patch('threading.Thread.start')
    mock_thread_start.side_effect = RuntimeError

    import qcodes.utils.slack  # pylint: disable=import-outside-toplevel
    _ = qcodes.utils.slack.Slack(config=slack_config)

    mock_thread_start.assert_called()

def test_slack_instance_should_start_and_stop(mocker):
    slack_config = {
        'bot_name': 'bot',
        'token': '123',
        'names': ['dummyuser']
    }
    mocker.patch('threading.Thread.start')

    import qcodes.utils.slack  # pylint: disable=import-outside-toplevel
    slack = qcodes.utils.slack.Slack(config=slack_config, interval=0)
    slack.stop()

    assert not slack._is_active

def test_slack_instance_should_return_username_from_id(mock_webclient, slack):
    def mock_users_info(user):
        if user == 'DU123':
            return {'user': {'name': 'dummyuser', 'id': 'DU123'}}
        else:
            return None

    mock_webclient.users_info.side_effect = mock_users_info

    assert {'name': 'dummyuser', 'id': 'DU123'} == slack.user_from_id('DU123')

def test_slack_instance_should_get_im_ids(mock_webclient):
    def conversations_history(channel, limit=None):
        if channel == 'CH234':
            messages = [{'user': 'DU123', 'text': f'm{i}',
                         'ts': f'{45.5 + i}'} for i in range(limit)]
            response = {'messages': messages}
            return response
        else:
            return None

    mock_webclient.conversations_history.side_effect = conversations_history

    slack = setup_slack()

    assert slack.users['dummyuser']['im_id'] == 'CH234'
    assert slack.users['dummyuser']['last_ts'] == pytest.approx(45.5)

def test_slack_instance_should_get_im_ids_with_zero_messages(mock_webclient):
    def conversations_history(channel, limit=None):
        if channel == 'CH234':
            response = {'messages': []}
            return response
        else:
            return None

    mock_webclient.conversations_history.side_effect = conversations_history
    slack = setup_slack()

    assert slack.users['dummyuser']['last_ts'] is None

def test_slack_instance_should_get_im_messages_w_count(slack, mock_webclient):
    def conversations_history(channel, limit=None):
        if channel == 'CH234':
            messages = [{'user': 'DU123', 'text': f'message{i}'}
                        for i in range(limit)]
            response = {'messages': messages}
            return response
        else:
            return None

    mock_webclient.conversations_history.side_effect = conversations_history

    messages = slack.get_im_messages('dummyuser', count=3)
    assert len(messages) == 3

def test_slack_instance_should_get_im_messages_without_channel(mock_webclient):
    def conversations_history(channel, limit=None):
        if channel == 'CH234':
            messages = [{'user': 'DU123', 'text': f'm{i}',
                         'ts': f'{45.5 + i}'} for i in range(limit)]
            response = {'messages': messages}
            return response
        else:
            return None

    mock_webclient.conversations_history.side_effect = conversations_history

    def mock_conversations_list(types):
        if 'im' in types.split(','):
            return {'channels': []}
        else:
            return None

    mock_webclient.conversations_list.side_effect = mock_conversations_list

    slack = setup_slack()

    messages = slack.get_im_messages('dummyuser')
    assert len(messages) == 0

def test_slack_instance_should_get_new_im_messages(mock_webclient):
    def generator_function():
        total = 8
        new = 3
        while True:
            new_messages = [{'user': 'DU123', 'text': f'm{i}',
                             'ts': f'{45.5 + i}'} for i in range(total)][-new:]
            response = {'messages': new_messages}
            yield response
            total += new

    generator = generator_function()

    def conversations_history(channel, limit=None, oldest=None):
        if channel == 'CH234':
            return next(generator)
        else:
            return None

    mock_webclient.conversations_history.side_effect = conversations_history

    slack = setup_slack()

    new_messages = slack.get_new_im_messages()
    assert len(new_messages['dummyuser']) == 3

def test_slack_instance_should_update(slack):
    slack.update()
    assert slack.tasks == []

def test_slack_instance_should_update_with_task_returning_false(slack):
    slack.add_task('finished', channel='CH234')
    slack.update()
    assert slack.tasks == []

def test_slack_instance_should_update_with_task_returning_true(slack, mocker):
    mocker.patch('qcodes.utils.slack.active_loop', return_value=not None)

    slack.add_task('finished', channel='CH234')
    slack.update()
    task_added = slack.tasks[-1]

    assert 'Slack.check_msmt_finished' in str(task_added.func)

def test_slack_instance_should_update_with_exception(slack, mocker):
    method_name = 'qcodes.utils.slack.Slack.get_new_im_messages'
    mock_get_new_im_messages = mocker.patch(method_name)
    mocker.patch('warnings.warn')
    mocker.patch('logging.info')

    for exception in [ReadTimeout, HTTPError, ConnectTimeout,
                      ReadTimeoutError('pool', 'url', 'message')]:
        mock_get_new_im_messages.side_effect = exception
        slack.update()
        assert slack.tasks == []

def test_slack_instance_should_give_help_message(slack):
    message = slack.help_message()
    expected_message = '\nAvailable commands: `plot`, `msmt`, ' \
                       '`measurement`, `notify`, `help`, `task`'
    assert message == expected_message

def test_slack_instance_should_handle_messages(mock_webclient, slack):
    messages = {'dummyuser': [{'user': 'DU123', 'text': 'help'}]}
    slack.handle_messages(messages)
    expected_text = 'Results: \nAvailable commands: `plot`, ' \
                    '`msmt`, `measurement`, `notify`, `help`, `task`'
    expected_output = {'channel': 'CH234',
                       'text': expected_text}

    mock_webclient.chat_postMessage.assert_called_with(**expected_output)

def test_slack_inst_should_handle_messages_w_args_kw(mock_webclient, slack):
    text = 'task finished key=1'
    messages = {'dummyuser': [{'user': 'DU123', 'text': text}]}
    slack.handle_messages(messages)
    expected_output = {'channel': 'CH234', 'text': 'Added task "finished"'}
    mock_webclient.chat_postMessage.assert_called_with(**expected_output)

def test_slack_inst_should_handle_messages_w_parameter(mock_webclient, slack):
    slack.commands.update({'comm': Parameter(name='param')})
    messages = {'dummyuser': [{'user': 'DU123', 'text': 'comm'}]}
    slack.handle_messages(messages)
    expected_output = {'channel': 'CH234', 'text': 'Executing comm'}
    mock_webclient.chat_postMessage.assert_called_with(**expected_output)

def test_slack_inst_should_handle_messages_w_exception(mock_webclient, slack):
    messages = {'dummyuser': [{'user': 'DU123', 'text': 'help toomany'}]}
    slack.handle_messages(messages)
    text = 'TypeError: help_message() takes 1 positional argument ' \
           'but 2 were given\n'
    expected_output = {'channel': 'CH234', 'text': AnyStringWith(text)}
    mock_webclient.chat_postMessage.assert_called_with(**expected_output)

def test_slack_inst_should_handle_messages_w_unkn_cmd(mock_webclient, slack):
    messages = {'dummyuser': [{'user': 'DU123', 'text': 'comm'}]}
    slack.handle_messages(messages)
    text = 'Command comm not understood. Try `help`'
    expected_output = {'channel': 'CH234', 'text': text}
    mock_webclient.chat_postMessage.assert_called_with(**expected_output)

def test_slack_inst_should_add_unknown_task_command(mock_webclient, slack):
    slack.add_task('tcomm', channel='CH234')
    text = 'Task command tcomm not understood'
    expected_output = {'channel': 'CH234', 'text': text}
    mock_webclient.chat_postMessage.assert_called_with(**expected_output)

def test_slack_inst_should_upload_latest_plot(mock_webclient, slack, mocker):
    method_name = 'qcodes.utils.slack.BasePlot.latest_plot'
    mocker.patch(method_name, return_value=not None)
    mocker.patch('os.remove')
    slack.upload_latest_plot(channel='CH234')
    expected_output = {'channels': 'CH234', 'file': AnyStringWith('.jpg')}
    mock_webclient.files_upload.assert_called_with(**expected_output)

def test_slack_inst_should_not_fail_upl_latest_wo_plot(mock_webclient, slack):
    slack.upload_latest_plot(channel='CH234')
    expected_output = {'channel': 'CH234', 'text': 'No latest plot'}
    mock_webclient.chat_postMessage.assert_called_with(**expected_output)

def test_slack_inst_should_print_measurement(mock_webclient, slack, mocker):
    dataset = mocker.MagicMock()
    dataset.fraction_complete.return_value = 0.123
    mocker.patch('qcodes.utils.slack.active_data_set', return_value=dataset)

    slack.print_measurement_information(channel='CH234')

    print(mock_webclient.chat_postMessage.calls)

    text1 = f"Measurement is {0.123 * 100:.0f}% complete"
    expected_out1 = {"channel": "CH234", "text": text1}
    expected_out2 = {"channel": "CH234", "text": AnyStringWith("MagicMock")}
    actual = mock_webclient.chat_postMessage.call_args_list
    expected = [call(**expected_out1), call(**expected_out2)]
    assert actual == expected

def test_slack_inst_should_print_measurement_wo_latest(mock_webclient, slack):
    slack.print_measurement_information(channel='CH234')
    expected_output = {'channel': 'CH234', 'text': 'No latest dataset found'}
    mock_webclient.chat_postMessage.assert_called_with(**expected_output)


import botocore, boto3, colorama, datetime, sys
colorama.init()

from argparse import ArgumentParser
from time import sleep
from .const import complete_status, status_color
from .utils import start_boto_session, run_cloudformation_cli

def main():
  parser = ArgumentParser(prog='cfn-watcher', description='Wrapper around aws cloudformation which monitors the stack create/update')
  parser.add_argument('--profile', help='awscli configured profile')
  parser.add_argument('--stack-name', help='CloudFormation Stack Name')
  args, unknown = parser.parse_known_args()

  if len(sys.argv) < 2:
    parser.print_usage()
    sys.exit(1)

  return_code = run_cloudformation_cli(sys.argv[1:])

  if return_code is 0 and args.stack_name is not None:
    monitor_stack(args.profile, args.stack_name)
  else:
    sys.exit(return_code)

def monitor_stack(profile, stack_name):
  session = start_boto_session(profile)

  cfn_client = session.client('cloudformation')

  while True:
    try:
      response = cfn_client.describe_stacks(StackName=stack_name)
    except botocore.exceptions.ClientError as e:
      print("{timestamp} {color}ERROR{reset_color}: {error}".format(
        timestamp=datetime.datetime.now().strftime('%H:%M:%S'),
        color=colorama.Fore.RED,
        reset_color=colorama.Style.RESET_ALL,
        error=e))

      break

    try:
      response_stack = response['Stacks'][0]
      response_status = response_stack['StackStatus']

      if 'StackStatusReason' in response_stack:
        response_reason = ": " + response_stack['StackStatusReason']
      else:
        response_reason = ""

      print("{timestamp} {region}: {color}{response_status}{reset_color}: {stack_name}{response_reason}".format(
        timestamp=datetime.datetime.now().strftime('%H:%M:%S'),
        region=session.region_name,
        color=status_color[response_status],
        response_status=response_status,
        reset_color=colorama.Style.RESET_ALL,
        response_reason=response_reason,
        stack_name=response_stack['StackName']))

      if response_status in complete_status:
        break
      else:
        sleep(5)

    except KeyError as e:
      sys.stderr.write(u'{red}{bold}ERROR: Failed to retrieve stack status{reset}: {red}{error}{reset}\n'.format(
        red=colorama.Fore.RED,
        bold=colorama.Style.BRIGHT,
        error=e,
        reset=colorama.Style.RESET_ALL))

      break


from django.conf.urls import include, url

from . import views

from django.urls import path

urlpatterns = [

    url(r'^$', views.inicio, name="inicio"),
    url('perris/inicio', views.redirigir, name="redirigir"),
    url('perris/login', views.login , name="login"),
    url('perris/disponibles', views.perros_disponibles , name="perros_disponibles"),
    url('administrador',views.administrador, name="adm.inicio" ),
    path('agregarperro', views.agregarperro, name='new_post_perro'),
    path('eliminar/<int:pk>', views.eliminarperro, name='delete_post_perro'),
    url(r'^perro/(?P<pk>[0-9]+)/$', views.detallesperro,name='detail_post_perro'),
    path('perro/<int:pk>/editar/', views.editarperro, name='edit_post_perro'),

]


#!/usr/bin/env python
# -*- coding : utf8 -*-
from time import time as get_time
import memcache
import logging
from memcache_connection import ClusterClient
from conf_discover import get_cluster_info
from cache_metrics import CacheMetrics, NullMetrics

_METRIC_PREFIX = "web.memcacheclient"

class MemcacheClient(object):
    version = 1

    def __init__(self, service, psm=None, servers=None, key_tmpl=None, timeout=60, enable_metrics=True, **kwargs):
        """
        """
        cluster, psm, servers, service = get_cluster_info(service, psm, servers)
        self.cluster = cluster
        self.psm = psm
        self.servers = servers
        self.key_tmpl = key_tmpl
        self.timeout = timeout
        self.enable_metrics = enable_metrics
        self.kwargs = kwargs
        self.client = ClusterClient.get(cluster, servers, **kwargs)
        if enable_metrics:
            self.metrics = CacheMetrics(cluster, _METRIC_PREFIX, psm, service)
        else:
            self.metrics = NullMetrics()

    def __del__(self):
        if ClusterClient:
            ClusterClient.release(self.client, self.cluster, self.servers, **self.kwargs)

    def _make_key(self, key, key_tmpl=None):
        if key_tmpl:
            key = key_tmpl % key
        elif self.key_tmpl:
            key = self.key_tmpl % key

        return key

    def _get_timeout(self, timeout):
        return timeout if timeout!=None else self.timeout

    def _get_args(self, key, key_tmpl, timeout):
        return self._make_key(key, key_tmpl), self._get_timeout(timeout)

    def set(self, key, val, time=0, min_compress_len=0, key_tmpl=None):
        ts = get_time()

        key, time = self._get_args(key, key_tmpl, time)
        result = self.client.set(key, val, time, min_compress_len)

        self.metrics.add_call_metrics('set', get_time() - ts, result != 0)
        return result

    def get(self, key, key_tmpl=None):
        ts = get_time()

        key = self._make_key(key, key_tmpl)
        result = self.client.get(key)

        self.metrics.add_call_metrics('get', get_time() - ts, True)
        self.metrics.add_hit_miss_metrics('get', 1 if result != None else 0, 1 if result == None else 0)
        return result

    def add(self, key, val, time=0, min_compress_len=0, key_tmpl=None):
        ts = get_time()

        key, time = self._get_args(key, key_tmpl, time)
        result = self.client.add(key, val, time, min_compress_len)

        self.metrics.add_call_metrics('add', get_time() - ts, True)
        return result

    def append(self, key, val, time=0, min_compress_len=0, key_tmpl=None):
        ts = get_time()

        key, time = self._get_args(key, key_tmpl, time)
        result = self.client.append(key, val, time, min_compress_len)

        self.metrics.add_call_metrics('append', get_time() - ts, result != 0)
        return result

    def delete(self, key, time=0, key_tmpl=None):
        ts = get_time()

        key, time = self._get_args(key, key_tmpl, time)
        time = None
        result = self.client.delete(key, time)

        self.metrics.add_call_metrics('delete', get_time() - ts, result != 0)
        return result

    def _mset(self, mapping, time=0, key_prefix='', min_compress_len=0, key_tmpl=None):
        ts = get_time()

        time = self._get_timeout(time)
        ret = []
        if len(key_prefix) > 0:
            ret = self.client.set_multi(mapping, time, key_prefix, min_compress_len)
        else:
            key_map = {key: self._make_key(key, key_tmpl) for key in mapping.iterkeys()}
            reverse_key_map = {v: k for k, v in key_map.iteritems()}
            notset_keys = self.client.set_multi(
                {key_map[key]: val for key, val in mapping.iteritems()},
                time, '', min_compress_len)
            ret = [reverse_key_map[key] for key in notset_keys]

        self.metrics.add_call_metrics('mset', get_time() - ts, len(ret) == 0)
        return ret

    def mset(self, mapping, time=0, key_prefix='', min_compress_len=0, key_tmpl=None, split_num=1024):
        split_count = 0
        final = []
        if len(mapping) <= split_num:
            final = self._mset(mapping, time, key_prefix, min_compress_len, key_tmpl)
        else:
            def push_result(result):
                if isinstance(result, list):
                    for k in result:
                        final.append(k)

            sending = {}
            for key, value in mapping.items():
                sending[key] = value
                if len(sending) >= split_num:
                    push_result(self._mset(mapping, time, key_prefix, min_compress_len, key_tmpl))
                    split_count += 1
                    sending.clear()
            if len(sending):
                push_result(self._mset(mapping, time, key_prefix, min_compress_len, key_tmpl))
                split_count += 1

        self.metrics.add_batch_num_metrics('mset', len(mapping), split_count)
        return final

    def set_multi(self, mapping, time=0, key_prefix='', min_compress_len=0, key_tmpl=None, split_num=1024):
        return self.mset(mapping, time, key_prefix, min_compress_len, key_tmpl, split_num)

    def get_multi(self, keys, key_prefix='', key_tmpl=None, split_num=1024):
        return self.mget(keys, key_prefix, key_tmpl, split_num)

    def _mget(self, keys, key_prefix='', key_tmpl=None):
        ts = get_time()

        result = {}
        if len(key_prefix) > 0:
            result = self.client.get_multi(keys, key_prefix)
        else:
            key_map = {key: self._make_key(key, key_tmpl) for key in keys}
            reverse_key_map = {v: k for k, v in key_map.iteritems()}
            mapping = self.client.get_multi([key_map[key] for key in keys])
            result = {reverse_key_map[key]: val for key, val in mapping.iteritems()}

        self.metrics.add_call_metrics('mget', get_time() - ts, True)
        return result

    def mget(self, keys, key_prefix='', key_tmpl=None, split_num=1024):
        split_count = 0
        final = {}
        if len(keys) <= split_num:
            final = self._mget(keys, key_prefix, key_tmpl)
        else:
            def push_result(result):
                if isinstance(result, dict):
                    if isinstance(result, dict):
                        for k, v in result.items():
                            final[k] = v

            sending = []
            for key in keys:
                sending.append(key)
                if len(sending) >= split_num:
                    push_result(self._mget(sending, key_prefix, key_tmpl))
                    split_count += 1
                    sending = []
            if len(sending):
                push_result(self._mget(sending, key_prefix, key_tmpl))
                split_count += 1

        self.metrics.add_hit_miss_metrics('mget', len(final), len(keys) - len(final))
        self.metrics.add_batch_num_metrics('mget', len(keys), split_count)
        return final

    def delete_multi(self, keys, time=0, key_prefix='', key_tmpl=None):
        ts = get_time()

        time = None    #memcache not support time for delete
        result = 1
        if len(key_prefix) > 0:
            result = self.client.delete_multi(keys, time, key_prefix)
        else:
            new_keys = [self._make_key(key, key_tmpl) for key in keys]
            result = self.client.delete_multi(new_keys, time)

        self.metrics.add_call_metrics('delete_multi', get_time() - ts, result != 0)
        return result

    def replace(self, key, val, time=0, min_compress_len=0, key_tmpl=None):
        ts = get_time()
        key, time = self._get_args(key, key_tmpl, time)
        result = self.client.replace(key, val, time, min_compress_len)
        self.metrics.add_call_metrics('replace', get_time() - ts, result != 0)
        return result

    def prepend(self, key, val, time=0, min_compress_len=0, key_tmpl=None):
        ts = get_time()
        key, time = self._get_args(key, key_tmpl, time)
        result = self.client.prepend(key, val, time, min_compress_len)
        self.metrics.add_call_metrics('prepend', get_time() - ts, result != 0)
        return result

    def cas(self, key, val, time=0, min_compress_len=0, key_tmpl=None):
        ts = get_time()
        key, time = self._get_args(key, key_tmpl, time)
        result = self.client.cas(key, val, time, min_compress_len)
        self.metrics.add_call_metrics('cas', get_time() - ts, result != 0)
        return result

    def reset_cas(self):
        self.client.reset_cas()

    def gets(self, key, key_tmpl=None):
        ts = get_time()
        key = self._make_key(key, key_tmpl)
        result = self.client.gets(key)
        self.metrics.add_call_metrics('gets', get_time() - ts, True)
        self.metrics.add_hit_miss_metrics('gets', 1 if result != None else 0, 1 if result == None else 0)
        return result

    def incr(self, key, delta=1, key_tmpl=None):
        ts = get_time()
        key = self._make_key(key, key_tmpl)
        result = self.client.incr(key, delta)
        self.metrics.add_call_metrics('incr', get_time() - ts, True)
        return result

    def decr(self, key, delta=1, key_tmpl=None):
        ts = get_time()
        key = self._make_key(key, key_tmpl)
        result = self.client.decr(key, delta)
        self.metrics.add_call_metrics('decr', get_time() - ts, True)
        return result

#
#to compatible with pyutil/memcache/client.py
class PyutilMemcacheClient(object):

    def __init__(self, service, psm=None, servers=None, key_tmpl=None, timeout=60, enable_metrics=True, **kwargs):
        """
        """
        self.client = MemcacheClient(service, psm, servers, key_tmpl, timeout, enable_metrics, **kwargs)
        if self.client.key_tmpl == None:
            self.client.key_tmpl = self.client.cluster + '?%s'

    def set(self, key, val, timeout=None, key_tmpl=None):
        return self.client.set(key, val, timeout, key_tmpl=key_tmpl)

    def get(self, key, key_tmpl=None):
        return self.client.get(key, key_tmpl=key_tmpl)

    def add(self, key, val, timeout=None, key_tmpl=None):
        return self.client.add(key, val, timeout, key_tmpl=key_tmpl)

    def append(self, key, val, timeout=None, key_tmpl=None):
        return self.client.append(key, val, timeout, key_tmpl=key_tmpl)

    def delete(self, key, key_tmpl=None):
        return self.client.delete(key, key_tmpl=key_tmpl)

    def mset(self, mapping, timeout=None, key_tmpl=None, split_num=1024):
        return self.client.mset(mapping, timeout, key_tmpl=key_tmpl, split_num=split_num)

    def set_multi(self, mapping, timeout=None, key_tmpl=None, split_num=1024):
        return self.mset(mapping, timeout, key_tmpl, split_num)

    def get_multi(self, keys, key_tmpl=None, split_num=1024):
        return self.mget(keys, key_tmpl, split_num)

    def mget(self, keys, key_tmpl=None, split_num=1024):
        return self.client.mget(keys, key_tmpl=key_tmpl, split_num=split_num)

    def delete_multi(self, keys, timeout=0, key_tmpl=None):
        return self.client.delete_multi(keys, timeout, key_tmpl=key_tmpl)

    def _len(self, data):
        if isinstance(data, basestring):
            return len(data)
        elif isinstance(data, (list, tuple)):
            l = 0
            for v in data:
                l += self._len(v)
            return l
        elif isinstance(data, dict):
            l = 0
            for k, v in data.items():
                l += self._len(k) + self._len(v)
            return l
        elif data is None or data is True or data is False:
            return 0
        elif isinstance(data, (int, float, long)):
            return len(str(data))
        else:
            return 0

    def replace(self, key, val, timeout=0, key_tmpl=None):
        return self.client.replace(key, val, timeout, key_tmpl=key_tmpl)

    def prepend(self, key, val, timeout=0, key_tmpl=None):
        return self.client.prepend(key, val, timeout, key_tmpl=key_tmpl)

    def cas(self, key, val, timeout=0, key_tmpl=None):
        return self.client.cas(key, val, timeout, key_tmpl=key_tmpl)

    def reset_cas(self):
        return self.client.reset_cas()

    def gets(self, key, key_tmpl=None):
        return self.client.gets(key, key_tmpl=key_tmpl)

    def incr(self, key, delta=1, key_tmpl=None):
        return self.client.incr(key, delta, key_tmpl=key_tmpl)

    def decr(self, key, delta=1, key_tmpl=None):
        return self.client.decr(key, delta, key_tmpl=key_tmpl)


def main():
    s1 = input()
    n = int(input())

    c = 0
    for i in range(n):
        s2 = input()
        s2 = s2[-len(s1):] + s2
        if s1 in s2:
            c += 1
    print(c)

if __name__ == '__main__':
    main()


import web

urls = (
    '/(.*)', 'Prime'
)
app = web.application(urls, globals())

class Prime:
    def GET(self, number):
        try:
            nr = int(number)
            for d in range(2, nr//2):
                if nr%d == 0:
                    return False
            return True

        except Exception:
            return "not a number"

if __name__ == "__main__":
    app.run()


import Orange.evaluation
import matplotlib.pyplot as plt
from scipy.stats import friedmanchisquare
from scipy.stats import rankdata
import sys

def omnibus_posthoc(resultados, filename):
    # Friedman + Nemenyi
    stat, p = friedmanchisquare(resultados['Logistic Regression'], resultados['Balanced Random Forest'],
                                resultados['Balanced Bagging'], resultados['Easy Ensemble'], resultados['RUS Boost'])
    sys.stdout = open("results/friedman_nemenyi-" + filename + ".txt", "w")
    print(print('stat=%.3f, p=%.3f' % (stat, p)))
    if p > 0.05:
        print('Nao ha dif estatistica')
    else:
        print('Ha dif estatistica')

    # Ranking Data
    # * -1 porque quanto menor melhor
    ranking = (resultados * -1).apply(lambda row: rankdata([row['Logistic Regression'], row['Balanced Random Forest'],
                                                           row['Balanced Bagging'], row['Easy Ensemble'],
                                                           row['RUS Boost']]), axis=1)
    # Imprime o resultado do teste de friedman e nemenyi
    print(ranking)
    sys.stdout.close()

    names = resultados.columns.values
    avranks = ranking.mean()
    cd = Orange.evaluation.compute_CD(avranks, len(ranking), alpha="0.1")  # numero de linhas da tabela
    Orange.evaluation.graph_ranks(avranks, names, cd=cd, width=6, textspace=1.5)
    plt.savefig('results/critical_distance-' + filename)


from django.forms import ModelForm,Textarea
from django import forms
from .models import Notes

class Notes_Form(forms.ModelForm):
    class Meta:
        model = Notes
        fields=['title','note']
        widgets = {
            'note': Textarea(attrs={'cols': 80, 'rows': 20}),
        }