from CMGTools.HToZZTo4Leptons.macros.plotters.TreePlotter import *
from CMGTools.HToZZTo4Leptons.macros.stat.PDFFactory import *
from CMGTools.HToZZTo4Leptons.macros.param.HiggsTheory import *
from CMGTools.HToZZTo4Leptons.tools.fullPath import getFullPath

import ROOT
import array
import os
ROOT.gSystem.Load("libHiggsAnalysisCombinedLimit")

class HiggsShapes(object):
    def __init__(self,finalstate,cuts,masses,prod,period):
        #make pdfs
        if os.path.exists(finalstate+'_'+period+'.json'):
            self.fpjson=open(finalstate+'_'+period+'.json')
            self.data = json.load(self.fpjson)
        else:
            self.data=dict()

        if 'shape' not in self.data:
            self.data['shape']=dict()

        self.theory = HiggsTheory(getFullPath('data/YR_'+period+'.json'))

        self.graph_m0    = ROOT.TGraphErrors()
        self.graph_sigma = ROOT.TGraphErrors()
        self.graph_alphaL = ROOT.TGraphErrors()
        self.graph_alphaR = ROOT.TGraphErrors()
        self.graph_nL = ROOT.TGraphErrors()
        self.graph_nR = ROOT.TGraphErrors()

        self.finalstate=finalstate
        self.period=period

        for i,mass in enumerate(masses):
            self.w = ROOT.RooWorkspace('w')
            self.pdfFactory = PDFFactory(self.w)
#            self.w.factory('H_Mass[125,70,800]')
            plotter = TreePlotter("All_"+period+"/"+prod+str(mass)+".root","FourLeptonTreeProducer/tree",'1')

            #data MC corrections
            plotter.addCorrectionFactor('eventWeight','eventWeight',0.0,'lnN')
            plotter.addCorrectionFactor('vertexWeight','vertexWeight',0.0,'lnN')
            plotter.addCorrectionFactor('eff','H_eff',0.0,'lnN')
            #define final state separation here
            finalStateCut=cuts
            if finalstate=='MuMu':
                finalStateCut+='&&abs(H_Z1_leg1_PdgId)==13&&abs(H_Z2_leg1_PdgId)==13'
            if finalstate=='EleEle':
                finalStateCut+='&&abs(H_Z1_leg1_PdgId)==11&&abs(H_Z2_leg1_PdgId)==11'
            if finalstate=='MuEle':
                finalStateCut+='&&abs(H_Z1_leg1_PdgId)!=abs(H_Z2_leg1_PdgId)'

            #make weighted dataset

            theory = self.theory.getInterp(mass)
            width=float(theory['width'])

            data,obs = plotter.makeDataSet('H_Mass',finalStateCut,[mass-50-3*width],[mass+50+5*width],"data_obs",500000)
            getattr(self.w,'import')(data)

            #make pdf
            if mass<=160.:
                self.pdfFactory.makeDCB('pdf'+finalstate+str(mass),'H_Mass',mass,finalstate)
                self.w.var('CMS_scale_m').setConstant(1)
                self.w.var('CMS_scale_e').setConstant(1)
                self.w.var('CMS_res_m').setConstant(1)
                self.w.var('CMS_res_e').setConstant(1)
                self.w.var('m0').setVal(mass+(26.5702) + (-0.692027*mass) + (0.00680888*mass*mass) + (-3.24132e-05*mass*mass*mass) + (7.37704e-08*mass*mass*mass*mass) + (-6.37514e-11*mass*mass*mass*mass*mass))

            else:
                if mass<399:
                    self.pdfFactory.makeBW('pdfBW'+finalstate+str(mass),'H_Mass')
                    self.w.var('width').setConstant(1)
                else:
                    self.pdfFactory.makeBWHighMass('pdfBW'+finalstate+str(mass),'H_Mass')
                self.w.var('MH').setVal(float(mass))
                self.w.var('MH').setConstant(1)

                self.pdfFactory.makeDCB('pdfRes'+finalstate+str(mass),'H_Mass',0.0,finalstate)
                self.w.var('CMS_scale_m').setConstant(1)
                self.w.var('CMS_scale_e').setConstant(1)
                self.w.var('CMS_res_m').setConstant(1)
                self.w.var('CMS_res_e').setConstant(1)

                self.pdfFactory.makeConvolution('pdf'+finalstate+str(mass),'H_Mass','pdfBW'+finalstate+str(mass),'pdfRes'+finalstate+str(mass))
                self.w.var('m0').setVal((26.5702) + (-0.692027*mass) + (0.00680888*mass*mass) + (-3.24132e-05*mass*mass*mass) + (7.37704e-08*mass*mass*mass*mass) + (-6.37514e-11*mass*mass*mass*mass*mass))

            #set defaults
            self.w.var('alphaL').setVal((-21.6016) + (0.602213*mass) + (-0.00620193*mass*mass) + (3.03257e-05*mass*mass*mass) + (-7.05553e-08*mass*mass*mass*mass) + (6.28615e-11*mass*mass*mass*mass*mass))
            self.w.var('alphaR').setVal((110.088) + (-2.74006*mass) + (0.0263436*mass*mass) + (-0.000120005*mass*mass*mass) + (2.59845e-07*mass*mass*mass*mass) + (-2.1514e-10*mass*mass*mass*mass*mass))
            self.w.var('nL').setVal((-261.056) + (5.8594*mass) + (-0.047926*mass*mass) + (0.000186032*mass*mass*mass) + (-3.48259e-07*mass*mass*mass*mass) + (2.53758e-10*mass*mass*mass*mass*mass))
            self.w.var('sigma').setVal((-42.9661) + (1.16387*mass) + (-0.0116153*mass*mass) + (5.54285e-05*mass*mass*mass) + (-1.25301e-07*mass*mass*mass*mass) + (1.07428e-10*mass*mass*mass*mass*mass))

            result = self.w.pdf('pdf'+finalstate+str(mass)).fitTo(self.w.data('data_obs'),ROOT.RooFit.SumW2Error(1),ROOT.RooFit.Save(1),ROOT.RooFit.Minimizer("Minuit2"))

            c=ROOT.TCanvas('c')
            c.cd()

            frame=self.w.var('H_Mass').frame()
            self.w.data('data_obs').plotOn(frame)
            self.w.pdf('pdf'+finalstate+str(mass)).plotOn(frame)
            frame.Draw()
            c.SaveAs('validation_pdf'+finalstate+str(mass)+'_'+period+'.png')

            self.data['shape'][mass]=dict()
            for param in self.pdfFactory.params:
                self.data['shape'][mass][param]=dict()
                self.data['shape'][mass][param]['value']=self.w.var(param).getVal()
                self.data['shape'][mass][param]['error']=self.w.var(param).getError()
                getattr(self,'graph_'+param).SetPoint(i,float(mass),self.w.var(param).getVal())
                getattr(self,'graph_'+param).SetPointError(i,0.0,self.w.var(param).getError())

        self.fit()
        self.save()

    def save(self):
        f=open(self.finalstate+'_'+self.period+'.json','w')
        json.dump(self.data,f)
        f.close()

    def fit(self,mini=0.,maxi=1000):

        fil=open('shape_'+self.finalstate+'_'+self.period+'.txt','w')

        f=ROOT.TF1("f","pol5", mini, maxi)
        self.data['shapeFormula'] =dict()

        for param in ['m0','sigma','alphaL','nL','alphaR','nR']:
            getattr(self,'graph_'+param).Fit(f)
            getattr(self,'graph_'+param).SetMarkerStyle(20)

            c=ROOT.TCanvas('c')
            c.cd()
            getattr(self,'graph_'+param).Draw("AP")
            f.Draw("LSAME")
            c.SaveAs('shape_'+self.finalstate+'_'+self.period+'_'+param+'.png')

            formula = '('+str(f.GetParameter(0))+'+('+str(f.GetParameter(1))+'*@0)+'+'+('+str(f.GetParameter(2))+'*@0*@0)+('+str(f.GetParameter(3))+'*@0*@0*@0)+('+str(f.GetParameter(4))+'*@0*@0*@0*@0)+('+str(f.GetParameter(5))+"*@0*@0*@0*@0*@0))"
            self.data['shapeFormula'][param] = formula
            fil.write(param +'='+str(f.GetParameter(0))+'+('+str(f.GetParameter(1))+'*@0)+'+'+('+str(f.GetParameter(2))+'*@0*@0)+('+str(f.GetParameter(3))+'*@0*@0*@0)+('+str(f.GetParameter(4))+'*@0*@0*@0*@0)+('+str(f.GetParameter(5))+"*@0*@0*@0*@0*@0)")
        fil.close()


# -*- coding: utf-8 -*-
"""
Created on tue sept 12 12:00:00 2017

@author: tm
This is some tools for graphs
"""

class Graph():
    """
    Class for an oriented graph:
    [[-1,-1],
    [1,2]]
    The graph have a link from <1> to <0> with a weight of 1
    and one from <1> to <1> with a weight of 2
    """
    def __init__(self, dim):
        self.dim = dim
        self.g   = []
        self.parcour = []
        for x in range(dim):
            self.g.append([-1]*dim)
        # self.buildParcour()

    def addLink(self, p1, p2, val):
        """
        This function set the path from <p1>, to <p2> with the value <val>
        """
        self.g[p1][p2] = val

    def buildParcour(self):
        """
        GOAL: fill in the self.parcour as a list of graph
        list of graph for each letter as a starting point
        """
        self.parcour = [[([], -1) for x in range(self.dim)]\
                                    for y in range(self.dim)]
        for z in range(self.dim): #For each letters
            this = [[([], -1, False) for x in range(self.dim)] for y in range(self.dim)]
            #[] -> Path, -1 -> Weight, False -> Has already been seen
            for y in range(self.dim):
                currentID = -1
                if this[0] == [([], -1, False)] * self.dim:
                    currentID = 0
                else:
                    MIN = (float('inf'), -1)
                    for ID in range(self.dim):
                        for x in range(self.dim):
                            if not this[ID][x][2] and\
                               this[ID][x][1] != -1 and\
                               this[ID][x][1] < MIN[0]:
                                MIN = (this[ID][x][1], x)
                    currentID = MIN[1]
                print("Picking ")
                #Get new weights
                for x in range(self.dim):
                    if not this[currentID][x][2]: #Not visited yet
                        prev = this[currentID][currentID][1] + self.g[currentID][x]
                        if this[currentID][x][1] != -1 and this[currentID][x][1] < prev:
                            this[currentID][x] = (this[currentID][currentID][0] + [x], prev, False)

    def getPath(self, p1, p2):
        """
        <p1> and <p2> letters ID
        This fuction return (<p>, <l>) as:
            p: is the list of points
            l: is a lenght of the path to take
        if no route is found, return ([], -1)
        """
        return self.parcour[p1][p1][p2]

    def toDot(self, name="my_graph"):
        """
        This function transphorm the graph into DOT format
        It is usefull to print it with Graphviz
        """
        res = "digraph " + name + "{\n"
        for y in range(self.dim):
            for x in range(self.dim):
                if self.g[x][y] != -1:
                    res += "{} -> {};\n".format(x, y)
        return res + "}"


import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from keras.models import load_model
from keras.models import Sequential, Model
from keras.layers import Dense,Dropout,LSTM, Input
from keras import initializers
import keras.backend as K

from sklearn import metrics
import pandas as pd
import time
import pdb
import utils

isLSTM=False

m=128
length=128
epoch=500
batch_size= 5120

chs = 100000
nrof_aut = int(chs/m)
nchs = nrof_aut*m

#===========================================================================
y_file = "csapuf-128-res-stb-nf-2642048"
#===========================================================================

dfx=pd.read_csv('dataset/dataset-128bit-challenge/csa_stable_chs/stb_parity-2642048.csv',header=None)
dfy=pd.read_csv('response/seed42/csapuf/CRPseed29/{}.csv'.format(y_file),header=None)
#===========================================================================
# fix random seed for reproducibility
np.random.seed(31)
# split into input (X) and output (Y) variables
X = dfx.values[:nchs].reshape(-1, m, length+1)
Y = dfy.values[:nrof_aut]

nbatches = Y.shape[0]
train_test_idx = np.arange(nbatches)
# Shuffle
np.random.shuffle(train_test_idx)
X = X[train_test_idx]
Y = Y[train_test_idx]

train_test_split = int(nbatches*0.9)
train_features = X[:train_test_split].reshape(-1,length+1)
test_features = X[train_test_split:].reshape(-1,length+1)
train_labels = Y[:train_test_split].flatten()
test_labels = Y[train_test_split:].flatten()

if isLSTM:
    train_features = train_features.reshape(-1, m, length+1)
    test_features = test_features.reshape(-1, m, length+1)
    train_labels = train_labels.reshape(-1, m, 1)
    test_labels = test_labels.reshape(-1, m, 1)

#pdb.set_trace()

# create model
if isLSTM:
    X_input = Input(train_features.shape[1:])
    X = LSTM(50, return_sequences=True, input_shape = (m,length+1), name = 'LSTM_1')(X_input)
    X = LSTM(50, return_sequences=True)(X)
    #X = LSTM(50, return_sequences=True)(X)
    #X = LSTM(50, return_sequences=True)(X)
    #X = Dense(50, activation='relu')(X)
    #X = Dense(50, activation='relu')(X)
    X = Dense(1, activation='sigmoid')(X)
    ##The model object
    model = Model(inputs = X_input, outputs = X, name='LSTMModel')
else:
    model = Sequential()
    model.add(Dense(15,input_dim=length+1, activation='relu'))
    model.add(Dense(50, activation='relu'))
    #model.add(Dense(200,input_dim=length+1, activation='relu'))
    #model.add(Dense(200, activation='relu'))
    #model.add(Dense(200, activation='relu'))
    #model.add(Dense(200, activation='relu'))
    #model.add(Dense(200, activation='relu'))
    #model.add(Dense(200, activation='relu'))
    model.add(Dense(1, activation='sigmoid'))

# print the model
model.summary()

# Compile model
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])#,f1])

# fit the model
results = model.fit(
 train_features, train_labels,
 epochs= epoch,
 #verbose=1,
 batch_size = batch_size,
 validation_data = (test_features, test_labels)
)
# evaluvate the model
scores = model.evaluate(test_features, test_labels)
print("\n{}: {}, std={}".format(model.metrics_names[1], scores[1], np.std(scores[1])))

#pdb.set_trace()
#print("save model...")
#outfile = "csapuf-128-noisyres-600k_train"
#model.save('model/{}.h5'.format(outfile))


import torch
import torch.nn as nn
from Model import Model
import os
import cv2
from Split import splitImg

# 定义数据
# x:输入数据
# y:标签

def GetLabels(txt):

    labels = []
    fh = open(txt, 'r')
    imgs = []
    for line in fh:
        line = line.strip('\n')
        line = line.rstrip()
        words = line.split()
        print("  line =  ")
        print(line)
        phase = words[3]
        color = words[4]
        temp = [phase,color]
        labels.append(temp)

    return labels

def MyDataSets(imgPath0,labels):

    AllPositionList = []
    ThreshVal =200
    positionIndex= 0

    for filename in os.listdir(imgPath0):
        print(filename)
        imgPath = imgPath0 + filename

        # 读入原始图像
        origineImage = cv2.imread(imgPath)

        (h, w,_) = origineImage.shape
        img = origineImage[0:int(h / 1), :]
        AllPositionList = splitImg(img, positionIndex, ThreshVal,56,labels)
        print(" len(AllPositionList) = ")
        print(len(AllPositionList))

if __name__ == '__main__':

    # txt = "20210612-5.txt"
    txtPath = "F:/liudongbo/dataSet/20210809-1.txt"
    labels = GetLabels(txtPath)
    print("  labels  =  ")
    print(labels)

    imgFile = "F:/liudongbo/dataSet/bmp/20210809-1/"

    MyDataSets(imgFile,labels)

    x = torch.Tensor([[0.2, 0.4], [0.2, 0.3], [0.3, 0.4],[0.9,0.6]])
    y = torch.Tensor([[0.6], [0.5], [0.7],[0.8]])

    net = Model()
    Epoch = 10000

    for i in range(Epoch):
        # print("  i = ")
        # print(i)
        # print(" x = ")
        # print(x)
        # print("  y = ")
        # print(y)
        net.train(x, y)

    PATH = "F:/liudongbo/projects/net_params.pkl"
    torch.save(net, PATH)

    x1 = torch.Tensor([[0.9, 0.6]])
    out = net.test(x1)
    print(out)  # 输出结果 tensor([[0.5205]], grad_fn=<AddmmBackward>)


from django.conf import settings
from django.contrib.auth.models import User
from django.core.mail import EmailMultiAlternatives, get_connection
from django.template import loader

def send_email(recipients, subject, template, context):
    """
    Sends a multi-part (text and optionally HTML) email generated from templates
    """
    html_template = loader.get_template(template + ".html")
    text_template = loader.get_template(template + ".txt")

    html = html_template.render(context)
    text = text_template.render(context)

    send_raw_email(recipients, subject, text, html)

def send_raw_email(recipients, subject, text, html):
    """
    Sends and multi-part (text and optionally HTML) email to a list of users or email addresses
    """
    to_addresses = []
    for recipient in recipients:
        if isinstance(recipient, User):
            to_addresses.append(recipient.email)
        elif isinstance(recipient, str):
            to_addresses.append(recipient)
        else:  # pragma: no cover
            raise ValueError("Email recipients must users or email addresses")

    from_address = getattr(settings, "DEFAULT_FROM_EMAIL", "website@casepro.io")

    if getattr(settings, "SEND_EMAILS", False):
        # send individual messages so as to not leak users email addresses, but use bulk send operation for speed
        messages = []
        for to_address in to_addresses:
            message = EmailMultiAlternatives(subject, text, from_email=from_address, to=[to_address])
            if html:
                message.attach_alternative(html, "text/html")
            messages.append(message)

        get_connection().send_messages(messages)
    else:  # pragma: no cover
        print("FAKE SENDING this email to %s:" % ", ".join(to_addresses))
        print("--------------------------------------- text -----------------------------------------")
        print(text)
        if html:
            print("--------------------------------------- html -----------------------------------------")
            print(html)


from pprint import pprint

with open("recipes.txt", "r", encoding='utf-8-sig') as f:
    cook_book = {}

    while True:
        name = f.readline().rstrip()
        if not name:
            break
        number = int(f.readline().rstrip())
        ingredient = []
        for i in range(number):
            split_str = f.readline().rstrip().split(' | ')
            ingredient_list = {'ingredient_name': split_str[0],
                               'quantity': int(split_str[1]),
                               'measure': split_str[2]}
            ingredient.append(ingredient_list)
        f.readline()
        cook_book[name] = ingredient

def get_shop_list_by_dishes(dishes=[], person_count=0):
    list_by_dishes = {}
    for dish in dishes:
        if dish in cook_book:
            cook_book_dish = cook_book[dish]
            for i in cook_book_dish:
                quantity_i = 0
                if i['ingredient_name'] in list_by_dishes:
                    quantity_i = list_by_dishes[i['ingredient_name']]['quantity']
                list_by_dishes.update(
                    {i['ingredient_name']: {'measure': i['measure'],
                                            'quantity': quantity_i + i['quantity'] * person_count}
                     })
        else:
            print(f'Блюдо \'{dish}\' отсутствует, проверьте правильность ввода.')
    return list_by_dishes

pprint(get_shop_list_by_dishes(['Фахитос', 'Умлет'], 2))


l = input().split()
l = list(''.join([str(i)*int(l[i]) for i in range(10)]))
l.sort()
count = 0
for i in l:
    if i == '0':
        count = count + 1
    else:
        break
print("{}{}".format(l[count],''.join(l[:count]+l[count+1:])))


#!/usr/bin/env python
# coding: utf-8

# In[26]:

######### Find the longest word
def longestWord(wordlist):
    result = wordlist[0]
    for x in range(1,len(wordlist)):
        len1 = len(wordlist[x])
        len2 = len(result)
        if len1 > len2:
            result = wordlist[x]

    print('The longest word from the list is: '+result)

wordlist = ['sourav','Ramakrishnan','Sagar','lavanya']
longestWord(wordlist)
#########


import numpy as np
import matplotlib.pyplot as plt

from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.gaussian_process.kernels import RBF,WhiteKernel
import logging

from GaussianProcessTools.stageopt import optimizer as stageOpt
from GaussianProcessTools.lineBO import optimizer as lineOpt

def ofunc(x):
    return 5 - x**2

def cfunc1(x):
    return 2 - (x + 0.5)**2

def cfunc2(x):
    return ((1 / (1 + np.exp(x))) - 0.5)*2

def test1D():
    inputs = {}
    inputs['bounds'] = np.array((-5,5)).reshape(-1,2)
    kernel = RBF(1.0,'fixed')
    inputs['gprf'] = GaussianProcessRegressor(kernel,alpha = 0.01)
    inputs['gprc'] = [GaussianProcessRegressor(kernel,alpha = 0.01),GaussianProcessRegressor(kernel,alpha = 0.01)]

    inputs['X0'] = np.array((-1.1)).reshape(-1,1)
    inputs['ofun'] = ofunc
    inputs['cfun'] = [cfunc1,cfunc2]

    inputs['Y0'] = ofunc(inputs['X0'])
    inputs['C0'] = np.array([f(inputs['X0']) for f in inputs['cfun']]).T[0]
    inputs['h'] = np.zeros(len(inputs['cfun']))
    inputs['n_grid'] = 50
    inputs['T0'] = 5
    inputs['verbose'] = False

    opt = optimizer.StageOpt(inputs)

def main():
    test1D()

main()
plt.show()


from openerp import fields, models, api, _
from openerp.exceptions import ValidationError

import logging

_logger = logging.getLogger(__name__)

class StockPicking(models.Model):

	_inherit = "stock.picking"

	issue_id = fields.Many2one("warehouse.stock.issue", string="Issue")
	request_id = fields.Many2one("warehouse.stock.request", string="Request")
	is_dc = fields.Boolean(string="DC", default=False)
	warehouse_dc_id = fields.Many2one('dc.warehouse', string="Dc")
	location_type = fields.Selection([('raw', 'Raw Materials'),
										('manufacture', 'Manufacturing'),
										('semi-finished', 'Semi Finished'),
										('finished', 'Finished'),
										], string='Type', compute="_get_location")
	sale_bill_type = fields.Selection([('normal', 'Normal'), ('special', 'Special'), ('extra', 'Extra')], string="Bill Type", compute="_get_values")
	normal_discount = fields.Float("Normal Disc", compute="_get_values")
	extra_discount = fields.Float("Extra Disc", compute="_get_values")

	@api.depends()
	def _get_values(self):
		for picking in self:
			if picking.group_id:
				sale_ids = self.env['sale.order'].search([('procurement_group_id', '=', picking.group_id.id)])
				if sale_ids:
					picking.sale_bill_type = sale_ids[0].partner_selling_type
					if sale_ids[0].partner_selling_type == "normal":
						picking.normal_discount = sale_ids[0].normal_disc
						picking.extra_discount = sale_ids[0].extra_discount
					elif sale_ids[0].partner_selling_type == "extra":
						picking.normal_discount = sale_ids[0].normal_disc
						picking.extra_discount = sale_ids[0].nonread_extra_disocunt
					else:
						picking.normal_discount = sale_ids[0].nonread_normal_disocunt
						picking.extra_discount = sale_ids[0].nonread_extra_disocunt
					# res[picking.id] = sale_ids[0].partner_selling_type

	@api.depends()
	def _get_location(self):
		for res in self:
			res.location_type = res.location_id.type

	@api.multi
	def action_create_dc(self):

		dc_id = self.create_dc()
		self.write({'warehouse_dc_id':dc_id.id})
		view_id = self.env.ref('multi_warehouse_stock_transfer.warehouse_dc_form')

		if dc_id:
			action =  {
				'name':'Warehouse DC',
				'view_type':'form',
				'view_mode':'form',
				'res_model':'dc.warehouse',
				'view_id':view_id.id,
				'type':'ir.actions.act_window',
				'res_id': dc_id.id,
				'domain':[('id','=',dc_id.id)],
				'target':'current',
			}
			return action

	def create_dc(self):

		dc_obj = self.env["dc.warehouse"]
		dc_line_obj = self.env["dc.warehouse.line"]
		if not self.warehouse_dc_id:
			dc_val = {
				'request_warehouse_from_id':self.issue_id.request_warehouse_from_id.id,
				'partner_id':self.issue_id.partner_id.id,
				'warehouse_id':self.issue_id.warehouse_id.id,
				'company_id':self.company_id.id,
				'request_date':self.issue_id.request_date,
				'issue_refernce':self.issue_id.id,
				'reference':self.issue_id.reference.id,
				'currency_id':self.issue_id.currency_id.id,
			}

			dc_id = dc_obj.create(dc_val)

			for move in self.move_lines:
				if move:
					for quant in move.quant_ids:
						if quant.qty > 0:
							dc_line_val = {
								'product_id':quant.product_id.id,
								'issued_quant':quant.qty,
								'batch':quant.lot_id.id,
								'ref_id':dc_id.id,
							}

							ctx = {}
							ctx.update({'company_id':self.company_id})
							dc_line = dc_line_obj.with_context(ctx).new(dc_line_val)
							dc_line.onchange_product_id()
							val = dc_line._convert_to_write({line:dc_line[line] for line in dc_line._cache})
							dc_line_id = dc_line_obj.create(val)

			return dc_id

class StockMove(models.Model):

	_inherit = "stock.move"

	issue_line_id = fields.Many2one("warehouse.stock.issue.line", string="Issue line")

	def action_done(self, cr, uid, ids, context=None):
		res = super(StockMove, self).action_done(cr, uid, ids, context=context)
		for move in self.browse(cr, uid, ids, context=context):
			if move.issue_line_id:
				quant = move.product_uom_qty
				move.issue_line_id.issued_quant += quant
				move.issue_line_id.state = 'issue'
				move.issue_line_id.transfer_id.state = 'issue'


from zope.site.hooks import getSite

from zope import interface

from domsense.homeboxes.interfaces import IHomeBoxContainer
from domsense.homeboxes.config import CONTAINER_DEFAULT_ID
from domsense.homeboxes.config import CONTAINER_DEFAULT_TITLE

def setupVarious(context):

    if context.readDataFile('domsense.homeboxes.import_various.txt') is None:
        return

    site = getSite()
    if not hasattr(site,CONTAINER_DEFAULT_ID):
        site.invokeFactory('Folder',
                           CONTAINER_DEFAULT_ID,
                           title=CONTAINER_DEFAULT_TITLE)
        folder = getattr(site,CONTAINER_DEFAULT_ID)
        folder.markCreationFlag()
        interface.alsoProvides(folder,IHomeBoxContainer)

        # omit from nav
        folder.getField('excludeFromNav').set(folder,True)
        folder.reindexObject()

        # set constrain on type
        if not folder.getConstrainTypesMode():
            folder.setConstrainTypesMode(1)
        # flush allowed types
        folder.setLocallyAllowedTypes(())
        # set proper type
        folder.setLocallyAllowedTypes(['HomeBox',])


import csv
import os
import subprocess
import sys
from tqdm import tqdm

def main():
	queries_dir = os.path.join(os.getcwd(), 'pquery_fastas');
	query_files = os.listdir(queries_dir)
	query_paths = [os.path.join('pquery_fastas', qfile) for qfile in query_files]

	out_paths = []
	for i in tqdm(range(len(query_paths))):
		with open(query_paths[i], 'r') as qfile:
			first_line = qfile.readline().split()
			query_id = 'pse_hmmer_' + first_line[0][1:]
		cur_out_path = os.path.join('pseudo_hmmer_csv', query_id)
		out_paths.append(cur_out_path)
	# print(query_paths[:5])
	# print(out_paths[:5])
	for i in tqdm(range(len(query_paths))):
		# print(query_paths[i])
		subprocess.call(['./phmmer', '-o', '/dev/null', '--tblout', out_paths[i], query_paths[i], "../pseq_db.fa"])

if __name__ == "__main__":
	main()


import os
import json
from unittest import mock
from requests import Response
from packageship import BASE_PATH

def compare_two_values(obj1, obj2):
    """

    Args:
        obj1:object1 It can be a data type in Python,
                and can be converted by using the str() method
        obj2:object2 same as obj1

    Returns: True or False

    """
    # With the help of the str() method provided by python,It's so powerful
    return obj1 == obj2 or (isinstance(obj1, type(obj2)) and
                            "".join(sorted(str(obj1))) == "".join(sorted(str(obj2))))

def get_correct_json_by_filename(filename):
    """

    Args:
        filename: Correct JSON file name without suffix

    Returns: list this json file's content

    """
    json_path = os.path.join(os.path.dirname(BASE_PATH),
                             "test",
                             "common_files",
                             "correct_test_result_json",
                             "{}.json".format(filename))
    try:
        with open(json_path, "r", encoding='utf-8') as json_fp:
            correct_list = json.loads(json_fp.read())
    except FileNotFoundError:
        return []

    return correct_list


import sys
from functools import partial
from multiprocessing import Process, current_process

from pulsar import system, HaltServer, MonitorStarted
from pulsar.utils.security import gen_unique_id
from pulsar.utils.pep import itervalues

from .proxy import ActorProxyMonitor, get_proxy
from .access import get_actor, set_actor, logger, _StopError, SELECTORS
from .threads import Thread
from .mailbox import MailboxClient, MailboxProtocol, ProxyMailbox
from .futures import multi_async, Future, add_errback
from .eventloop import EventLoop
from .protocols import TcpServer
from .consts import *

if sys.platform == 'win32':     # pragma    nocover
    signal = None
else:
    import signal

__all__ = ['Concurrency', 'concurrency']

def concurrency(kind, actor_class, monitor, cfg, **params):
    '''Function invoked by the :class:`.Arbiter` or a :class:`.Monitor` when
spawning a new :class:`.Actor`. It created a :class:`.Concurrency` instance
which handle the initialisation and the life of an :class:`.Actor`.

:parameter kind: Type of concurrency.
:parameter monitor: The monitor (or arbiter) managing the :class:`.Actor`.
:return: a :class:`.Councurrency` instance.
'''
    maker = concurrency_models.get(kind)
    if maker:
        c = maker()
        return c.make(kind, actor_class, monitor, cfg, **params)
    else:
        raise ValueError('Concurrency %s not supported in pulsar' % kind)

class Concurrency(object):
    '''Actor :class:`.Concurrency`.

    Responsible for the actual spawning of actors according to a
    concurrency implementation. Instances are picklable
    and are shared between the :class:`.Actor` and its
    :class:`.ActorProxyMonitor`.
    This is an abstract class, derived classes must implement the
    ``start`` method.

    :param concurrency: string indicating the concurrency implementation.
        Valid choices are ``monitor``, ``process``, ``thread``.
    :param actor_class: :class:`.Actor` or one of its subclasses.
    :param timeout: timeout in seconds for the actor.
    :param kwargs: additional key-valued arguments to be passed to the actor
        constructor.
    '''
    _creation_counter = 0

    def make(self, kind, actor_class, monitor, cfg, name=None, aid=None, **kw):
        self.__class__._creation_counter += 1
        self.aid = aid or gen_unique_id()[:8]
        self.age = self.__class__._creation_counter
        self.name = name or actor_class.__name__.lower()
        self.kind = kind
        self.cfg = cfg
        self.actor_class = actor_class
        self.params = kw
        self.params['monitor'] = monitor
        return self.get_actor()

    @property
    def unique_name(self):
        return '%s(%s)' % (self.name, self.aid)

    def __repr__(self):
        return self.unique_name
    __str__ = __repr__

    def before_start(self, actor):
        pass

    def is_process(self):
        return False

    def is_arbiter(self):
        return False

    def selector(self):
        '''Return a selector instance.

        By default it return nothing so that the best handler for the
        system is chosen.
        '''
        return SELECTORS[self.cfg.selector]()

    def run_actor(self, actor):
        '''Start running the ``actor``.
        '''
        set_actor(actor)
        actor.mailbox.start_serving()
        actor._loop.run_forever()

    def setup_event_loop(self, actor):
        '''Set up the event loop for ``actor``.
        '''
        actor._logger = self.cfg.configured_logger(actor.name)
        loop = EventLoop(self.selector(), logger=actor._logger,
                         iothreadloop=True)
        actor.mailbox = self.create_mailbox(actor, loop)

    def hand_shake(self, actor):
        '''Perform the hand shake for ``actor``

        The hand shake occurs when the ``actor`` is in starting state.
        It performs the following actions:

        * set the ``actor`` as the actor of the current thread
        * bind two additional callbacks to the ``start`` event
        * fire the ``start`` event

        If the hand shake is successful, the actor will eventually
        results in a running state.
        '''
        try:
            assert actor.state == ACTOR_STATES.STARTING
            if actor.cfg.debug:
                actor.logger.debug('starting handshake')
            a = get_actor()
            if a is not actor and a is not actor.monitor:
                set_actor(actor)
            actor.bind_event('start', self._switch_to_run)
            actor.bind_event('start', self.periodic_task)
            actor.bind_event('start', self._acknowledge_start)
            actor.fire_event('start')
        except Exception as exc:
            actor.stop(exc)

    def get_actor(self):
        self.daemon = False
        self.params['monitor'] = get_proxy(self.params['monitor'])
        # make sure these parameters are picklable
        #pickle.dumps(self.params)
        return ActorProxyMonitor(self)

    def create_mailbox(self, actor, loop):
        '''Create the mailbox for ``actor``.'''
        client = MailboxClient(actor.monitor.address, actor, loop)
        loop.call_soon_threadsafe(self.hand_shake, actor)
        client.bind_event('finish', lambda _, **kw: loop.stop())
        return client

    def _install_signals(self, actor):
        proc_name = "%s-%s" % (actor.cfg.proc_name, actor)
        if system.set_proctitle(proc_name):
            actor.logger.debug('Set process title to %s', proc_name)
        system.set_owner_process(actor.cfg.uid, actor.cfg.gid)
        if signal:
            actor.logger.debug('Installing signals')
            for sig in system.EXIT_SIGNALS:
                try:
                    actor._loop.add_signal_handler(
                        sig, self.handle_exit_signal, actor, sig)
                except ValueError:
                    pass

    def periodic_task(self, actor, **kw):
        '''Implement the :ref:`actor period task <actor-periodic-task>`.

        This is an internal method called periodically by the
        :attr:`.Actor._loop` to ping the actor monitor.
        If successful return a :class:`asyncio.Future` called
        back with the acknowledgement from the monitor.
        '''
        actor.next_periodic_task = None
        ack = None
        if actor.is_running():
            if actor.cfg.debug:
                actor.logger.debug('notify monitor')
            # if an error occurs, shut down the actor
            ack = actor.send('monitor', 'notify', actor.info())
            add_errback(ack, actor.stop)
            next = max(ACTOR_TIMEOUT_TOLE*actor.cfg.timeout, MIN_NOTIFY)
        else:
            next = 0
        actor.next_periodic_task = actor._loop.call_later(
            min(next, MAX_NOTIFY), self.periodic_task, actor)
        return ack

    def stop(self, actor, exc):
        '''Gracefully stop the ``actor``.
        '''
        if actor.state <= ACTOR_STATES.RUN:
            # The actor has not started the stopping process. Starts it now.
            actor.state = ACTOR_STATES.STOPPING
            actor.event('start').clear()
            if exc:
                actor.exit_code = getattr(exc, 'exit_code', 1)
                if actor.exit_code == 1:
                    actor.logger.critical('Stopping', exc_info=True)
                elif actor.exit_code:
                    actor.stream.writeln(str(exc))
                else:
                    actor.logger.info('Stopping')
            else:
                if actor.logger:
                    actor.logger.debug('stopping')
                actor.exit_code = 0
            stopping = actor.fire_event('stopping')
            actor.close_executor()
            if not stopping.done() and actor._loop.is_running():
                actor.logger.debug('async stopping')
                stopping.add_done_callback(lambda _: self._stop_actor(actor))
            else:
                self._stop_actor(actor)
        elif actor.stopped():
            # The actor has finished the stopping process.
            actor.fire_event('stop')
        return actor.event('stop')

    def _stop_actor(self, actor):
        '''Exit from the :class:`.Actor` domain.'''
        actor.state = ACTOR_STATES.CLOSE
        if actor._loop.is_running():
            actor.logger.debug('Closing mailbox')
            actor.mailbox.close()
        else:
            actor.exit_code = 1
            actor.mailbox.abort()
            self.stop(actor, 0)

    def _switch_to_run(self, actor, exc=None):
        if exc is None and actor.state < ACTOR_STATES.RUN:
            actor.state = ACTOR_STATES.RUN
        elif exc:
            actor.stop(exc)

    def _acknowledge_start(self, actor, exc=None):
        if exc is None:
            actor.logger.info('started')
        else:
            actor.stop(exc)

class ProcessMixin(object):

    def is_process(self):
        return True

    def before_start(self, actor):  # pragma    nocover
        actor.start_coverage()
        self._install_signals(actor)

    def handle_exit_signal(self, actor, sig):
        actor.logger.warning("Got %s. Stopping.", system.SIG_NAMES.get(sig))
        actor._loop.exit_signal = sig
        raise _StopError

class MonitorMixin(object):

    def get_actor(self):
        return self.actor_class(self)

    def start(self):
        '''does nothing'''
        pass

    def is_active(self):
        return self.actor.is_alive()

    @property
    def pid(self):
        return current_process().pid

############################################################################
##    CONCURRENCY IMPLEMENTATIONS
class MonitorConcurrency(MonitorMixin, Concurrency):
    ''':class:`.Concurrency` class for a :class:`.Monitor`.

    Monitors live in the **main thread** of the master process and
    therefore do not require to be spawned.
    '''
    def setup_event_loop(self, actor):
        actor._logger = self.cfg.configured_logger(actor.name)
        actor.mailbox = ProxyMailbox(actor)
        actor.mailbox._loop.call_soon(actor.start)

    def run_actor(self, actor):
        actor._loop.call_soon(self.hand_shake, actor)
        raise MonitorStarted

    def create_mailbox(self, actor, loop):
        raise NotImplementedError

    def periodic_task(self, actor, **kw):
        '''Override the :meth:`.Concurrency.periodic_task` to implement
        the :class:`.Monitor` :ref:`periodic task <actor-periodic-task>`.'''
        interval = 0
        actor.next_periodic_task = None
        if actor.is_running():
            interval = MONITOR_TASK_PERIOD
            actor.manage_actors()
            actor.spawn_actors()
            actor.stop_actors()
            actor.monitor_task()
        actor.next_periodic_task = actor._loop.call_later(
            interval, self.periodic_task, actor)

    def _stop_actor(self, actor):

        def _cleanup(_):
            if not actor.terminated_actors:
                actor.monitor._remove_actor(actor)
            actor.fire_event('stop')

        return actor.close_actors().add_done_callback(_cleanup)

class ArbiterConcurrency(MonitorMixin, ProcessMixin, Concurrency):
    '''Concurrency implementation for the :class:`.Arbiter`
    '''
    def is_arbiter(self):
        return True

    def before_start(self, actor):  # pragma    nocover
        '''Daemonise the system if required.
        '''
        if actor.cfg.daemon:
            system.daemonize()
        actor.start_coverage()
        self._install_signals(actor)

    def create_mailbox(self, actor, loop):
        '''Override :meth:`.Concurrency.create_mailbox` to create the
        mailbox server.
        '''
        mailbox = TcpServer(MailboxProtocol, loop, ('127.0.0.1', 0),
                            name='mailbox')
        # when the mailbox stop, close the event loop too
        mailbox.bind_event('stop', lambda _, **kw: loop.stop())
        mailbox.bind_event(
            'start',
            lambda _, **kw: loop.call_soon(self.hand_shake, actor))
        return mailbox

    def periodic_task(self, actor, **kw):
        '''Override the :meth:`.Concurrency.periodic_task` to implement
        the :class:`.Arbiter` :ref:`periodic task <actor-periodic-task>`.'''
        interval = 0
        actor.next_periodic_task = None
        if actor.is_running():
            # managed actors job
            interval = MONITOR_TASK_PERIOD
            actor.manage_actors()
            for m in list(itervalues(actor.monitors)):
                if m.started() and not m.is_running():
                    actor._remove_actor(m)
        actor.next_periodic_task = actor._loop.call_later(
            interval, self.periodic_task, actor)

    def _stop_actor(self, actor):
        '''Stop the pools the message queue and remaining actors.'''
        if actor._loop.is_running():
            self._exit_arbiter(actor)
        else:
            actor.logger.debug('Restarts event loop to stop actors')
            loop = actor._loop
            actor._loop.call_soon(self._exit_arbiter, actor)
            actor._run(False)

    def _exit_arbiter(self, actor, fut=None):
        if fut:
            actor.logger.debug('Closing mailbox server')
            actor.state = ACTOR_STATES.CLOSE
            actor.mailbox.close()
        else:
            actor.logger.debug('Close monitors and actors')
            active = multi_async((actor.close_monitors(),
                                  actor.close_actors()))
            active.add_done_callback(partial(self._exit_arbiter, actor))

def run_actor(self):
    self._actor = actor = self.actor_class(self)
    try:
        actor.start()
    finally:
        try:
            actor.cfg.when_exit(actor)
        except Exception:   # pragma    nocover
            pass
        log = actor.logger or logger()
        self.stop_coverage(actor)
        log.info('Bye from "%s"', actor)

class ActorProcess(ProcessMixin, Concurrency, Process):
    '''Actor on a Operative system process.

    Created using the python multiprocessing module.
    '''
    def run(self):  # pragma    nocover
        # The coverage for this process has not yet started
        run_actor(self)

    def stop_coverage(self, actor):
        actor.stop_coverage()

class TerminateActorThread(Exception):
    pass

class ActorThread(Concurrency, Thread):
    '''Actor on a thread in the master process.'''
    _actor = None

    def run(self):
        run_actor(self)

    def stop_coverage(self, actor):
        pass

    def loop(self):
        if self._actor:
            return self._actor._loop

concurrency_models = {'arbiter': ArbiterConcurrency,
                      'monitor': MonitorConcurrency,
                      'thread': ActorThread,
                      'process': ActorProcess}


import torch
import torch.nn as nn
import torch.optim as optim
import pandas as pd
import numpy as np
from torch.utils.data import TensorDataset, DataLoader
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import accuracy_score, f1_score, confusion_matrix
from numpy import vstack
from network import ContemptNet
from sklearn.model_selection import KFold
from sklearn import metrics
import matplotlib.pyplot as plt
from sklearn.preprocessing import MinMaxScaler
import seaborn as sn

def boolean_df(item_lists, unique_items):
# Create empty dict
    bool_dict = {}

    # Loop through all the tags
    for i, item in enumerate(unique_items):

        # Apply boolean mask
        bool_dict[item] = item_lists.apply(lambda x: 1 if item in x else 0)
    result = pd.DataFrame(bool_dict)
    # result['filename'] = filenames
    # Return the results as a dataframe
    return pd.DataFrame(bool_dict)

def to_1D(series):
    return pd.Series([x for _list in series for x in _list])

le = LabelEncoder()
def train_model(train_dl, model, criterion, optimizer):
    training_loss = []
    avg_loss = 0
    for i, data in enumerate(train_dl):
        # print()
        device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

        inputs, labels = data
        # print(type(labels))
        inputs = inputs.to(device)
        labels = labels.float().to(device)
        model = model.train()
        # print(model)
        # print("input shape: ", inputs.shape)
        # print(input)
        # forward + backward + optimize
        yhat = model(inputs)
        loss = criterion(yhat, labels)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        training_loss.append(loss.item())
    return np.mean(training_loss)

# make a class prediction for one row of data

def predict(row, model):
    # convert row to data
    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
    row = torch.tensor([row], dtype=torch.float32).to(device)
    model.eval()
    # make prediction
    yhat = model(row)
    # retrieve numpy array
    yhat = yhat.detach().cpu().numpy()
    return np.argmax(yhat)

def valid_model(valid_dl, model, criterion):
    valid_loss = []
    valid_acc = []
    predictions, actuals = list(), list()
    model.eval()
    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
    for i, data in enumerate(valid_dl):
        inputs, targets = data
        inputs = inputs.to(device)
        targets = targets.float().to(device)
        # retrieve numpy array
        yhat = model(inputs)
        # print("yhat shape: ", yhat.shape)
         # retrieve numpy array
        loss = criterion(yhat, targets)
        yhat = yhat.cpu().detach().numpy()
        actual = targets.cpu().numpy()
        # convert to class labels
        # yhat = np.argmax(yhat, axis=1)
        # reshape for stacking
        actual = actual.reshape((len(actual), len(label_cols)))
        # print("Actual: ", actual[0])
        yhat = yhat.reshape((len(yhat),  len(label_cols)))
        yhat = (yhat > 0.5)
        # print("predicted: ", yhat[0])

        # calculate accuracy
        predictions.append(yhat)
        actuals.append(actual)
        acc = metrics.jaccard_score(vstack(predictions), vstack(actuals), average='samples')
        # round to class values
        valid_loss.append(loss.item())
        valid_acc.append(acc)

    # f1_metric = f1_score(vstack(actuals), vstack(predictions), average = "macro")
    # print('F1 score:' , f1_metric)
    jac = metrics.jaccard_score(vstack(actuals), vstack(predictions), average = 'samples')
    print('Jaccard score:' , jac)
    return np.mean(valid_loss), np.mean(valid_acc)

def get_dataloaders(df, batch_size):

    Y = df[label_cols].values

    X = df.drop(metadata_cols + label_cols, axis=1)
    # Y = le.fit_transform(Y)
    # print(le.classes_)
    Y_tensor = torch.tensor(Y, dtype=torch.long)
    X_tensor = torch.tensor(X.values, dtype=torch.float32)

    dataset = TensorDataset(X_tensor, Y_tensor)

    dataloader = DataLoader(dataset, shuffle=True, batch_size=batch_size)
    return dataloader

batch_size = 32
epochs = 100

data_path = '../new_data/NA/na_dataset.csv'
X_df = pd.read_csv(data_path, index_col=None)

Y_df = pd.read_csv('../new_data/NA/na_labels.csv', usecols=['filename', 'emotions'], index_col='filename')
Y_df["emotions"] = Y_df["emotions"].apply(eval)
unique_items = to_1D(Y_df["emotions"]).unique()
labels_expanded = boolean_df(Y_df['emotions'], unique_items)
# labels_expanded.set_index('filename')
X_df['none']  = np.NaN
X_df['furious']  = np.NaN
X_df['anger']  = np.NaN
X_df['annoyed']  = np.NaN
X_df['contempt']  = np.NaN
X_df['disgust']  = np.NaN
X_df['hatred']  = np.NaN

for index, row in X_df.iterrows():
    # print(index, row)
    filename = X_df.iloc[index]['filename']
    # print(labels_expanded.loc[filename]['none':'hatred'].to_list())
    X_df.at[index,'none'] = labels_expanded.at[filename,'none']
    X_df.at[index,'furious'] = labels_expanded.at[filename,'furious']
    X_df.at[index,'anger'] = labels_expanded.at[filename,'anger']
    X_df.at[index,'annoyed'] = labels_expanded.at[filename,'annoyed']
    X_df.at[index,'contempt'] = labels_expanded.at[filename,'contempt']
    X_df.at[index,'disgust'] = labels_expanded.at[filename,'disgust']
    X_df.at[index,'hatred'] = labels_expanded.at[filename,'hatred']

cols_to_scale = list (
    set(X_df.columns.to_list()) - set(['frame', 'face_id', 'culture', 'filename', 'timestamp', 'confidence','success', 'none', 'furious', 'anger', 'annoyed', 'contempt', 'disgust', 'hatred'])
)
scaler = MinMaxScaler()
X_df[cols_to_scale] = scaler.fit_transform(X_df[cols_to_scale])
metadata_cols = ['frame', 'face_id', 'culture', 'filename', 'timestamp']
label_cols = ['none', 'furious', 'anger', 'annoyed', 'contempt', 'disgust', 'hatred']

test_videos = ['na/vid_6.mp4', 'na/vid_19.mp4', 'na/vid_43.mp4', 'na/vid_25.mp4', 'na/vid_23.mp4', 'na/vid_10_1.mp4', 'na/vid_72.mp4', 'na/vid_34.mp4', 'na/vid_90.mp4', 'na/vid_92.mp4', 'na/vid_39.mp4', 'na/vid_30.mp4', 'na/vid_3.mp4', 'na/vid_33.mp4', 'na/vid_4.mp4', 'na/vid_31.mp4', 'na/vid_53.mp4', 'na/vid_52.mp4', 'na/vid_55.mp4', 'na/vid_59.mp4', 'na/vid_22.mp4', 'na/vid_11.mp4', 'na/vid_79.mp4', 'na/vid_54.mp4', 'na/vid_87.mp4', 'na/vid_63.mp4', 'na/vid_12.mp4', 'na/vid_10_2.mp4', 'na/vid_97.mp4', 'na/vid_70.mp4', 'na/vid_42.mp4', 'na/vid_49.mp4', 'na/vid_77.mp4']
# df = df[df['gender'] == 'male']
kfold = KFold(5, True, 1)
# target_culture = 'Persian'
# df = df[(df['culture'] == target_culture)]
# SPLITTING HELD-OUT DATA
videos = X_df['filename'].unique()

# videos must be array to be subscriptable by a list
videos = np.array(list(set(videos) - set(test_videos)))
# Removing test videos from train dataset
test_df = X_df[X_df['filename'].isin(test_videos)]
y_test = test_df[label_cols].values
X_test = test_df.drop(columns = ['frame', 'face_id', 'culture', 'filename', 'timestamp', 'confidence','success']).values
X_df = X_df[~X_df['filename'].isin(list(test_videos))]
# df = df[df['culture'] != 'Persian']
splits = kfold.split(videos)
kfold_valid_acc = []
kfold_test_acc = []

# TRAINING
test_df_copy = test_df.drop(['frame', 'face_id', 'culture', 'filename', 'confidence','success'], axis=1)
for (i, (train, test)) in enumerate(splits):
    print('%d-th split: train: %d, test: %d' % (i+1, len(videos[train]), len(videos[test])))
    train_df = X_df[X_df['filename'].isin(videos[train])]
    valid_df = X_df[X_df['filename'].isin(videos[test])]

    train_dataloader = get_dataloaders(train_df, batch_size)
    valid_dataloader = get_dataloaders(valid_df, batch_size)
    net = ContemptNet()
    if torch.cuda.is_available():
        net.cuda()
    # optimizer = optim.Adam(net.parameters(), lr=0.005, weight_decay=1e-5)
    optimizer = optim.ASGD(net.parameters(), lr=0.005)
    criterion = nn.BCEWithLogitsLoss()
    train_losses = []
    valid_losses = []
    valid_acc = []
    for epoch in range(epochs):
        print('Epoch {}/{}'.format(epoch, epochs - 1))
        print('-' * 10)
        epoch_train_loss = train_model(train_dataloader, net, criterion, optimizer)
        epoch_valid_loss, epoch_valid_acc = valid_model(valid_dataloader, net, criterion)
        train_losses.append(epoch_train_loss)
        valid_losses.append(epoch_valid_loss)
        print('[%d] Training loss: %.3f' %
                    (epoch + 1, epoch_train_loss ))
        print('[%d] Validation loss: %.3f' %
                    (epoch + 1, epoch_valid_loss ))
        print('[%d] Validation accuracy: %.3f' %
                    (epoch + 1, epoch_valid_acc ))
        valid_acc.append(epoch_valid_acc)
    kfold_valid_acc.append(valid_acc[-1])
    ## Plot the curves
    plt.plot(train_losses, label='training loss')
    plt.plot(valid_losses, label='validation loss')

    plt.legend(bbox_to_anchor=(1.05, 1.0), loc='upper left')
    plt.show()

    ## Testing
    Yhat = list()
    for row in test_df_copy.values:
        p = predict(row, net)
        Yhat.append(p)

    # test_df['integer_emotion'] = Y
    # test_df['predicted'] = le.inverse_transform(Yhat)
    # print(test_df.sample(n=20))
    # test_acc = accuracy_score(le.fit_transform(test_df['emotion'].values), Yhat)
    test_acc = metrics.jaccard_score(test_df[label_cols].values, Yhat, average='samples')
    # cf_matrix = confusion_matrix(le.fit_transform(test_df['emotion'].values), Yhat)
    # print('********Confusion Matrix*********\n', cf_matrix)
    # df_cm = pd.DataFrame(cf_matrix/np.sum(cf_matrix), index=le.inverse_transform([0,1,2]), columns=le.inverse_transform([0,1,2]))
    # plt.figure(figsize=(10,7))
    # sn.heatmap(df_cm, annot=True, fmt='.2%')
    # plt.ylabel('True label')
    # plt.xlabel('Predicted label')
    # plt.show()
    # misclassified_test_df = test_df[test_df.predicted != test_df.emotion]
    # test_f1 = f1_score(le.fit_transform(test_df['emotion'].values), Yhat, average='macro')
    kfold_test_acc.append(test_acc)
    print('Test accuracy: %.3f' % (test_acc))
    # print('Test F1-Score: %.3f' % (test_f1))

test_df.to_csv('test_result.csv')
print('Average Test Accuracy on 5-Fold CV: %.3f' % (np.mean(kfold_test_acc)))
print('Average Validation Accuracy on 5-Fold CV: %.3f' % (np.mean(kfold_valid_acc)))


"""
    9.4 Write a program to read through the mbox-short.txt
        and figure out who has sent the greatest number of mail messages.
        The program looks for 'From ' lines and takes the second word of those lines as the person who sent the mail.
        The program creates a Python dictionary that maps the sender's mail address
        to a count of the number of times they appear in the file. After the dictionary is produced,
        the program reads through the dictionary using a maximum loop to find the most prolific committer.
"""

inp = input("Enter file: ")
my_file = open(inp)
my_dict = dict()                # create empty dictionary using dict constructor

for line in my_file:
    line = line.rstrip()
    if not line.startswith("From "):
        continue
    words = line.split()
    for word in words:
        word = words[1]
        if word not in my_dict:
            my_dict[word] = 1
        else:
            my_dict[word] = my_dict[word] + 1

        # or use the get method i.e my_dict[word] = my_dict.get(word, 0) + 1

print(my_dict)

my_email = None
email_count = None
for key, val in list(my_dict.items()):             # my_dict.items() creates a list of tuples with both key and value
    if email_count == None or val > email_count:   # checks for most prolific committer
        my_email = key
        email_count = val

print(my_email, email_count)                       # print the email and count of the most prolific committer

PYTHON 4 EVERYONE/Python Data Structures/WEEK 5/assignment_9.4.py


from multiprocessing import Pool, cpu_count
import logging
from time import time

logger = logging.getLogger()
stream_handler = logging.StreamHandler()
logger.addHandler(stream_handler)
logger.setLevel(logging.DEBUG)

def factorize(*number):

    final_list = []
    for num in number:
        final_list.append(list(filter(lambda x: num % x == 0, range(1, num+1))))

    logger.debug(f"Result - {final_list}")
    return final_list

if __name__ == '__main__':
    start = time()
    with Pool(cpu_count()) as pool:
        result = pool.apply_async(factorize, (128, 255, 99999, 10651060))
        a,b,c,d = result.get()
    # a,b,c,d = factorize(128, 255, 99999, 10651060)
    end = time()
    print(f"Result - {end-start} seconds")


import copy
from threading import local

from numpy import empty, empty_like
input_files = ["input.txt", "test_input.txt"]

file = 0
with open(input_files[file], 'r') as f:
    start_grid =  [list(a.strip()) for a in f.readlines()]

def get_adj(grid, pos):
    neighbors = [(1,0), (0,1), (-1,0), (0,-1), (1,1), (-1,1), (1,-1), (-1,-1)]
    neighbors = [x for x in neighbors if 0<=x[0]+pos[0]<len(grid) and 0<=x[1]+pos[1]<len(grid[0])]
    neighbors = [(pos[0]+x[0], pos[1]+x[1]) for x in neighbors]

    return sum([grid[x[0]][x[1]] == "#" for x in neighbors])

def get_los(grid, pos):
    directions = [(1,0), (0,1), (-1,0), (0,-1), (1,1), (-1,1), (1,-1), (-1,-1)]
    count = 0
    for direction in directions:
        completed = False
        cur = pos
        while 0<=cur[0]+direction[0]<len(grid) and 0<=cur[1]+direction[1]<len(grid[0]) and not completed:
            cur = (cur[0] + direction[0], cur[1] + direction[1])
            if grid[cur[0]][cur[1]] != ".":
                completed = True
                if grid[cur[0]][cur[1]] == "#": count += 1

    return count

def solve(mode):
    updated = True
    grid = copy.deepcopy(start_grid)
    seating = {"los":get_los, "adj":get_adj}
    empty_limit = {"los":5, "adj":4}
    while updated:
        updated = False
        local_grid = copy.deepcopy(grid)

        for i in range(len(grid)):
            for j in range(len(grid[i])):
                if grid[i][j] == "L" and seating[mode](grid, (i, j)) == 0:
                    local_grid[i][j] = "#"
                    updated = True
                if grid[i][j] == "#" and seating[mode](grid, (i, j)) >= empty_limit[mode]:
                    local_grid[i][j] = "L"
                    updated = True
        grid = local_grid

    print(sum([x.count("#") for x in grid]))

solve("adj")
solve("los")


from django.conf.urls import url, include
from django.contrib import admin
from loginRegistration import views

urlpatterns = [
    url('admin/', admin.site.urls),

    url(r'^login/$', views.Login, name='Login'),
    url(r'^registration/$', views.Register, name='Register'),
    url(r'^logout/$', views.Logout, name='Logout'),

    url('main/', include("main.urls"), name='main'),
    url('board/', include("board.urls"), name='board'),
]


import string
import re
import html
import urllib.request
import json
import os
import pickle

from Sastrawi.Stemmer.StemmerFactory import StemmerFactory

def clean_text_classification(text):
    text = text.lower()

    text = re.sub(r'@[A-Za-z0-9]+', ' ', text)
    text = re.sub(r'#[A-Za-z0-9]+', ' ', text)
    text = re.sub(r'\w+:\/\/\S+', ' ', text)
    text = re.sub(r'[^a-zA-Z0-9]', ' ', text)

    text = re.sub(r' +', ' ', text)

    return ' '.join(text.split())

def clean_text_pos(text):
    text = html.unescape(text)
    text = re.sub(r'#[A-Za-z0-9]+', ' ', text)
    text = re.sub(r'\w+:\/\/\S+', ' ', text)

    text = re.sub(r' +', ' ', text)
    text = os.linesep.join([s for s in text.splitlines() if s])
    text = ', '.join(text.split('\n'))

    return text

def replace_slang(text):
    with open("extra/akronim.json", "r") as f:
        acronym = json.load(f)

    res = []

    for w in text.split(' '):
        if w in acronym:
            res.append(acronym[w])
        else:
            res.append(w)

    return ' '.join(res)

def stemming(text):
    factory = StemmerFactory()
    stemmer = factory.create_stemmer()

    stem_text = stemmer.stem(text)

    return stem_text

def clean_proc_pos(text):
    text = clean_text_pos(text)
    text = replace_slang(text)

    return text

def clean_proc_clf(text):
    text = clean_text_classification(text)
    text = replace_slang(text)

    text = stemming(text)

    return text

if __name__ == "__main__":
    text = 'Bakor Pemandu ITS mengundang Pemandu Aktif ITS untuk duduk bareng ngobrolin LKMM TD pada:\n\n📆 Selasa - Rabu, 20-21 Feruari\n2018\n🕛 18.00 - 21.30 WIB\n📍 SCC Lt. 3\n👔 Standar Kuliah\n\n"Raise your standards to create change!" - An Iota of Truth\n\n#OborBakor\n#BAKORITS\n#ITSSurabaya https://t.co/v6oFZcWLFv'

    clean_text = clean_proc_pos(text)
    print(clean_text)

    clean_text = clean_proc_clf(text)
    print(clean_text)


"""
Receiving a multi-channel time series from LSL
"""

from pylsl import StreamInlet, resolve_stream
from icecream import ic
# from Util import util

def main():
    # First resolve an eeg stream on the lab network
    ic("looking for a EEG Stream")

    # Discovering an available streams on the LSL network that matches a specific set of criteria.
    eeg_streams = resolve_stream('name', 'Unicorn')

    stream_names = []
    print("Stream Information: ")
    for stream in eeg_streams:
        stream_names.append(stream.name())
        print(stream.name())
        # print(util.obtain_stream_channel_names(stream))

    # Creating an inlet for the stream
    eeg_inlet = StreamInlet(eeg_streams[0])

    while True:
        # Getting a new sample with a time stamp
        sample, timestamp = eeg_inlet.pull_sample() # It doesn't put back the element, it removes the elements once it is called.
        ic(timestamp, sample)

if __name__ == '__main__':
    main()


import json
import typing
from datetime import datetime

from discord.ext.commands import Greedy
from tortoise.exceptions import OperationalError

from utils.Database import ReactWatch, WatchedEmoji, Guild, BugReportingChannel

import discord
from discord import NotFound, HTTPException, Forbidden, TextChannel
from discord.ext import commands, tasks

from cogs.BaseCog import BaseCog
from utils import Utils, Configuration, Lang, Logging

class ReactMonitor(BaseCog):

    def __init__(self, bot):
        super().__init__(bot)
        self.excluded_channels_key = "react_watch_excluded_channels"
        self.rbu_interrupt_key = "react_watch_interrupt"
        self.rbu_running_key = "react_watch_running"
        self.react_watch_servers = set()
        self.min_react_lifespan = dict()
        self.recent_reactions = dict()
        self.react_removers = dict()
        self.mute_duration = dict()
        self.react_adds = dict()
        self.guilds = dict()
        self.emoji = dict()
        self.mutes = dict()
        self.started = False

    async def on_ready(self):
        for guild in self.bot.guilds:
            await self.init_guild(guild.id)
        if not self.check_reacts.is_running():
            self.check_reacts.start()
        self.started = True

    async def init_guild(self, guild_id):
        watch, created = await ReactWatch.get_or_create(serverid=guild_id)
        self.mutes[guild_id] = Configuration.get_persistent_var(f"react_mutes_{guild_id}", dict())
        self.min_react_lifespan[guild_id] = Configuration.get_persistent_var(f"min_react_lifespan_{guild_id}", 0.5)
        self.mute_duration[guild_id] = watch.muteduration

        # track react add/remove per guild
        self.recent_reactions[guild_id] = dict()
        self.react_removers[guild_id] = dict()
        self.react_adds[guild_id] = dict()

        # list of emoji to watch
        self.emoji[guild_id] = dict()
        for e in await watch.emoji:
            self.emoji[guild_id][e.emoji] = e

        # enable listening if set in db
        if watch.watchremoves:
            await self.activate_react_watch(guild_id)

        self.guilds[guild_id], created = await Guild.get_or_create(serverid=guild_id)

    def cog_unload(self):
        self.check_reacts.cancel()

    @commands.Cog.listener()
    async def on_guild_join(self, guild):
        await self.init_guild(guild.id)

    @commands.Cog.listener()
    async def on_guild_remove(self, guild):
        Configuration.del_persistent_var(f"min_react_lifespan_{guild.id}", True)
        Configuration.del_persistent_var(f"react_mutes_{guild.id}", True)
        del self.mutes[guild.id]
        del self.mute_duration[guild.id]
        del self.min_react_lifespan[guild.id]
        del self.recent_reactions[guild.id]
        del self.react_removers[guild.id]
        del self.react_adds[guild.id]
        del self.emoji[guild.id]
        del self.guilds[guild.id]
        if guild.id in self.react_watch_servers:
            await self.deactivate_react_watch(guild.id)
        watch = await ReactWatch.get(serverid=guild.id)
        for e in await watch.emoji:
            await e.delete()
        await watch.delete()

    @commands.Cog.listener()
    async def on_raw_reaction_add(self, event):
        await self.store_reaction_action(event)

    @commands.Cog.listener()
    async def on_raw_reaction_remove(self, event):
        await self.store_reaction_action(event)

    @commands.Cog.listener()
    async def on_member_join(self, member):
        if member.guild.id not in self.mutes:
            return

        if str(member.id) in self.mutes[member.guild.id]:
            guild = member.guild
            guild_config = await self.bot.get_guild_db_config(guild.id)
            if guild_config and guild_config.mutedrole:
                try:
                    mute_role = guild.get_role(guild_config.mutedrole)
                    await member.add_roles(mute_role)
                    log_msg = f"{Utils.get_member_log_name(member)} joined while still muted for banned reacts\n--- I **muted** them... **again**"
                    await self.bot.guild_log(guild.id, log_msg)
                except Exception as e:
                    await Utils.handle_exception("reactmon failed to mute member", self.bot, e)
            else:
                await self.bot.guild_log(guild.id, "**I can't re-mute for reacts because `!guildconfig` mute role is not set.")

    async def activate_react_watch(self, guild_id):
        # store setting in db, and add to list of listening servers
        watch = await ReactWatch.get(serverid=guild_id)
        watch.watchremoves = True
        await watch.save()
        self.react_watch_servers.add(guild_id)

    async def deactivate_react_watch(self, guild_id):
        # store setting in db, and remove from list of listening servers
        watch = await ReactWatch.get(serverid=guild_id)
        watch.watchremoves = False
        await watch.save()
        self.react_watch_servers.remove(guild_id)

    async def is_user_event_ignored(self, event):
        ignored_channels = [row.channelid for row in await BugReportingChannel.all()]
        is_ignored_channel = event.channel_id in ignored_channels
        guild = self.bot.get_guild(event.guild_id)
        if not guild:
            # Don't listen to DMs
            return True
        is_bot = event.user_id == self.bot.user.id
        member = guild.get_member(event.user_id)

        if member is None:
            return True  # ignore reaction events from departing members

        is_mod = member and member.guild_permissions.ban_members
        is_admin = await self.bot.member_is_admin(event.user_id)
        has_admin = False

        for role in member.roles:
            if role in Configuration.get_var("admin_roles", []):
                has_admin = True

        # ignore bot, ignore mod, ignore admin users and admin roles
        if is_bot or is_mod or is_admin or has_admin or is_ignored_channel:
            return True
        return False

    async def cog_check(self, ctx):
        return ctx.guild and (ctx.author.guild_permissions.ban_members or await self.bot.permission_manage_bot(ctx))

    @tasks.loop(seconds=1.0)
    async def check_reacts(self):
        now = datetime.now().timestamp()
        for guild_id in self.recent_reactions:
            try:
                for user_id, mute_time in dict(self.mutes[guild_id]).items():
                    if float(mute_time) + float(self.mute_duration[guild_id]) < now:
                        try:
                            guild = self.bot.get_guild(guild_id)
                            guild_config = await self.bot.get_guild_db_config(guild_id)
                            if guild_config and guild_config.mutedrole:
                                mute_role = guild.get_role(guild_config.mutedrole)
                                member = guild.get_member(int(user_id))
                                if mute_role in member.roles:
                                    await member.remove_roles(mute_role)
                                del self.mutes[guild_id][user_id]
                        except Exception as e:
                            log_channel = await self.bot.get_guild_log_channel(guild_id)
                            del self.mutes[guild_id][user_id]
                            await log_channel.send(f'Failed to unmute user ({user_id}) <@{user_id}>... did they leave the server?')
                            # await Utils.handle_exception('react watch unmute failure', self.bot, e)
                rr = self.recent_reactions[guild_id]
                adds = {t: e for (t, e) in rr.items() if e.event_type == "REACTION_ADD"}

                # creat list of adds
                for t, e in adds.items():
                    self.react_adds[guild_id][t] = e
                # cull out expired ones
                for t, e in dict(self.react_adds[guild_id]).items():
                    if t + self.min_react_lifespan[guild_id] < now:
                        # add reaction is too far in the past. remove from the list
                        del self.react_adds[guild_id][t]

                # loop over a copy of recent_reactions so we can remove items
                for timestamp, event in dict(self.recent_reactions[guild_id]).items():
                    # remove this one from the list
                    del self.recent_reactions[guild_id][timestamp]

                    p = getattr(self, 'process_'+event.event_type.lower())
                    await p(timestamp, event)

            except Exception as ex:
                await Utils.handle_exception('react watch loop error...', self.bot, ex)

    @commands.group(name="reactmonitor",
                    aliases=['reactmon', 'reactwatch', 'react', 'watcher'],
                    invoke_without_command=True)
    @commands.guild_only()
    @commands.bot_has_permissions(embed_links=True)
    async def react_monitor(self, ctx: commands.Context):
        """
        List the watched emoji and their settings
        """
        max_fields = 24
        watch = await ReactWatch.get(serverid=ctx.guild.id)
        embed = discord.Embed(
            timestamp=ctx.message.created_at,
            color=Utils.COLOR_LIME,
            title=Lang.get_locale_string("react_monitor/info_title", ctx, server_name=ctx.guild.name))

        embed.add_field(name="Monitor React Removal", value="Yes" if watch.watchremoves else "No")

        if watch.watchremoves:
            embed.add_field(name="Reaction minimum lifespan", value=f"{self.min_react_lifespan[ctx.guild.id]} seconds")

        embed.add_field(name="Mute duration", value=Utils.to_pretty_time(self.mute_duration[ctx.guild.id]))

        embed.add_field(name="__                                             __",
                        value="__                                             __",
                        inline=False)

        if ctx.guild.id in self.emoji:
            for key, emoji in self.emoji[ctx.guild.id].items():
                if len(embed.fields) == max_fields:
                    await ctx.send(embed=embed)
                    embed = discord.Embed(
                        color=Utils.COLOR_LIME,
                        title="...")
                embed.add_field(
                    name=f"{emoji.emoji}",
                    value=self.describe_emoji_watch_settings(emoji),
                    inline=True)

        await ctx.send(embed=embed)

    @staticmethod
    def describe_emoji_watch_settings(emoji):
        flags = [f"__*{name}*__" for name in ['log', 'remove', 'mute'] if getattr(emoji, name)]
        val = ' | '.join(flags) if flags else '__*no action*__'
        return val

    @react_monitor.command(aliases=["new", "edit", "update"])
    @commands.guild_only()
    async def add(self, ctx: commands.Context, emoji, log: bool = True, remove: bool = False, mute: bool = False ):
        """
        Add an emoji to the reaction watchlist

        emoji: The emoji to add
        log: Boolean - Log use of this emoji
        remove: Boolean - Auto-remove when this emoji is used
        mute: Boolean - Auto-mute members who use this emoji
        """
        try:
            watch, created = await ReactWatch.get_or_create(serverid=ctx.guild.id)
            new_emoji, created = await WatchedEmoji.get_or_create(watcher_id=watch.id, emoji=emoji)
            new_emoji.log = log
            new_emoji.remove = remove
            new_emoji.mute = mute
            await new_emoji.save()
            self.emoji[ctx.guild.id][emoji] = new_emoji
        except Exception as e:
            await Utils.handle_exception("failed to add emoji to watch list", self.bot, e)

        await ctx.send(f"`{emoji}` is now on the watch list with settings:\n"
                       f"{self.describe_emoji_watch_settings(self.emoji[ctx.guild.id][emoji])}")

    @react_monitor.group(name="clean", invoke_without_command=True)
    @commands.guild_only()
    async def clean(self, ctx):
        await ctx.send_help(ctx.command)

    @clean.command(aliases=["user", "byuser"])
    @commands.guild_only()
    async def clean_by_user(
            self,
            ctx: commands.Context,
            target: discord.User,
            check_channels: Greedy[discord.TextChannel] = True,
            count: int = 200):
        Configuration.set_persistent_var(self.rbu_interrupt_key, False)
        channels = ctx.guild.channels if check_channels is True else check_channels
        if check_channels is True:
            await ctx.send(f"Looking for reacts on the {count} most recent messages in all available channels")
        else:
            list_of_channels = "\n".join([c.mention for c in channels])
            await ctx.send(f"Looking for reacts on the {count} most recent messages "
                           f"in the following channels:\n{list_of_channels}")
        excluded_channels = Configuration.get_persistent_var(self.excluded_channels_key, [])
        for channel in channels:
            if isinstance(channel, TextChannel):
                if channel.id in excluded_channels:
                    await ctx.send(f"<#{channel.id}> skipped")
                    continue
                try:
                    i = 0
                    await ctx.send(f"checking <#{channel.id}>...")
                    async for message in channel.history(limit=count):
                        interrupt = Configuration.get_persistent_var(self.rbu_interrupt_key, False)
                        if interrupt:
                            Configuration.set_persistent_var(self.rbu_interrupt_key, False)
                            await ctx.send(f"__remove react by user__ operation halted.")
                            return
                        i += 1
                        for react in message.reactions:
                            async for user in react.users():
                                if user.id == target.id:
                                    await message.clear_reaction(react.emoji)
                                    await ctx.send(
                                        f"__{i}.__ react {react.emoji} by userid ({user.id}) "
                                        f"removed from message {message.jump_url}")
                                    continue
                except Forbidden:
                    await ctx.send(f"I can't access message history in channel <#{channel.id}>")
        await ctx.send("All done cleaning reacts")

    @clean.command(aliases=["stop", "stopclean", "stop_clean"])
    @commands.guild_only()
    async def stop_clean_by_user(self, ctx):
        Configuration.set_persistent_var(self.rbu_interrupt_key, True)
        await ctx.send(f"attempting to halt __remove react by user__ operation. If this doesn't work, KILL THE BOT!")

    @clean.command(aliases=["excludedchannels", "list_excluded", "listexcluded"])
    @commands.guild_only()
    async def excluded_channels(self, ctx):
        excluded_channels = Configuration.get_persistent_var(self.excluded_channels_key, [])
        excluded_channels = [self.bot.get_channel(c).mention for c in excluded_channels]
        if not excluded_channels:
            await ctx.send(f"No channels are excluded from react remove_by_user")
        else:
            channel_list = "\n".join(excluded_channels)
            await ctx.send(f"Channels excluded from react remove_by_user:\n{channel_list}")

    @clean.command(aliases=["exclude", "excludechannels", "exclude_channels"])
    @commands.guild_only()
    async def exclude_channel(self, ctx, channels: Greedy[discord.TextChannel]):
        """
        Exclude channel from checks for react spam
        :param ctx:
        :param channels:
        :return:
        """
        excluded_channels = Configuration.get_persistent_var(self.excluded_channels_key, [])
        excluded_channels = set(excluded_channels)
        added = []
        for channel in channels:
            if channel.id not in excluded_channels:
                excluded_channels.add(channel.id)
                added.append(channel.mention)
        Configuration.set_persistent_var(self.excluded_channels_key, list(excluded_channels))
        if not added:
            await ctx.send(f"No channels added to exclusion list")
        else:
            await ctx.send(f"Added {' '.join(added)} to react spam channel exclusion list")

    @clean.command(aliases=["unexclude", "unexcludechannels", "unexclude_channels"])
    @commands.guild_only()
    async def unexclude_channel(self, ctx, channels: Greedy[discord.TextChannel]):
        """
        Exclude channel from checks for react spam
        :param ctx:
        :param channels:
        :return:
        """
        excluded_channels = Configuration.get_persistent_var(self.excluded_channels_key, [])
        excluded_channels = set(excluded_channels)
        removed = []
        for channel in channels:
            if channel.id in excluded_channels:
                excluded_channels.remove(channel.id)
                removed.append(channel.mention)
        Configuration.set_persistent_var(self.excluded_channels_key, list(excluded_channels))
        if not removed:
            await ctx.send(f"No channels removed from exclusion list")
        else:
            await ctx.send(f"Removed {' '.join(removed)} from react spam channel exclusion list")

    @react_monitor.command(aliases=["rem", "del", "delete"])
    @commands.guild_only()
    async def remove(self, ctx: commands.Context, emoji):
        """
        Remove an emoji from the watch list

        emoji: The emoji to remove
        """
        try:
            watch_row = await WatchedEmoji.get(watcher__serverid=ctx.guild.id, emoji=emoji)
            await watch_row.delete()
            del self.emoji[ctx.guild.id][emoji]
            await ctx.send(f"I removed `{emoji}` from the watch list")
        except OperationalError:
            await ctx.send(f"I couldn't find `{emoji}` on the emoji watch list, so I didn't remove it.")
        except Exception as e:
            await Utils.handle_exception("react remove failed", self.bot, e)

    @react_monitor.command(aliases=['on'])
    @commands.guild_only()
    async def monitor_removal_on(self, ctx: commands.Context):
        """
        Turn ON monitor for spammy fast-removal of reactions
        """
        if ctx.guild.id in self.react_watch_servers:
            await ctx.send("React monitor is already on")
        else:
            await self.activate_react_watch(ctx.guild.id)
            await ctx.send("I'm on the lookout for reaction spam!")

    @react_monitor.command(aliases=['off'])
    @commands.guild_only()
    async def monitor_removal_off(self, ctx: commands.Context):
        """
        Turn OFF monitor for spammy fast-removal of reactions
        """
        if ctx.guild.id in self.react_watch_servers:
            await self.deactivate_react_watch(ctx.guild.id)
            await ctx.send("OK, I'll stop watching for reaction spams")
        else:
            await ctx.send("React monitor is already off")

    @react_monitor.command(aliases=["time", "reacttime"])
    @commands.guild_only()
    async def react_time(self, ctx: commands.Context, react_time: float):
        """
        Reacts removed before this duration will trigger react-watch

        react_time: time in seconds, floating point e.g. 0.25
        """
        self.min_react_lifespan[ctx.guild.id] = react_time
        Configuration.set_persistent_var(f"min_react_lifespan_{ctx.guild.id}", react_time)
        await ctx.send(f"Reactions that are removed before {react_time} seconds have passed will be flagged")

    @react_monitor.command(aliases=["list", "mutes"])
    @commands.guild_only()
    async def list_mutes(self, ctx: commands.Context):
        if self.mutes[ctx.guild.id]:
            react_muted = list()
            guild_config = await self.bot.get_guild_db_config(ctx.guild.id)
            mute_role = ctx.guild.get_role(guild_config.mutedrole)

            for member_id, timestamp in self.mutes[ctx.guild.id].items():
                member = ctx.guild.get_member(int(member_id))
                if member is not None:
                    long_name = Utils.get_member_log_name(member)
                    if mute_role not in member.roles:
                        # panic because role is not present when it should be
                        await ctx.send(f"{long_name} should be muted for banned reacts... but isn't. ***WHY NOT??***")
                    react_muted.append(long_name)

            names = "\n".join(react_muted)
            await ctx.send(f"__Members muted for banned reacts:__\n{names}")
        else:
            await ctx.send(f"Nobody is muted for banned reacts")

    @react_monitor.command(aliases=["purge", "purgemutes"])
    @commands.guild_only()
    async def purge_mutes(self, ctx: commands.Context):
        if self.mutes[ctx.guild.id]:
            react_unmuted = list()
            guild_config = await self.bot.get_guild_db_config(ctx.guild.id)
            mute_role = ctx.guild.get_role(guild_config.mutedrole)

            for member_id, timestamp in dict(self.mutes[ctx.guild.id]).items():
                member = ctx.guild.get_member(int(member_id))
                if member is not None:
                    await member.remove_roles(mute_role)
                    del self.mutes[ctx.guild.id][member_id]
                    long_name = Utils.get_member_log_name(member)
                    react_unmuted.append(long_name)

            names = "\n".join(react_unmuted)
            await ctx.send(f"__React mutes purged:__\n{names}")
        else:
            await ctx.send(f"Nobody is muted for banned reacts. Can't purge.")

    @react_monitor.command(aliases=["mutetime", "mute"])
    @commands.guild_only()
    async def mute_time(self, ctx: commands.Context, mute_time: float):
        """
        Set the duration for mutes given when mute-enabled reacts are used

        mute_time: time in seconds, floating point e.g. 0.25
        """
        self.mute_duration[ctx.guild.id] = mute_time
        watch, created = await ReactWatch.get_or_create(serverid=ctx.guild.id)
        watch.muteduration = mute_time
        await watch.save()
        t = Utils.to_pretty_time(mute_time)
        await ctx.send(f"Members will now be muted for {t} when they use restricted reacts")

    async def store_reaction_action(self, event):
        if not self.started or await self.is_user_event_ignored(event):
            return

        # Add event to dict for tracking fast removal of reactions
        now = datetime.now().timestamp()
        guild = self.bot.get_guild(event.guild_id)
        try:
            self.recent_reactions[guild.id][now] = event
        except KeyError as e:
            Logging.debug(f"React Monitory Key error: {json.dumps(e)}")

    async def process_reaction_add(self, timestamp, event):
        emoji_used = event.emoji
        member = event.member
        guild = self.bot.get_guild(event.guild_id)
        channel = self.bot.get_channel(event.channel_id)
        log_channel = await self.bot.get_guild_log_channel(event.guild_id)

        # Act on log, remove, and mute:
        e_db = None
        if str(emoji_used) in self.emoji[event.guild_id]:
            e_db = self.emoji[event.guild_id][str(emoji_used)]
            if not e_db.log and not e_db.remove and not e_db.mute:
                # No actions to take. Stop processing
                return

        if not e_db:
            return

        # check mute/warn list for reaction_add - log to channel
        # for reaction_add, remove if threshold for quick-remove is passed
        try:
            # message fetch is API call. Only do it if needed
            message = await channel.fetch_message(event.message_id)
        except (NotFound, HTTPException) as e:
            # Can't track reactions on a message I can't find
            # Happens for deleted messages. Safe to ignore.
            # await Utils.handle_exception(f"Failed to get message {channel.id}/{event.message_id}", self, e)
            return

        log_msg = f"{Utils.get_member_log_name(member)} used emoji "\
                  f"[ {emoji_used} ] in #{channel.name}.\n"\
                  f"{message.jump_url}"

        if e_db.remove:
            await message.clear_reaction(emoji_used)
            log_msg = f"{log_msg}\n--- I **removed** the reaction"

        if e_db.mute:
            guild_config = await self.bot.get_guild_db_config(guild.id)
            if guild_config and guild_config.mutedrole:
                try:
                    mute_role = guild.get_role(guild_config.mutedrole)
                    await member.add_roles(mute_role)
                    self.mutes[guild.id][str(member.id)] = timestamp
                    Configuration.set_persistent_var(f"react_mutes_{guild.id}", self.mutes[guild.id])
                    log_msg = f"{log_msg}\n--- I **muted** them"
                except Exception as e:
                    await Utils.handle_exception("reactmon failed to mute member", self.bot, e)
            else:
                await self.bot.guild_log(event.guild_id, "**I can't mute for reacts because `!guildconfig` mute role is not set.")

        if (e_db.log or e_db.remove or e_db.mute) and log_channel:
            await log_channel.send(log_msg)

    async def process_reaction_remove(self, timestamp, event):
        # TODO: Evaluate - count react removal and auto-mute for hitting threshold in given time?
        #  i.e. track react-remove-count per user over time. if count > x: mute/warn

        # Add user_id to dict of recent reaction removers with timestamp
        # now = datetime.now().timestamp()
        # self.react_removers[event.guild_id][event.user_id] = now

        # listening setting only apples to quick-remove
        server_is_listening = event.guild_id in self.react_watch_servers
        if not server_is_listening or await self.is_user_event_ignored(event):
            return

        # check recent reacts to see if they match the remove event
        for t, add_event in self.react_adds[event.guild_id].items():
            # Criteria for skipping an event in the list
            not_message = add_event.message_id != event.message_id
            not_user = add_event.user_id != event.user_id

            age = timestamp - t
            expired = 0 > age > self.min_react_lifespan[event.guild_id]
            if expired or not_message or not_user:
                # message id and user id must match remove event, and must not be expired
                continue

            # This user added a reaction that was removed within the warning time window
            guild = self.bot.get_guild(event.guild_id)
            member = guild.get_member(event.user_id)
            emoji_used = str(event.emoji)
            channel = self.bot.get_channel(event.channel_id)
            log_channel = await self.bot.get_guild_log_channel(guild.id)
            # ping log channel with detail
            if log_channel:
                content = f"{Utils.get_member_log_name(member)} " \
                          f"quick-removed [ {emoji_used} ] react from a message in {channel.mention}"
                try:
                    message = await channel.fetch_message(event.message_id)
                    content = f"{content}\n{message.jump_url}"
                except (NotFound, HTTPException) as e:
                    pass
                await log_channel.send(content)

async def setup(bot):
    await bot.add_cog(ReactMonitor(bot))


"""
Servo module using hardware PWM through pigpio.

The pigpiod daemon must be running in the background for these classes.

This also requires one of the hardware PWM0/PWM1 channels to first be
configured on one of the IO pins. The following worked to get PWM1
exposed on pin 19, in /boot/config.txt. You can also set up both
channels with a different dtoverlay.

dtoverlay=pwm,pin=19
"""

try:
    import pigpio
    HAS_PIGPIO = True
except ImportError:
    HAS_PIGPIO = False
import time
import numpy as np

class Servo(object):

    def __init__(self, pin=19, low=2.75, high=12.5, posrange=180, frequency=50):
        """
        low, high:  duty cycles limit in percent
        range:      range in some meaningful units, the motor will run
                    from 0 to +posrange
        frequency:  PWM frequency in Hz
        pin:        GPIO pin where the PWM channel is exposed
        """
        self._low = float(low)
        self._high = float(high)
        self._range = float(posrange)
        self._freq = frequency
        self._pin = pin
        self._duty = low + (high - low) / 2.0

        if not HAS_PIGPIO:
            raise RuntimeError('Need pigpio for this operation.')
        self._pi = pigpio.pi()
        self._update()

    def _update(self):
        self._pi.hardware_PWM(self._pin, self._freq, self._duty / 100 * 1e6)

    @property
    def position(self):
        return (self._duty - self._low) / (self._high - self._low) * self._range

    @position.setter
    def position(self, val):
        self._duty = self._low + val / self._range * (self._high - self._low)
        self._update()

class PenLifter(Servo):

    def __init__(self, up_pos=0, down_pos=90, move_time=.5, **kwargs):
        super(self.__class__, self).__init__(**kwargs)

        self._up = up_pos
        self._down = down_pos
        self._time = move_time
        self.up()

    def up(self):
        dt = self._time / 30.0
        for val in np.linspace(self.position, self._up, 30):
            self.position = val
            time.sleep(dt)
        #self.position = self._up

    def down(self):
        dt = self._time / 30.0
        for val in np.linspace(self.position, self._down, 30):
            self.position = val
            time.sleep(dt)
        #self.position = self._down


"""
utils.py
"""
import csv
import sys
import socket
import json
import re
import logging
import hashlib
import requests
import ssdeep
import pefile
import configuration_file as cfg
from werkzeug.utils import secure_filename

def break_csv(lines):
    """
    empty
    """
    for line in lines:
        if line.startswith('#'):
            lines.remove(line)
    reader = csv.reader(lines, quotechar='"',
                        delimiter=',',
                        quoting=csv.QUOTE_ALL,
                        skipinitialspace=True)
    next(reader)
    return reader

def init_connection_urlhaus():
    """
    Initialisation, connection to Url Haus
    Params:
    Returns:
    """
    try:
        logging.info('Fetching latest URLhaus list...')
        request = requests.get(cfg.urlhaus["link"])
        if request.status_code == 200:
            logging.info('Processing URLhaus list...')
            return request
        logging.error('Problem connecting to URLhaus. Status code: %i', request.status_code)

    except requests.exceptions.ConnectionError as error_connection:
        logging.warning('Problem connecting to URLhaus. Error: %i', error_connection)

    except Exception as exception:
        logging.warning('Problem connecting to URLhaus. Aborting task.')
        logging.exception(sys.exc_info())
        logging.exception(type(exception))
        logging.exception(exception.args)
        logging.exception(exception)

    return []

def clean_url(url):
    """
    Remove extraneous characters from URL.
    Params:
    - url: (type: string) URL to clean.
    Returns:
    - url: (type: string) clean URL.
    """
    if url is None:
        return None
    if '??' in url:
        url = url.split('??')[0]
    if url.endswith('?'):
        url = url[:-1]
    if '`' in url:
        url = url.replace('`', '')
    return url

def get_name_file_from_url(url):
    """
    Get name file from url
    Params:
    - url: (type: string).
    Returns:
    - name: (type: string).
    """

    last_slash = url.rfind('/')
    return url[last_slash+1:]

def save_infos_from_csv(item):
    """
    SAVE INFOS FROM CSV
    Params:
    - item: (type: string) URL.
    Returns:
    - id_malware, date_added, url_malware, url_status, threat, tags, urlhaus_link, reporter.
    """
    id_malware = item[0]
    date_added = item[1]
    url_malware = clean_url(item[2])
    if item[3] == "online":
        url_status = True
    else:
        url_status = False
    threat = item[4]
    tags = item[5]
    urlhaus_link = item[6]
    reporter = item[7]
    return id_malware, date_added, url_malware, url_status, threat, tags, urlhaus_link, reporter

def save_vt_report(vt_report, id_malware):
    with open(str(id_malware) + ".vt", "w+") as outfile:
        json.dump(vt_report, outfile)
        outfile.close()

def get_ip_geo_from_url(input):
    """Mofication Parser url et recuperer la partie essentiel pour analyser apres"""
    compile_rule1 = re.compile('^(?:http:\/\/|www\.|https:\/\/)([^\/]+)')
    match1 = re.match(compile_rule1, input)
    """Verifier cela est une addresse IP ou pas"""
    compile_rule = re.compile(r'(?<![\.\d])(?:\d{1,3}\.){3}\d{1,3}(?![\.\d])')
    match = re.match(compile_rule, match1.group(1))
    """Si cela est une addresse IP"""
    if match:
        ip = match.group()
    else:
        try:
            ip = socket.getaddrinfo(match1.group(1), 'http')[0][4][0]
        except:
            ip = "0.0.0.0"
    """A l aide du site ipstack, lire les données de cette addresse"""
    url = 'http://api.ipstack.com/' + ip + '?access_key=' + cfg.ipstack['apikey']
    try:
        a = requests.get(url)
        data = a.json()
    except:
        return "none", "none", "none", "none", "none", "none", "none", "none", "none", 0.0, 0.0, "none"
    country_name = data.get('country_name')
    ip = data.get('ip')
    continent_code = data.get('continent_code')
    continent_name = data.get('continent_name')
    country_code = data.get('country_code')
    region_code = data.get('region_code')
    region_name = data.get('region_name')
    city = data.get('city')
    zip = data.get('zip')
    latitude = data.get('latitude')
    longitude = data.get('longitude')
    full_report = data
    return ip, country_name, country_code, continent_name, continent_code, region_name, region_code, city, zip, latitude, longitude, full_report

def get_md5(fname):
    """
    Get malwate MD5
    Params:
    -fname
    Returns:
    - rslt
    """

    hash_md5 = hashlib.md5()
    with open(cfg.malware["malware_path"] + secure_filename(fname), 'rb') as f:
        for chunk in iter(lambda: f.read(4096), b""):
            hash_md5.update(chunk)
    rslt = hash_md5.hexdigest()
    return rslt

def get_sha256(fname):
    """
    Get malware sha256
    Params:
    -fname
    Returns:
    - rslt
    """

    hash_sha256 = hashlib.sha256()
    with open(cfg.malware["malware_path"]  + secure_filename(fname), 'rb') as f:
        for chunk in iter(lambda: f.read(4096), b""):
            hash_sha256.update(chunk)
    rslt = hash_sha256.hexdigest()
    return rslt

def get_sha1(fname):
    """
    Get malware sha1
    Params:
    -fname
    Returns:
    - rslt
    """

    hash_sha1 = hashlib.sha1()
    with open(cfg.malware["malware_path"]  + secure_filename(fname), 'rb') as f:
        for chunk in iter(lambda: f.read(4096), b""):
            hash_sha1.update(chunk)
    rslt = hash_sha1.hexdigest()
    return rslt

def get_ssdeep(fname):
    """
    Get malware ssdeep
    Params:
    -fname
    Returns:
    - rslt
    """

    hash_ssdeep = ssdeep.Hash()
    with open(cfg.malware["malware_path"]  + secure_filename(fname), 'rb') as f:
        for chunk in iter(lambda: f.read(4096), b""):
            hash_ssdeep.update(chunk)
    rslt = hash_ssdeep.digest()
    return rslt

def get_imphash(fname):
    """
    If the malwre is an exec get the impash
    Params:
    -fname, db
    Returns:
    - imps
    """
    try:
        imps = []
        p = pefile.PE(cfg.malware["malware_path"]  + secure_filename(fname))
        imphash = p.get_imphash()
        imps.append(imphash)
        return str(imps)
    except:
        return None

def is_exe(tags):
    """
    If the malwre is an exec
    Params:
    -tags (source csv urlhaus)
    Returns:
    - boolean
    """
    if "exe" in tags:
        return True
    return False


def ficha(nome='<desconhecido>', gols=0):
    print(f'O jogador {nome} fez {gols} gols.')

def entrar_gols(nome):
    while True:
        gols = input('Digite a quantidade de gols: ')
        if not (gols.isdigit()) or int(gols) < 0:
            print('\n\033[1;31mERRO! Entrada inválida! Tente novamente!\033[m\n')
        else:
            if nome == '':
                if int(gols) == 0:
                    ficha()
                else:
                    ficha(gols=int(gols))
            else:
                ficha(nome, int(gols))
            break

def main():
    nome = input('Entre com o nome do jogador: ')
    entrar_gols(nome)

if __name__ == '__main__':
    main()