# -*- coding: utf-8 -*-

import streamlit as st
from PIL import Image
import cv2
import numpy as np

def main():

    selected_box = st.sidebar.selectbox(
    'Choose any one',
    ('Welcome','Image Processing', 'Video', 'Face Detection',  'Object Detection')
    )

    if selected_box == 'Welcome':
        welcome()
    if selected_box == 'Image Processing':
        photo()
    if selected_box == 'Video':
        video()
    if selected_box == 'Face Detection':
        face_detection()
    if selected_box == 'Object Detection':
        object_detection()

def welcome():

    st.title('Image Processing using Streamlit')

    st.subheader('A simple app that shows few image processing algorithms.'
             )

    st.image('ImageProcessing.jpg',use_column_width=True)

def load_image(filename):
    image = cv2.imread(filename)
    return image

def photo():

    st.header("Thresholding, Edge Detection and Contours")

    if st.button('Original Image of Minion'):

        original = Image.open('minion.jpg')
        st.image(original, use_column_width=True)

    image = cv2.imread('minion.jpg')
    image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

    x = st.slider('Change Threshold value',min_value = 50,max_value = 255)

    ret,thresh1 = cv2.threshold(image,x,255,cv2.THRESH_BINARY)
    thresh1 = thresh1.astype(np.float64)
    st.image(thresh1, use_column_width=True,clamp = True)

    st.text("Bar Chart of the image")
    histr = cv2.calcHist([image],[0],None,[256],[0,256])
    st.bar_chart(histr)

    if st.button('Original Image of Cartoon'):

        original = Image.open('cartoon1.jpg')
        st.image(original, use_column_width=True)

    st.text("Press the button below to view Canny Edge Detection Technique")
    if st.button('Canny Edge Detector'):
        image = load_image("cartoon.jpg")
        edges = cv2.Canny(image,50,300)
        st.image(edges,use_column_width=True,clamp=True)

    y = st.slider('Change Value to increase or decrease contours',min_value = 50,max_value = 255)

    if st.button('Contours'):
        im = load_image("cartoon1.jpg")

        imgray = cv2.cvtColor(im,cv2.COLOR_BGR2GRAY)
        ret,thresh = cv2.threshold(imgray,y,255,0)
        contours, hierarchy = cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)

        img = cv2.drawContours(im, contours, -1, (0,255,0), 2)

        st.image(thresh, use_column_width=True, clamp = True)
        st.image(img, use_column_width=True, clamp = True)

def video():
    uploaded_file = st.file_uploader("Choose a video file to play")
    if uploaded_file is not None:
         bytes_data = uploaded_file.read()

         st.video(bytes_data)

    video_file = open('video.mp4', 'rb')

    video_bytes = video_file.read()
    st.video(video_bytes)

def face_detection():
    st.header("Face Detection using haarcascade")

    if st.button('See Original Image'):

        original = Image.open('friends.jpg')
        st.image(original, use_column_width=True)

    image2 = cv2.imread("friends.jpg")

    face_cascade = cv2.CascadeClassifier("haarcascade_frontalface_default.xml")
    faces = face_cascade.detectMultiScale(image2)
    print(f"{len(faces)} faces detected in the image.")
    for x, y, width, height in faces:
        cv2.rectangle(image2, (x, y), (x + width, y + height), color=(255, 0, 0), thickness=2)

    cv2.imwrite("faces.jpg", image2)

    st.image(image2, use_column_width=True,clamp = True)

def object_detection():

    st.header('Object Detection')
    st.subheader("Object Detection is done using different haarcascade files.")
    img = load_image("clock.jpg")
    img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

    clock = cv2.CascadeClassifier('haarcascade_clock.xml')
    found = clock.detectMultiScale(img_gray,minSize =(20, 20))

    amount_found = len(found)
    st.text("Detecting a clock from an image")
    if amount_found != 0:
        for (x, y, width, height) in found:

            cv2.rectangle(img_rgb, (x, y),
                          (x + height, y + width),
                          (0, 255, 0), 5)
    st.image(img_rgb, use_column_width=True,clamp = True)

    st.text("Detecting eyes from an image")

    image = load_image("eyes.jpg")
    img_gray_ = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    img_rgb_ = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)

    eye = cv2.CascadeClassifier('haarcascade_eye.xml')
    found = eye.detectMultiScale(img_gray_,
                                       minSize =(20, 20))
    amount_found_ = len(found)

    if amount_found_ != 0:
        for (x, y, width, height) in found:

            cv2.rectangle(img_rgb_, (x, y),
                              (x + height, y + width),
                              (0, 255, 0), 5)
        st.image(img_rgb_, use_column_width=True,clamp = True)

if __name__ == "__main__":
    main()


# -*- coding:utf-8 -*-
'''
@  author Yanqun Pan.
@  09/14 2018
@ State Key Laboratory of Esturine and Coastal Research,East China Normal University
'''
import time
from datetime import datetime
import datetime as dt
from Cprotocol import *
import Cprotocol
import numpy as np
import math,os
from apscheduler.schedulers.blocking import BlockingScheduler
import argparse
from enum import Enum
import sys
import configparser
from tkinter import *

scheduler = None
INVALID = -999

def calTimeAngel(longitude,longitude_timezone,time):
    '''
    计算时角
    :param longitude: 当地经度
    :param longitude_timezone: 当地时区所在经度，如北京时间120度
    :param time: 当地时区时间
    :return: 时角（度）
    '''
    #计算真太阳时
    # temp = time + dt.timedelta(minutes=(longitude_timezone-longitude)*4)
    temp = time + dt.timedelta(minutes=(longitude - longitude_timezone) * 4)
    realSolarTime = temp.hour+temp.minute/60.0+temp.second/3600.0
    # print(realSolarTime)
    timeAngle = (realSolarTime-12)*15
    return timeAngle

def calDeclination(month,day):
    '''
    计算太阳赤纬（度）
    :param month: 月份
    :param day: 天
    :return: 太阳赤纬（度）
    '''
    delta = datetime(2001, month, day)-datetime(2001, 1, 1)
    days = delta.days+1
    b = 2*np.pi*(days-1)/365.0
    declination = 0.006918-0.399912*np.cos(b)+0.070257*np.sin(b)-\
                  0.006758*np.cos(2*b)+0.000907*np.sin(2*b)-\
                  0.002697*np.cos(3*b)+0.00148*np.sin((3*b))

    return math.degrees(declination)

def calSolarElevationAzmuth(latitude,timeangle,declination):
    '''
    计算太阳高度和方位角
    :param latitude: 地理纬度
    :param timeangle: 时角
    :param declination: 太阳赤纬
    :return: 太阳高度角（度）和 方位角（度），正南方为0度，以东为负，以西为正
    '''
    sinHs = np.sin(latitude/180*np.pi)*np.sin(declination/180*np.pi)+\
           np.cos(latitude/180*np.pi)*np.cos(declination/180*np.pi)*np.cos(timeangle/180*np.pi)

    cosHs = np.cos(np.arcsin(sinHs))

    cosAs = (sinHs*np.sin(latitude/180*np.pi)-np.sin(declination/180*np.pi))/(cosHs*np.cos(latitude/180*np.pi))
    # print(cosAs)

    return np.arcsin(sinHs)*180/np.pi,np.arccos(cosAs)*180/np.pi

class Eaction(Enum):
    '''
    动作枚举
    '''
    left = 1
    right = 2
    up = 3
    down =4
    leftup = 5
    leftdown = 6
    rightup = 7
    rightdown = 8
    reset = 9

class CloudPlatform():
    def __init__(self,longitude,latitude,AzimuthSensorIni,AzimuthDelta,AzimuthSensorMin=0,AzimuthSensorMax=360,port='COM5',baudrate=9600,TIMEZONE='E08'):
        '''

        :param longitude: location of the platform deployed
        :param latitude:location of the platform deployed
        :param TIMEZONE: default East 8+, Beijing Time
        '''
        port, baudrate, bytesize, parity, stopbits, timeout = port, baudrate, 8, 'E', 1, 0.1
        try:
            self.pPort = serial.Serial(port=port, baudrate=baudrate, bytesize=bytesize, parity=parity, stopbits=stopbits,
                                  timeout=timeout)
        except SerialException as se:
            print('open serial[%s] failed.%s' % (port,se))
            sys.exit()

        self.AzimuthSensorIni = AzimuthSensorIni
        self.AzimuthDelta = AzimuthDelta

        self.protocolD = CprotocolD(self.pPort, pLen=7)
        # self.protocolD = None
        # self.pProtocol = protocolD

        self.currentHorizon = self.AzimuthSensorIni
        self.longitude,self.latitude = longitude,latitude
        self.AzimuthSensorMin = AzimuthSensorMin
        self.AzimuthSensorMax = AzimuthSensorMax
        self.scheduler = None
        self.pan_speed = Cprotocol.DEFAULT_PAN_SPEED
        print('init')

    def reset(self):
        '''
        让云台转到初始位置
        :return:
        '''
        self.pan_speed = Cprotocol.DEFAULT_PAN_SPEED
        if self.currentHorizon-self.AzimuthSensorIni >= 180:
            self.pan_speed = 30
            # 先左转180度,为了防止总是往一个方向转，线发生线缠绕的情况
            secs = 180 / (self.pan_speed * Cprotocol.PAN_SPEED_DEGREE)
            self.protocolD.left(secs,self.pan_speed)
            # print('左转了%d秒'%(secs))
        print('云台开始复位: %s' % (datetime.now()))
        self.protocolD.resetHorizon()
        self.currentHorizon = self.AzimuthSensorIni
        print('云台已复位: %s'%(datetime.now()))

    def reset_man(self):
        '''
        人工复位，根据self.currentHorizon 复位，由于 self.protocolD.resetHorizon()有不执行的问题，只能采用该方法
        缺点是：由于Cprotocol.PAN_SPEED_DEGREE存在误差，可能会导致并没有真正复位，日积月累角度差会不断增加。
        :return:
        '''
        print('云台开始复位: %s' % (datetime.now()))
        if self.currentHorizon!=self.AzimuthSensorIni:
            delta = self.currentHorizon - self.AzimuthSensorIni
            self.pan_speed = 30 if abs(delta)>10 else Cprotocol.DEFAULT_PAN_SPEED
            secs = abs(delta) / (self.pan_speed * Cprotocol.PAN_SPEED_DEGREE)

            print('当前HyperSAS方位：%.2f,初始方位：%.2f,角度差：%.2f,按当前速度%.3f/s,需%s转%.3f秒。'
                  % (self.currentHorizon, self.AzimuthSensorIni, delta, self.pan_speed * Cprotocol.PAN_SPEED_DEGREE,
                     "左" if delta < 0 else "右", secs))
            if delta<0:
                self.protocolD.right(secs,self.pan_speed)
            else:
                self.protocolD.left(secs, self.pan_speed)
        self.protocolD.resetHorizon() #执行最好，不执行也没办法，影响不大
        self.currentHorizon = self.AzimuthSensorIni
        print('云台已复位: %s' % (datetime.now()))

    def init(self):
        '''
        首先让云台转到初始位置，然后让传感器转到正北方向
        :return:
        '''
        self.reset()
        self.pan_speed = Cprotocol.DEFAULT_PAN_SPEED
        if abs(self.AzimuthSensorIni) > 10:
            self.pan_speed = 30
        secs = abs(self.AzimuthSensorIni)/(self.pan_speed*Cprotocol.PAN_SPEED_DEGREE)
        print("%s" % (datetime.now()))
        print('当前HyperSAS方位：%.2f,按当前速度%.3f/s,至正北需%s转%.3f秒。' % (self.AzimuthSensorIni, self.pan_speed*Cprotocol.PAN_SPEED_DEGREE, "左" if self.AzimuthSensorIni>0 else "右",secs))
        if self.AzimuthSensorIni > 0:
            self.protocolD.left(secs,self.pan_speed)
        else:
            self.protocolD.right(secs,self.pan_speed)
        self.currentHorizon = 0
        print("HyperSAS传感器已旋转至正北方向，%s"%(datetime.now()))

    def getIdealAzimuth(self,sa):
        '''
        :param sa:太阳方位角，以正北为基准
        :return: HySAS 理想角度, 不在范围内，返回-999
        '''
        AzimuthSensor = sa - self.AzimuthDelta
        if AzimuthSensor < self.AzimuthSensorMin:
            AzimuthSensor = sa + self.AzimuthDelta
            if AzimuthSensor > self.AzimuthSensorMax:
                return INVALID
        return AzimuthSensor

    def first_adjust(self):
        '''
        init后立即调整
        :return:
        '''
        self.pan_speed = Cprotocol.DEFAULT_PAN_SPEED
        now = datetime.now()
        timeangle = calTimeAngel(self.longitude, 120, now)
        # print(timeangle)
        dec = calDeclination(now.month, now.day)
        # print(dec)
        se, sa = calSolarElevationAzmuth(self.latitude, timeangle, dec)
        # print(sa)
        # 以正北为基准
        if timeangle > 0:
            sa = 180 + sa  ## 当前太阳的方位角
        else:
            sa = 180 - sa
        print(now)
        AzimuthSensor = self.getIdealAzimuth(sa=sa)
        if AzimuthSensor == INVALID:
            print("当前HyperSAS方位：%.2f,太阳方位：%.2f,HyperSAS理想方位%.2f不在[%.2f,%.2f]内，无法测量。"%(self.currentHorizon,
                                                                                sa,AzimuthSensor,self.AzimuthSensorMin,self.AzimuthSensorMax))
            return
        else:
            if AzimuthSensor > 10:
                self.pan_speed = 30
            else:
                self.pan_speed = Cprotocol.DEFAULT_PAN_SPEED

            secs = AzimuthSensor / (self.pan_speed * Cprotocol.PAN_SPEED_DEGREE)  ##当前平台的方位角与太阳的方位角之差，在当前速度下所需时间

            print('当前HyperSAS方位：%.2f,太阳方位：%.2f,角度差：%.2f,按当前速度%.3f/s,需右转%.3f秒。'
                  % (self.currentHorizon, sa, AzimuthSensor, self.pan_speed * Cprotocol.PAN_SPEED_DEGREE, secs))
            self.protocolD.right(secs,pan_speed=self.pan_speed)
            self.currentHorizon = AzimuthSensor

    def auto(self):
        self.pan_speed = Cprotocol.DEFAULT_PAN_SPEED
        now = datetime.now()
        timeangle = calTimeAngel(self.longitude, 120, now)
        # print(timeangle)
        dec = calDeclination(now.month, now.day)
        # print(dec)
        se,sa = calSolarElevationAzmuth(self.latitude, timeangle, dec)
        # print(sa)

        #以正北为基准
        if timeangle >0:
            sa = 180 + sa ## 当前太阳的方位角
        else:
            sa = 180 - sa

        # AzimuthSensor = 360 - np.abs(self.AzimuthSensorIni -(sa-self.AzimuthDelta))
        # AzimuthSensor = sa - self.AzimuthDelta
        print("\n %s"%(now))
        AzimuthSensor = self.getIdealAzimuth(sa=sa)
        if AzimuthSensor == INVALID:
            print("当前HyperSAS方位：%.2f,太阳方位：%.2f,HyperSAS理想方位%.2f不在[%.2f,%.2f]内，无法测量。" % (self.currentHorizon,sa, AzimuthSensor,
                                                                                self.AzimuthSensorMin,
                                                                                self.AzimuthSensorMax))
            return

        delta = AzimuthSensor-self.currentHorizon
        if abs(delta) > 10:
            self.pan_speed = 30

        secs = abs(delta) / (self.pan_speed * Cprotocol.PAN_SPEED_DEGREE)  ##当前平台的方位角与太阳的方位角之差，在当前速度下所需时间

        print('当前HyperSAS方位：%.2f,太阳方位：%.2f,理想方位：%.2f,角度差：%.2f,按当前速度%.3f/s,需%s转%.3f秒。'
              % (self.currentHorizon, sa,AzimuthSensor, delta, self.pan_speed*Cprotocol.PAN_SPEED_DEGREE, "左" if delta<0 else "右",secs))
        if delta <0: #左转
            self.protocolD.left(secs,pan_speed=self.pan_speed)
        else:
            self.protocolD.right(secs,pan_speed=self.pan_speed)

        self.currentHorizon = AzimuthSensor

    def setRuntime(self,start,end):
        self.starthour = int(start.split(':')[0])
        self.endhour = int(end.split(':')[0])

    def setResettime(self,hour,min):
        self.resethour,self.resetmin = hour,min

    def do(self,action):
        try:
            if action == Eaction.left.value:
                self.protocolD.left()
            if action == Eaction.right.value:
                self.protocolD.right()
        except KeyboardInterrupt:
            self.protocolD.stop()
        finally:
            self.closePort()

        if action == Eaction.reset.value:
            self.protocolD.resetHorizon()

    def closePort(self):
        if self.pPort.isOpen():
            self.pPort.close()

    def isRuning(self):
        response = self.protocolD.getHorizonLoc()
        print(str(response))
        if response==None or len(response)<4:
            return False
        return True

def autoRun(pCloudPlatform):
    scheduler = BlockingScheduler()
    print('开始时间%d点，结束时间%d点'%(pCloudPlatform.starthour,pCloudPlatform.endhour+1))
    scheduler.add_job(pCloudPlatform.auto, 'cron',minute='*/10', hour='%d-%d'%(pCloudPlatform.starthour,pCloudPlatform.endhour))
    scheduler.add_job(pCloudPlatform.reset_man, 'cron',hour='%d'%(pCloudPlatform.resethour),minute='%d'%(pCloudPlatform.resetmin))
    print('Press Ctrl+{0} to exit'.format( 'C' if os.name == 'nt' else 'Break'))
    try:
        scheduler.start()
    except (KeyboardInterrupt,SystemExit):
        # pCloudPlatform.init()
        pCloudPlatform.reset()
        time.sleep(30)
        scheduler.shutdown()
    finally:
        pCloudPlatform.closePort()

def usage():
    print('Cloud Platform')

if __name__ == '__main__':

    config =configparser.ConfigParser()
    try:
        config.read_file(open("config.ini", 'r'))
    except:
        print('read config.ini failed!')
        sys.exit(0)

    longitude,latitude = float(config['location']['longitude']),float(config['location']['latitude'])
    print("location: %s, %s"%(longitude,latitude))
    AzimuthSensorIni = float(config['HyperSAS']['AzimuthSensor'])
    print("HypySAS initial azimuth:%.2f"%AzimuthSensorIni)
    AzimuthDelta = float(config['HyperSAS']['AzimuthDelta']) #测量时的sensor 与 太阳的角度差
    print("azimuth delta:%.2f"%(AzimuthDelta))
    StartTime, EndTime = config['HyperSAS']['StartTime'],config['HyperSAS']['EndTime']
    AzimuthSensorMin = float(config['HyperSAS']['AzimuthSensorMin'])
    AzimuthSensorMax = float(config['HyperSAS']['AzimuthSensorMax'])
    ResetTimeHour,ResetTimeMin = int(config['HyperSAS']['ResetTime'].split(':')[0]),int(config['HyperSAS']['ResetTime'].split(':')[1])

    print("azimuth range for sensor: %.2f, %.2f" % (AzimuthSensorMin, AzimuthSensorMax))

    Port, Baudrate = config['serial']['Port'], int(config['serial']['Baudrate'])

    parser = argparse.ArgumentParser()
    parser.add_argument("-d", "--do",help="action of the cloudplatform: left = 1 right = 2 up = 3\n"
                                                       "down =4leftup = 5leftdown = 6rightup = 7\n"
                                                       "rightdown = 8reset = 9",type=int)
    parser.add_argument("-s", "--speed",default = 10,help="speed, (1-63)")
    parser.add_argument("-a", "--auto",default=False,help="automaticlly adjust the angle")

    args = parser.parse_args()

    v_auto = args.auto
    v_action = args.do
    if v_action == None and v_auto == False:
        print('当前非自动模式，请指定动作指令: -d left right ....')
        sys.exit(0)
    pCloudPlatform = CloudPlatform(longitude, latitude,AzimuthSensorIni,AzimuthDelta,AzimuthSensorMin=AzimuthSensorMin,AzimuthSensorMax=AzimuthSensorMax,
                                   port=Port,baudrate=Baudrate)
    pCloudPlatform.setRuntime(StartTime,EndTime)
    pCloudPlatform.setResettime(ResetTimeHour,ResetTimeMin)

    if pCloudPlatform.isRuning()==False:
        print('云台没有连接成功，请检查电源！')
        sys.exit(0)

    # print (v_auto)
    if bool(v_auto) == True:
        print('当前将按照自动模式运行！')
        pCloudPlatform.init()
        print('init finished.')
        time.sleep(30)
        pCloudPlatform.first_adjust()
        time.sleep(30)
        autoRun(pCloudPlatform)
        # print(pCloudPlatform.pPort.closed)
        # pCloudPlatform.protocolD.left(10)
        pass

    else:
        # pCloudPlatform.do(action=v_action)
        print('暂未实现,哈哈哈')
        pass


'''
Uses previously saved weights and biases to predict
sin model
'''
import tensorflow as tf

def approx():

	x = tf.placeholder(tf.float32)

	with tf.Session() as sess:
		sess.run(tf.global_variables_initializer())
		loader = tf.train.import_meta_graph('sin-model.meta')
		loader.restore(sess, 'sin-model')

		graph = tf.get_default_graph()

		x = graph.get_tensor_by_name("input:0")

		input_val = input("Enter value to be approximated: \n")

		input_val = float(input_val)

		feed_dict = {x:input_val}

		output = graph.get_tensor_by_name("output:0")

		print(sess.run(output, feed_dict))


#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Description

"""

import sys
import os
import argparse
import random
import numpy as np
import torch
from ptranking.utils.args.argsUtil import ArgsUtil

from ptranking.ltr_adhoc.eval.ltr import LTREvaluator, LTR_ADHOC_MODEL
from ptranking.ltr_tree.eval.ltr_tree import TreeLTREvaluator, LTR_TREE_MODEL
from ptranking.ltr_adversarial.eval.ltr_adversarial import AdLTREvaluator, LTR_ADVERSARIAL_MODEL

def setup_seed(seed):
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
    torch.backends.cudnn.deterministic = True
"""
The command line usage:

(1) Without using GPU
python pt_ranking.py -model ListMLE -dir_json /home/dl-box/WorkBench/Dropbox/CodeBench/GitPool/wildltr_ptranking/testing/ltr_adhoc/json/

(2) Using GPU
python pt_ranking.py -cuda 0 -model ListMLE -dir_json /home/dl-box/WorkBench/Dropbox/CodeBench/GitPool/wildltr_ptranking/testing/ltr_adhoc/json/

"""

if __name__ == '__main__':

    """
    >>> Learning-to-Rank Models <<<
    (1) Optimization based on Empirical Risk Minimization
    -----------------------------------------------------------------------------------------
    | Pointwise | RankMSE                                                                   |
    -----------------------------------------------------------------------------------------
    | Pairwise  | RankNet                                                                   |
    -----------------------------------------------------------------------------------------
    | Listwise  | LambdaRank % ListNet % ListMLE % RankCosine %  ApproxNDCG %  WassRank     |
    |           | STListNet  % LambdaLoss                                                   |
    -----------------------------------------------------------------------------------------

    (2) Adversarial Optimization
    -----------------------------------------------------------------------------------------
    | Pointwise | IRGAN_Point                                                               |
    -----------------------------------------------------------------------------------------
    | Pairwise  | IRGAN_Pair                                                                |
    -----------------------------------------------------------------------------------------
    | Listwise  | IRGAN_List                                                                |
    -----------------------------------------------------------------------------------------

    (3) Tree-based Model (provided by LightGBM)
    -----------------------------------------------------------------------------------------
    | LightGBMLambdaMART                                                                    |
    -----------------------------------------------------------------------------------------

    >>> Supported Datasets <<<
    -----------------------------------------------------------------------------------------
    | LETTOR    | MQ2007_Super %  MQ2008_Super %  MQ2007_Semi %  MQ2008_Semi                |
    -----------------------------------------------------------------------------------------
    | MSLRWEB   | MSLRWEB10K %  MSLRWEB30K                                                  |
    -----------------------------------------------------------------------------------------
    | Yahoo_LTR | Set1 % Set2                                                               |
    -----------------------------------------------------------------------------------------
    | ISTELLA_LTR | Istella_S | Istella | Istella_X                                         |
    -----------------------------------------------------------------------------------------

    """

    parser = argparse.ArgumentParser('Run pt_ranking.')
    parser.add_argument('-cuda', type=int, help='specify the gpu id if needed, such as 0 or 1.', default=None)

    ''' path of json files specifying the evaluation details '''
    parser.add_argument('-dir_json', type=str, help='the path of json files specifying the evaluation details.')

    ''' learning rate for pretraining'''
    parser.add_argument('-pretrain_lr', type=float, help='the learning rate for pretraining.')

    ''' path of json files specifying the evaluation details '''
    parser.add_argument('-finetune_lr', type=float, help='the learning rate for finetuning.')

    ''' the output_dir '''
    parser.add_argument('-trial_num', type=int, help='the trial id.')

    ''' the augmentation '''
    parser.add_argument('-aug_type', type=str, help='the type of augmentation.')

    ''' the augmentation percentage '''
    parser.add_argument('-aug_percent', type=float, help='the percentage of augmentation.')

    ''' dim '''
    parser.add_argument('-dim', type=int, help='the dimension of pretrainer.')

    ''' layers '''
    parser.add_argument('-layers', type=int, help='the number of layers.')

    ''' pretrainer '''
    parser.add_argument('-pretrainer', type=str, help='the type of pretrainer.')

    ''' shrink '''
    parser.add_argument('-shrink', type=float, help='How much to shrink the train set')

    ''' freeze '''
    parser.add_argument('-freeze', type=int, help='Whether to freeze prev layers')

    ''' number of probing layers '''
    parser.add_argument('-probe_layers', type=int, help='Number of probing layers')

    ''' finetune_only '''
    parser.add_argument('-finetune_only', type=int, help='no pretrain')

    ''' finetune_trial '''
    parser.add_argument('-finetune_trials', type=float, help='finetuning trial id')

    argobj = parser.parse_args()

    if argobj.pretrainer == 'LightGBMLambdaMART':
        evaluator = TreeLTREvaluator()
        evaluator.run(model_id=argobj.pretrainer, dir_json=argobj.dir_json, config_with_json=True, argobj=argobj)

    else:
        if argobj.freeze:
            print('Finetune FROZEN')
        evaluator = LTREvaluator(cuda=argobj.cuda)
        # setup_seed(0)
        if argobj.aug_type != 'none' and not argobj.finetune_only:
            print('Starting pretraining!', sys.stderr)
            argobj.is_pretraining = True
            evaluator.run(model_id=argobj.pretrainer, dir_json=os.path.join(argobj.dir_json, '{0}/'.format(argobj.pretrainer)), config_with_json=True, argobj=argobj)

        print('Starting finetuning!', sys.stderr)
        argobj.is_pretraining = False
        if argobj.aug_type == 'none':
            evaluator.run(model_id="LambdaRank", dir_json=os.path.join(argobj.dir_json, 'lambdarank/'), config_with_json=True, argobj=argobj)
        else:
            evaluator.run(model_id="LambdaRankTune", dir_json=os.path.join(argobj.dir_json, 'lambdaranktune/'), config_with_json=True, argobj=argobj)


#Import libraries
import streamlit as st
import matplotlib.pyplot as plt
import pandas as pd
from pickle import load

#
st.title('2016 Election Modeling')
st.markdown('See how your county voted in the 2016 election relative to its regression predicted vote proportion')

#
df = pd.read_csv('Data/election_full_dataset.csv')
counties = df['countystate'].values

#
label = 'Select a county name from the dropdown menu below (785 largest US counties only)'
county = st.selectbox(label, counties)

#
county_datapoint = df[df['countystate']==county]
features = ['incomeroot','medianageroot','unemployment','bachelorsdegree','white','hispanic',
           'popdensityroot','veteran%root','disability%root','manufacturing/capitaroot',
           'agriculture/capita','mining/capita']
x = county_datapoint[features]
y = county_datapoint['voteproportion'].values[0]*100

#
ss = load(open('Pickles/standard_scaler.pickle', 'rb'))
x = ss.transform(x)

#
lr = load(open('Pickles/linear_regression.pickle', 'rb'))
y_pred = lr.predict(x)[0]*100

#
columns = ['County', 'State', 'Predicted Trump Vote %', 'Actual Trump Vote %']
output = pd.DataFrame(columns=columns)
output.loc[0] = [county.split(',')[0], county.split(',')[-1], y_pred, y]
output.index = [' ']

#
st.table(output)


from debug import debug, warning, error, info
import messaging

import pickle
import pipeline

#
# BEWARE:
#  This little bit of unpickle machinery allows old saved pipelines to be
#  loaded if any of the data structures or messages involved in setting the
#  pipeline state change.
#
#  You SHOULD NOT RELY ON THESE HOOKS on anything other than a temporary basis:
#  once you can load your old pipeline, save it in the new format, check that
#  it works, then remove all copies of the old pipeline, and remove the hook
#  code that you add here.
#

def makeNewLocalNodeInput(input, subtype, schedType=messaging.InputSchedType.Must_Be_New, compatible_subtypes=None):
    if compatible_subtypes is None:
        compatible_subtypes = [subtype]
    if subtype == messaging.ParamValueType.BoundedFloatType:
        compatible_subtypes.append(messaging.ParamValueType.floatType)
        compatible_subtypes.append(messaging.ParamValueType.int32Type)
    if subtype == messaging.ParamValueType.floatType:
        compatible_subtypes.append(messaging.ParamValueType.BoundedFloatType)
    if subtype == messaging.ParamValueType.int32Type:
        compatible_subtypes.append(messaging.ParamValueType.BoundedFloatType)
    return messaging.LocalNodeInput(input, subtype, schedType, compatible_subtypes)

Unpickle_Filters = {
    'LocalNodeInput' : makeNewLocalNodeInput,
}

class FilterUnpickler(pickle.Unpickler):
    def __init__(self, file):
        pickle.Unpickler.__init__(self, file)

    def load_reduce(self):
        stack = self.stack
        args = stack.pop()
        func = stack[-1]
        if func.__name__ in Unpickle_Filters:
            warning('filtering pickle load of %s(%s)' % (func.__name__, args))
            value = Unpickle_Filters[func.__name__](*args)
        else:
            value = func(*args)
        stack[-1] = value
    pickle.Unpickler.dispatch[pickle.REDUCE] = load_reduce

#
# SIMILARLY, don't rely on these filters to image pipeline node parameters
# either: they're designed to scale into a progressive migrations system
# (multiple filters could easily be applied progressively), but performance
# would necessarily get pretty awful pretty quickly
#

def filterPercentileNodeParameters(params_in):
    params_out = {}
    for param, value in params_in.items():
        param_key = param.input if isinstance(param, messaging.LocalNodeInput) else param
        if param_key == 'percentile' and isinstance(value, float):
            params_out[param] = messaging.BoundedFloat(value, 0, 100, messaging.BoundedFloatType.Clamps)
        else:
            params_out[param] = value;
    return params_out

def filterClampXNodeParameters(params_in):
    params_out = {}
    range_max = None
    range_min = None
    for param, value in params_in.items():
        param_key = param.input if isinstance(param, messaging.LocalNodeInput) else param
        if param_key == 'Max':
            range_max = value
        elif param_key == 'Min':
            range_min = value
        else:
            params_out[param] = value;
    if range_min is not None and range_max is not None:
        params_out['Range'] = messaging.Range(float(range_min), float(range_max))
    return params_out

def filterLevelsNodeParameters(params_in):
    params_out = {}
    for param, value in params_in.items():
        param_key = param.input if isinstance(param, messaging.LocalNodeInput) else param
        if param_key == 'black level' and isinstance(value, int):
            params_out[param] = messaging.BoundedFloat(float(value), 0, 255, messaging.BoundedFloatType.Clamps)
        elif param_key == 'white level' and isinstance(value, int):
            params_out[param] = messaging.BoundedFloat(float(value), 0, 255, messaging.BoundedFloatType.Clamps)
        else:
            params_out[param] = value;
    return params_out

def filterGuiOutputNodeParameters(params_in):
    params_out = {}
    for param, value in params_in.items():
        param_key = param.input if isinstance(param, messaging.LocalNodeInput) else param
        if param_key == 'jpeg quality' and isinstance(value, int):
            params_out[param] = messaging.BoundedFloat(float(value), 0, 100, messaging.BoundedFloatType.Clamps)
        else:
            params_out[param] = value;
    return params_out

NodeParam_Filters = {
    messaging.NodeType.Percentile : filterPercentileNodeParameters,
    #messaging.NodeType.ClampInt   : filterClampXNodeParameters,
    #messaging.NodeType.ClampFloat : filterClampXNodeParameters,
    messaging.NodeType.Levels     : filterLevelsNodeParameters,
    messaging.NodeType.GuiOutput  : filterGuiOutputNodeParameters
}

def load(inf):
    '''Load a pipeline file from disk, and return its contents. This does not manipulate the image pipeline.'''
    saved = FilterUnpickler(inf).load()
    return saved

def dump(outf, state):
    '''Save a passed pipeline state to a file. This does not manipulate the image pipeline.'''
    pickle.dump(state, outf)


import sys
sys.path.insert(1, '../')

import numpy as np
import pandas as pd
import os
import shutil
from datetime import datetime

metadata_dob = pd.read_csv('itmt_v2/cvs_files/metadata_bch_scan_date.csv', header=0)
def find_age_gender(id_to_find):
    for idx, row in metadata_dob.iterrows():
        if str(id_to_find).zfill(7)==str(row['BCH_MRN']).zfill(7):
            #12/20/93
            dob=datetime.strptime(row['dob'], "%m/%d/%y")
            sex=row['sex']
            mri_date=datetime.strptime(row['scan_date'], "%m/%d/%y")
    age = (mri_date-dob).days/365
    if sex == "F":
        sex=2
    else:
        sex=1
    return round(age,2), sex

path = 'data/itmt2.0_checkedBK_bch/'
path_to = 'data/itmt2.0/'
df = pd.read_csv('data/itmt2.0_checkedBK_bch/annotations.csv', header=None)
df= df[df[1]==1] # select only the good cases

save_metadata = []

for idx in range(0,len(df)):
    file_name = df.iloc[idx][0]
    new_path = path_to+file_name.split(".")[0]+"/"
    if not os.path.exists(new_path):
        os.mkdir(new_path)
        shutil.copy(path+file_name.split(".")[0]+"_mask.nii.gz", new_path+"TM.nii.gz")

    shutil.copy('data/t1_mris/bch_reg_ench/z/'+file_name.split(".")[0]+'/'+file_name.split(".")[0]+'.nii', new_path+file_name.split(".")[0]+'.nii.gz')
    age, sex = find_age_gender(file_name.split(".")[0])
    save_metadata.append([file_name,age, sex,'BCH'])
    print(file_name,age, sex)

pd.DataFrame(save_metadata).to_csv("itmt_v2/cvs_files/itmt2.0_cancer.csv", header=["Filename","AGE_M","SEX","Dataset"], index=None)
#Filename,AGE_M,SEX,Dataset,train/test,Ok registered? Y/N


with open("18_input.txt") as fin:
    data = fin.read()

def evaluate(expression):
    #print("expression:", expression)

    # Goal: get rid of all the addition signs
    multiplicands = []

    # These still exist
    prevValue = ""

    # Current index
    index = 0

    # Boolean to tell whether the last argument is part of a sum
    lastIsAddition = False

    while index < len(expression):
        char = expression[index]

        if char.isdigit():
            prevValue += char

        elif char == "+":
            # Find where it ends!
            summands = [int(prevValue)]
            prevValue = ""
            index += 1

            while index < len(expression):
                char = expression[index]

                if char.isdigit():
                    prevValue += char

                elif char == "(":
                    # Balance the parentheses to get to the end
                    ls = 1 # Lefts
                    rs = 0 # Rights
                    startIndex = index + 1

                    while ls != rs and index < len(expression):
                        index += 1
                        ls += expression[index] == "("
                        rs += expression[index] == ")"

                    # Lefts and rights are balanced, evaluate what's inside
                    insideParentheses = expression[startIndex:index]
                    prevValue = evaluate(insideParentheses)

                elif char == "*":
                    break

                elif char == "+":
                    summands.append(int(prevValue))
                    prevValue = ""

                index += 1

            if index == len(expression):
                lastIsAddition = True

            if prevValue != "":
                summands.append(int(prevValue))

            #print("summands:", summands)

            multiplicands.append(sum(summands))
            prevValue = ""

        elif char == "*":
            multiplicands.append(int(prevValue))
            prevValue = ""

        elif char == "(":
            # Balance the parentheses to get to the end
            ls = 1 # Lefts
            rs = 0 # Rights
            startIndex = index + 1

            while ls != rs and index < len(expression):
                index += 1
                ls += expression[index] == "("
                rs += expression[index] == ")"

            # Lefts and rights are balanced, evaluate what's inside
            insideParentheses = expression[startIndex:index]
            #print("inside:", insideParentheses)
            prevValue = evaluate(insideParentheses)

        index += 1

    # Get the last multiplicand
    if prevValue != "" and not lastIsAddition:
        multiplicands.append(int(prevValue))

    #print(multiplicands)

    prod = 1
    for i in multiplicands:
        prod *= i

    return prod

ans = 0
for expression in data.split("\n"):
    expression = expression.replace(" ", "")
    ans += evaluate(expression)

print(ans)

#print(evaluate("((2 + 4 * 9) * (6 + 9 * 8 + 6) + 6) + 2 + 4 * 2"))


#!/usr/bin/env python
from __future__ import print_function

from collections import defaultdict, namedtuple
from operator import add
from math import floor

class AutoInc(object):
    # ''' Get a uniquely auto-incremented integer value '''
    __slots__ = ('v',)

    def __init__(self, value=0):
        self.v = int(value)

    def __call__(self):
        self.v += 1
        return self.v

    def add(self, value):
        self.v += int(value)
        return self.v

    def __int__(self):
        return self.v

    def __float__(self):
        return float(self.v)

    def __long__(self):
        return long(self.v)

    def __repr__(self):
        return str(self.v)

    def __str__(self):
        return str(self.v)

class Histogram(defaultdict):

    @staticmethod
    def lexcmp(k0, k1):
        """ Hacky comparator to privilege string length """
        return cmp(len(k1), len(k0)) or cmp(k0, k1)

    @staticmethod
    def autoinc():
        return AutoInc()

    def __init__(self, *args, **kwargs):
        for arg in args:
            argclsname = arg.__class__.__name__
            if argclsname.startswith('FrozenHistogram'):
                # reconstitute FrozenHistogram
                kwargs.update(arg._asdict())
            elif (argclsname.startswith('Histogram') or
                  argclsname.startswith('defaultdict') or
                  argclsname.startswith('dict')):
                # copy-construct
                kwargs.update(dict(arg))
            else:
                # normal arg: column name
                kwargs.update({ str(arg): AutoInc() })
        super(Histogram, self).__init__(AutoInc, *tuple(), **kwargs)

    def prettyprint(self, **kwargs):
        from pprint import pformat
        return "%s(%s,\n%s)" % (
            self.__class__.__name__,
            AutoInc.__name__,
            pformat(dict(self), **kwargs))

    def __repr__(self):
        return self.prettyprint()

    def __str__(self):
        longest_key = sorted(self.iterkeys(),
                             cmp=Histogram.lexcmp,
                             reverse=False)[0]
        return self.prettyprint(depth=10,
                                width=len(longest_key))

    def inc(self, idx):
        return self[str(idx)]()

    def add(self, idx, value):
        return self[str(idx)].add(value)

    def val(self, idx):
        return int(self[idx])

    def keyset_hash(self):
        return hex(abs(hash(self.iterkeys())))

    def frozentype(self, rename=True):
        return namedtuple('FrozenHistogram_%s' % self.keyset_hash(),
            tuple(self.iterkeys()), rename=rename)

    def freeze(self):
        return self.frozentype()(**self)

    def normalize(self):
        ceil = float(sum(int(v) for v in self.itervalues()))
        return self.frozentype()(**dict(
            (ituple[0], float(ituple[1]) / ceil) for ituple in self.iteritems()
        ))

    def max(self):
        """ Maximum value """
        return max(*(int(v) for v in self.values()))

    def min(self):
        """ Minimum value """
        return min(*(int(v) for v in self.values()))

    def sum(self):
        return reduce(add, (int(v) for v in self.itervalues()))

    def average(self):
        """ Histogram mean/average (float) """
        return float(self.sum()) / float(len(self))

    def avg(self):
        """ Histogram mean/average (integer) """
        return int(floor(self.average()))

    def range(self):
        """ Histogram numerical range (integer) """
        return self.max() - self.min()

    def valuelist(self):
        return sorted(int(v) for v in self.itervalues())

    def median(self):
        """ Median histogram value (integer, or int/2) """
        vals = self.valuelist()
        idx = len(vals) / 2
        if len(vals) % 2:
            return vals[idx+1]
        return float(vals[idx] + vals[idx+1]) / 2.0

    def keys_for_value(self, value):
        """ List of keys with the given integer value """
        return map(lambda pair: pair[0],
            filter(lambda pair: int(pair[1]) == int(value),
            self.iteritems()))

    def valueset(self):
        return set(self.valuelist())

if __name__ == '__main__':
    # Hh = Histogram('color', 'phone', 'zip', 'firstname', 'lastname')
    Hh = Histogram('color', 'phone', 'zip', firstname=10, lastname=10)
    Hp = Histogram(firstname=0, lastname=0)

    Hh['color']()
    Hh['color']()
    Hh['color']()
    Hh['color']()
    Hh['color']()
    Hh['color']()

    Hh['phone']()
    Hh['phone']()
    Hh['phone']()
    Hh['phone']()
    Hh['phone']()
    Hh['phone']()
    Hh['phone']()
    Hh['phone']()
    Hh['phone']()
    Hh['phone']()
    Hh['phone']()
    Hh['phone']()
    Hh['phone']()
    Hh['phone']()
    Hh['phone']()
    Hh['phone']()
    Hh['phone']()
    Hh['phone']()
    Hh['phone']()

    Hh['zip']()
    Hh['zip']()
    Hh['zip']()
    Hh['zip']()

    print(Hh)
    print(Hh.freeze()._asdict())
    print(Hh.normalize())
    print('')

    # print("MAX Hh: %s" % max(*(int(v) for v in Hh.values())))
    print("MIN Hh: %s" % Hh.min())
    print("MAX Hh: %s" % Hh.max())
    print("SUM Hh: %s" % Hh.sum())
    print("AVG Hh: %s" % Hh.avg())
    print("RANGE Hh: %s" % Hh.range())
    print("MEDIAN Hh: %s" % Hh.median())
    print("KEYS IN Hh FOR 19: %s" % Hh.keys_for_value(19))
    print("KEYS IN Hh FOR 777: %s" % Hh.keys_for_value(777))
    print("KEYS IN Hh FOR 0: %s" % Hh.keys_for_value(0))
    print('')

    hh = Histogram(Hh, 'iheard', 'youlike', yo=10, dogg=100)

    print(hh)
    print(hh.freeze()._asdict())
    print(hh.normalize())
    print('')

    print("MIN hh: %s" % hh.min())
    print("MAX hh: %s" % hh.max())
    print("SUM hh: %s" % hh.sum())
    print("AVG hh: %s" % hh.avg())
    print("RANGE hh: %s" % hh.range())
    print("MEDIAN hh: %s" % hh.median())
    print("KEYS IN hh FOR 19: %s" % hh.keys_for_value(19))
    print("KEYS IN hh FOR 777: %s" % hh.keys_for_value(777))
    print("KEYS IN hh FOR 0: %s" % hh.keys_for_value(0))
    print('')

    Hp.inc('color')
    Hp.inc('color')
    Hp.inc('color')
    Hp.inc('color')
    Hp.inc('color')
    Hp.inc('color')

    Hp.inc('zip')
    Hp.inc('zip')

    Hp.inc('phone')
    Hp.inc('phone')
    Hp.inc('phone')
    Hp.inc('phone')
    Hp.inc('phone')
    Hp.inc('phone')
    Hp.inc('phone')
    Hp.inc('phone')
    Hp.inc('phone')
    Hp.inc('phone')
    Hp.inc('phone')
    Hp.inc('phone')
    Hp.inc('phone')
    Hp.inc('phone')
    Hp.inc('phone')
    Hp.inc('phone')
    Hp.inc('phone')
    Hp.inc('phone')
    Hp.inc('phone')
    Hp.inc('phone')
    Hp.inc('phone')
    Hp.inc('phone')
    Hp.inc('phone')
    Hp.inc('phone')

    print(Hp)

    print(Hp.freeze()._asdict())
    print(Hp.normalize())

    print("MAX Hp: %s" % Hp.max())


#
# examples for how to use SfBS_utils
# reference: SfBS Page 30, Demo 1.1
from SfBS_utils import *

data = [7, 3, 9, 5, 4]

# compute sigma x
sigma_x = sfbs_sum(data)
print(sigma_x)

# compute (sigma x)'s square
square_sigma_x = sigma_x * sigma_x
print(square_sigma_x)

# compute sigma (x's square)
data_square = list()
for item in data:
    data_square.append(item * item)
sigma_x2 = sfbs_sum(data_square)
print(sigma_x2)

#compute sigma (x + 5)
data_add_5 = list()
for item in data:
    data_add_5.append(item + 5)
sigma_x_add_5 = sfbs_sum(data_add_5)
print(sigma_x_add_5)

data_sub_2 = list()
for item in data:
    data_sub_2.append(item - 2)
sigma_x_sub_2 = sfbs_sum(data_sub_2)
print(sigma_x_sub_2)


# std
import multiprocessing as mp

# local
from recipes.logging import LoggingMixin

class ConservativePool(mp.pool.Pool):
    """
    Subclass of Pool which avoid making a potentially huge list when mapping over a function
    and is therefor more memory efficient
    """

    def _map_async(self, func, iterable, mapper, chunksize, callback=None,
                   error_callback=None):
        """
        Helper function to implement map, starmap and their async counterparts.
        """
        if self._state != 0:
            raise ValueError("Pool not running")
            #

        if chunksize is None:
            raise ValueError('Need chunksize')

        task_batches = self.__class__._get_tasks(func, iterable, chunksize)
        result = mp.pool.MapResult(self._cache, chunksize, N, callback,
                                   error_callback=error_callback)
        self._taskqueue.put((((result._job, i, mapper, (x,), {})
                              for i, x in enumerate(task_batches)), None))
        return result

class ProcessPool(LoggingMixin):
    '''A trimmed down version of mp.pool.Pool'''

    def __init__(self, inq, outq=None, processes=None, **kws):

        self.inq = inq
        self.outq = outq
        self.maintain = kws.get('maintain')
        self._kws = kws

        #self._state = RUN

        if processes is None:
            processes = os.cpu_count() or 1

        self._processes = processes
        self._pool = []
        self._repopulate_pool()

        self._worker_handler = threading.Thread(
            target=ProcessPool._handle_workers, args=(self,))
        self._worker_handler.daemon = True
        self._worker_handler._state = RUN
        self._worker_handler.start()

        # self._worker_monitor = threading.Thread(target=ProcessPool._monitor_workers,
        # args=(self,))
        #self._worker_monitor._state = RUN
        # self._worker_monitor.start()

    def _maintain_pool(self):
        """Clean up any exited workers and start replacements for them.
        """
        if self._join_exited_workers():
            if self.maintain.is_set():
                self._repopulate_pool()
                time.sleep(0.1)

    def _join_exited_workers(self):
        """Cleanup after any worker processes which have exited due to reaching
        their specified lifetime.  Returns True if any workers were cleaned up.
        """
        cleaned = False
        for i in reversed(range(len(self._pool))):
            worker = self._pool[i]
            if worker.exitcode is not None:
                # worker exited
                self.logger.info('cleaning up worker %d' % i)
                worker.join()
                self.logger.info('joined {:s}', worker)
                cleaned = True
                del self._pool[i]

        #alive = sum([p.is_alive() for p in self._pool])
        #worked = [p._completed.value() for p in self._pool]
        #names = [p.name for p in self._pool]
        # self.logger.info('Done cleaning. workers: {:d}; active: {:d};\n{:s}\n{:s}'
                # '' % (len(self._pool), alive, str(names), str(worked)))
        return cleaned

    def _repopulate_pool(self):
        """Bring the number of pool processes up to the specified number,
        for use after reaping workers which have exited.
        """
        for i in range(self._processes - len(self._pool)):
            w = self.Process(self.inq, self.outq, **self._kws)
            self._pool.append(w)
            #w.name = w.name.replace('Process', 'PoolWorker')
            w.daemon = False
            w.start()
            self.logger.info('Added worker')

        #alive = sum([p.is_alive() for p in self._pool])
        self.logger.info('Repopulated: workers: {:d};', len(self._pool))

    @staticmethod
    def _handle_workers(pool):
        thread = threading.current_thread()

        # Keep maintaining workers maintain condition is set
        i = 0
        while thread._state == RUN:
            i += 1
            if not (i % 5):
                worked = [p._completed.value() for p in pool._pool]
                pool.logger.debug('Tasks completed: {:s}' % str(worked))
            pool._maintain_pool()
            time.sleep(0.1)

        pool.logger.debug('worker handler exiting')

    # @staticmethod
    # def _monitor_workers(pool):
        #thread = threading.current_thread()

        # while pool.maintain.is_set():#thread._state == RUN:
        ##alive = sum([p.is_alive() for p in pool._pool])
        # pool.logger.info('# workers: {:d}' % (len(pool._pool),))
        # time.sleep(0.1)

        ##alive = sum([p.is_alive() for p in pool._pool])
        #worked = [p._completed.value() for p in pool._pool]
        #names = [p.name for p in pool._pool]
        # pool.logger.info('workers: {:d};\n{:s}\n{:s}'
        # '' % (len(pool._pool), str(names), str(worked)))

        #pool.logger.debug('_monitor_workers exiting')

    #
    # def kill_switch(self, sentinel=None):
        #self.logger.info('%s adding %i sentinels' %(self, len(self._pool)))
        # for w in self._pool:
        # self.inq.put(sentinel)

    # def close(self):
        #self.logger.info('closing pool')

        # if self._state == RUN:
        #self._state = CLOSE
        #self._worker_handler._state = CLOSE

        #self.logger.info('Adding {:d} sentinels' %len(self._pool))
        # for w in self._pool:
        # self.inq.put(None)
        #self.logger.debug('sentinel added')

    def join(self):

        self.logger.debug('joining pool: {:s}', self)
        self.logger.info('# workers: {:d}; ', len(self._pool))

        # Add a poison pill for each process in the pool
        for _ in range(len(self._pool)):
            self.inq.put(SENTINEL)

        # NOTE:
        # We expect that the pool will join all the exited processes.  However,
        # sometimes we are stuck with processes that never read anything from the
        # queue at all during their lifetime (only the dark lord knows why this
        # happens). Here we circumvent the deadlock with a timeout and termination #HACK
        # FIXME: Figure out why these processes sometimes get stuck like this to
        # avoid all this cruft!
        TIMEOUT = 5
        kill_switch = False
        start = time.time()
        while time.time() - start <= TIMEOUT:
            if any(p.is_alive() for p in self._pool):
                time.sleep(.1)  # Just to avoid hogging the CPU
            else:
                self.logger.info('All processes successfully joined')
                break
        else:
            self.logger.warning('TIMEOUT!')
            kill_switch = True

        # stop worker handler that has been trying to join the exited processes
        self._worker_handler._state = CLOSE
        self._worker_handler.join()
        self.logger.debug('_worker_handler joined')

        # We only enter this if we didn't 'break' above.
        if kill_switch:
            self.logger.warning(
                'These processes did not join before %.1f timeout: (%s). '
                'Killing them now.' % (TIMEOUT, ', '.join(map(str, self._pool)))
            )
            for p in self._pool:
                p.terminate()
                p.join()

            self.logger.info('Killing spree complete')

        # the queue may not be empty because of passive workers
        # (or some other reason!!??)
        while True:
            leftover = []
            try:
                leftover.append(self.inq.get_nowait())
                self.inq.task_done()
            except Empty:
                logging.debug('Queue is empty!')
                break
        if len(leftover):
            self.logger.warning('The following was left in the queue: %s'
                                % str(leftover))

        # All the processes in the pool are now stopped
        self.inq.join()
        self.logger.info('Queue joined: %s' % self)