#!/usr/bin/env python
#-*- coding:utf-8 -*-

import os.path, collections, time
from PyQt4 import QtGui, QtCore, QtWebKit, QtNetwork

allowedExtensions = ['.html', '.css', '.js', '.xhtml', '.dhtml']

class CookieJar(QtNetwork.QNetworkCookieJar):
    def __init__(self, ctr):
        self.ctr = ctr
        self.init = False
        self.cookieStorage = []
        QtNetwork.QNetworkCookieJar.__init__(self)
        self.time = time.time()
        self.updateCookie()
        self.initCookie()
        self.init = True

    def initCookie(self):
        self.setAllCookies([QtNetwork.QNetworkCookie.parseCookies(c)[0] for c in self.cookieStorage])

    def readCookie(self):
        return [c.toRawForm().data().decode() for c in self.allCookies()]

    def saveCookie(self):
        self.cookieStorage = [c.toRawForm().data().decode() for c in self.allCookies()]

    def getSyncCookie(self):
        return self.ctr.getSyncCookie()

    def updateCookie(self):
        if time.time() - self.time < 5 and self.init is True:
            return

        syncCookie = self.getSyncCookie()
        selfCookieMap = {v.split(';')[0].split('=')[0]: v for v in
                         [c.toRawForm().data().decode() for c in self.allCookies()] if v is not None}

        selfCookieMap.update(syncCookie)
        self.cookieStorage = selfCookieMap.values()
        self.initCookie()
        self.time = time.time()

class NetworkAccessManager(QtNetwork.QNetworkAccessManager):
    def __init__(self, ctr, allowedExtensions, cache_size=100):
        self.ctr = ctr
        QtNetwork.QNetworkAccessManager.__init__(self)

        self.cookie = CookieJar(self)
        self.setCookieJar(self.cookie)

        # initialize the manager cache
        cache = QtNetwork.QNetworkDiskCache()
        #QDesktopServices.storageLocation(QDesktopServices.CacheLocation)
        cache.setCacheDirectory('webCache')

        # need to convert cache value to bytes
        cache.setMaximumCacheSize(cache_size * 1024 * 1024)
        self.setCache(cache)

        self.allowedExtensions = allowedExtensions

        self.connect(self, QtCore.SIGNAL('sslErrors(QNetworkReply*, const QList<QSslError>&)'), self.sslErrors)

    def sslErrors(self, reply, errors):
        '''
        requiredCert = reply.sslConfiguration().peerCertificate()
        defaultConfig = QtNetwork.QSslConfiguration.defaultConfiguration()
        caList = defaultConfig.caCertificates()
        caList.append(requiredCert)
        defaultConfig.setCaCertificates(caList)
        QtNetwork.QSslConfiguration.setDefaultConfiguration(defaultConfig)
        reply.setSslConfiguration(defaultConfig)'''
        reply.ignoreSslErrors()

    def createRequest(self, operation, request, data):
        self.cookie.updateCookie()

        if data is not None:
            print('******')
            print(data)

        #operation == self.GetOperation:
        request.setAttribute(QtNetwork.QNetworkRequest.CacheLoadControlAttribute, QtNetwork.QNetworkRequest.PreferCache)

        request = self.bookTicket(request)

        '''
        for i in (dir(request)):
            print(i)

        rawHeaderList = request.rawHeaderList()
        for i in rawHeaderList:
            print(request.rawHeader(i))
        '''

        reply = QtNetwork.QNetworkAccessManager.createRequest(self, operation, request, data)
        reply.error.connect(self.catchError)

        #reply.downloadProgress.connect(self.progress)
        reply.finished.connect(self.finished)

        return reply

    def finished(self):
        self.ctr.loadFinished()

    def bookTicket(self, request):

        if (request.url().toString().startswith(
                'https://dynamic.12306.cn/otsweb/order/querySingleAction.do?method=submutOrderRequest')
            or request.url().toString().startswith('http://127.0.0.1/env.php')):
            print('match')
            request.setRawHeader('Host', 'dynamic.12306.cn')
            request.setRawHeader('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8')

            request.setRawHeader('Referer', 'https://dynamic.12306.cn/otsweb/order/querySingleAction.do?method=init')
            request.setRawHeader('User-Agent', 'Mozilla/5.0 (MSIE 9.0 Windows NT 6.1 Trident/5.0)')
            request.setRawHeader('Connection', 'keep-alive')
            request.setRawHeader('Content-Type', 'application/x-www-form-urlencoded')
        '''
(Request-Line)	POST /otsweb/order/querySingleAction.do?method=submutOrderRequest HTTP/1.1
Host	dynamic.12306.cn
User-Agent	Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:21.0) Gecko/20100101 Firefox/21.0
Accept	text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
Accept-Language	zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3
Accept-Encoding	gzip, deflate
Referer	https://dynamic.12306.cn/otsweb/order/querySingleAction.do?method=init
Cookie	JSESSIONID=22FF47CF0FFEAC7FBD65DB984C7C836F; BIGipServerotsweb=2446590218.48160.0000
Connection	keep-alive
Content-Type	application/x-www-form-urlencoded
Content-Length	737

        '''
        return request

    def isForbidden(self, request):
        extensions = os.path.splitext(request.url().toString())

        if extensions is not None and extensions[1].lower() in self.allowedExtensions:
            return True
        else:
            return False

    def catchError(self, eid):
        if eid not in (301, ):
            print('Error:', eid, self.sender().url().toString())

    def readCookie(self):
        self.cookie.readCookie()

    def closeEvent(self):
        self.cookie.saveCookie()

    def getSyncCookie(self):
        return self.ctr.getSyncCookie()

class WebPage(QtWebKit.QWebPage):
    def __init__(self):
        QtWebKit.QWebPage.__init__(self)
        self.userAgent = 'Mozilla/5.0 (MSIE 9.0 Windows NT 6.1 Trident/5.0)'

    def userAgentForUrl(self, url):
        return self.userAgent

    def javaScriptAlert(self, frame, message):
        #pass
        print('Alert:', message)

    def javaScriptConsoleMessage(self, message, line_number, source_id):
        pass
        #print('Console:', message, line_number, source_id)

class WebView(QtWebKit.QWebView):
    _windows = set()

    def __init__(self, ctr, parent=None):

        '''
        connect(ui->webView,SIGNAL(linkClicked(QUrl)),this,SLOT(lick(QUrl)));//连接信号和槽
ui->webView->page()->setLinkDelegationPolicy(QWebPage::DelegateAllLinks);//当一个链接被激活
ui.webView->page()->mainFrame()->evaluateJavaScript("Test()");
        '''
        super(WebView, self).__init__(parent)
        self.ctr = ctr

        self.settings().setAttribute(QtWebKit.QWebSettings.DeveloperExtrasEnabled, True)
        self.settings().setAttribute(QtWebKit.QWebSettings.JavascriptEnabled, True)
        self.settings().setAttribute(QtWebKit.QWebSettings.JavascriptCanOpenWindows, True)
        self.settings().setAttribute(QtWebKit.QWebSettings.PluginsEnabled, True)

        webPage = WebPage()
        self.manager = NetworkAccessManager(self, allowedExtensions)
        webPage.setNetworkAccessManager(self.manager)
        self.setPage(webPage)
        self.connect(self, QtCore.SIGNAL('linkClicked(QUrl)'), lambda x: self.require(x))
        self.page().setLinkDelegationPolicy(QtWebKit.QWebPage.DelegateAllLinks)

        #self.require('http://127.0.0.1/env.php')
        #self.load(QtCore.QUrl('http://www.12306.cn/mormhweb/'))

    def loadFinished(self):
        self.page().mainFrame().evaluateJavaScript("test()");
        print('loadFinished')

    def closeEvent(self, event):
        self.manager.closeEvent()
        #print(self.manager.readCookie())
        self._removeWindow(self)
        event.accept()

    def createWindow(self, webWindowType):
        window = self.newWindow()
        if webWindowType == QtWebKit.QWebPage.WebModalDialog:
            window.setWindowModality(QtCore.Qt.ApplicationModal)
        window.show()
        return window

    def getSyncCookie(self):
        return self.ctr.getSyncCookie()

    def require(self, url):
        self.load(QtCore.QUrl(url))

    def post(self, data):
        if data is None or len(data) < 0:
            return
        formData = ''
        for i, v in data.items():
            if not i.lower() == 'url':
                formData += '<input type="text" name="%s" value="%s" />' % (i, v)

        form = '<form action="%s" method="post" name="postForm">%s<input type="submit" value="submit"></form>' % (
            data['url'], formData)
        #script = ''
        script = ''
        html = '%s%s' % (form, script)

        self.setHtml(html)

    def bookTicket(self, ticketData, condition):

        url = 'http://127.0.0.1/env.php'
        #url = 'https://dynamic.12306.cn/otsweb/order/querySingleAction.do?method=submutOrderRequest'

        data = ticketData.split("#")
        query = collections.OrderedDict()

        query['station_train_code'] = data[0]
        query['train_date'] = condition["orderRequest.train_date"]
        query['seattype_num'] = condition['seatTypeAndNum']
        query['from_station_telecode'] = data[4]
        query['to_station_telecode'] = data[5]
        query['include_student'] = condition['includeStudent']
        query['from_station_telecode_name'] = condition['orderRequest.from_station_name']
        query['to_station_telecode_name'] = condition['orderRequest.to_station_name']
        query['round_train_date'] = condition["roundTrainDate"]
        query['round_start_time_str'] = condition['roundStartTimeStr']
        query['single_round_type'] = condition['singleRoundType']
        query['train_pass_type'] = condition['trainPassType']
        query['train_class_arr'] = condition['trainClassArr']
        query['start_time_arr'] = condition['orderRequest.start_time_str']
        query['lishi'] = data[1]
        query['train_start_time'] = data[2]
        query['trainno4'] = data[3]
        query['arrive_time'] = data[6]
        query['from_station_name'] = data[7]
        query['to_station_name'] = data[8]
        query['from_station_no'] = data[9]
        query['to_station_no'] = data[10]
        query['ypInfoDetail'] = data[11]
        query['mmStr'] = data[12]
        query['locationCode'] = data[13]
        query['url'] = url

        self.post(query)

class SimpleBrowsers(QtGui.QDialog):
    def __init__(self, ctr, parent=None):
        super(SimpleBrowsers, self).__init__(parent)
        self.ctr = ctr
        self.webView = WebView(self)

        page = self.webView.page()
        #page.settings().setAttribute(QtWebKit.QWebSettings.JavascriptEnabled, False)
        page.settings().setAttribute(QtWebKit.QWebSettings.PluginsEnabled, True)
        page.settings().setAttribute(QtWebKit.QWebSettings.DeveloperExtrasEnabled, True)
        self.webInspector = QtWebKit.QWebInspector(self)
        self.webInspector.setPage(page)

        shortcut = QtGui.QShortcut(self)
        shortcut.setKey(QtCore.Qt.Key_F12)
        shortcut.activated.connect(self.toggleInspector)
        self.webInspector.setVisible(False)

        self.splitter = QtGui.QSplitter(self)
        self.splitter.setOrientation(QtCore.Qt.Vertical)

        self.verticalLayout = QtGui.QVBoxLayout()
        self.verticalLayout.addWidget(self.splitter)

        self.splitter.addWidget(self.webView)
        self.splitter.addWidget(self.webInspector)

        self.setLayout(self.verticalLayout)

    def closeEvent(self, event):
        self.webView.closeEvent(event)
        event.accept()

    def getSyncCookie(self):
        return self.ctr.getSyncCookie()

    def bookTicket(self, ticketData, condition):
        self.webView.bookTicket(ticketData, condition)

    def post(self, data):
        self.webView.post(data)

    def toggleInspector(self):
        self.webInspector.setVisible(not self.webInspector.isVisible())

class Ctr:
    def __init__(self):
        self.ticketData = 'K528#05:45#08:00#630000K5280C#GZQ#HYQ#13:45#广州#衡阳#01#05#1*****30124*****00081*****01023*****0028#A86124DD6525AE6502C2860A53F45662115F625406DEA2133AEA577F#Q6'
        self.condition = {'trainPassType': 'QB', 'orderRequest.to_station_name': '衡阳', 'orderRequest.trainCodeText': '',
                          'orderRequest.from_station_telecode': 'GZQ', 'roundStartTimeStr': '00:00--24:00',
                          'orderRequest.from_station_name': '广州', 'singleRoundType': 1,
                          'orderRequest.start_time_str': '00:00--24:00', 'trainClassArr': 'QB#D#Z#T#K#QT#',
                          'orderRequest.to_station_telecode': 'HYQ', 'roundTrainDate': '2013-06-27',
                          'orderRequest.train_date': '2013-06-27', 'seatTypeAndNum': '', 'includeStudent': '00'}

    '''
{'station_train_code':	'G101',
'train_date	:'2013-06-26',
'seattype_num':'',
'from_station_telecode'	:'VNP',
'to_station_telecode':	'AOH',
'include_student'	:'00',
'from_station_telecode_name':	'åäº¬',
'to_station_telecode_name'	:'ä¸æµ·',
'round_train_date'	:'2013-06-26',
'round_start_time_str'	:'00:00--24:00',
'single_round_type'	:'1',
'train_pass_type':'QB',
'train_class_arr':	'QB#D#Z#T#K#QT#',
'start_time_str':	'00:00--24:00',
'lishi'	:'05:23',
'train_start_time':	'07:00',
'trainno4'	:'240000G10102',
'arrive_time'	:'12:23',
'from_station_name' :	'åäº¬å',
'to_station_name'	:'ä¸æµ·è¹æ¡¥',
'from_station_no'	:'01',
'to_station_no'	:'08',
'ypInfoDetail'	:'O*****0216M*****01219*****0021',
'mmStr'	:'0C18275B4E6471D2D28186C83D976EDB16A862912EA6B33F9A4775D0',
'locationCode'	:'P3'}
       '''

    def getSyncCookie(self):
        return {'JSESSIONID': 'JSESSIONID=6538C6EA7DB3ECFC1D3EE8BAF6301896; domain=dynamic.12306.cn; path=/otsweb',
                'BIGipServerotsweb': 'BIGipServerotsweb=2111045898.62495.0000; domain=dynamic.12306.cn; path=/'}

'''
if __name__ == '__main__':
    import sys

    app = QtGui.QApplication(sys.argv)
    c = Ctr()
    b = SimpleBrowsers(c)
    b.bookTicket(c.ticketData, c.condition)
    b.show()
    app.exec_()'''


# -*- coding: utf-8 -*-
"""
Created on Thu Mar 21 18:56:06 2019

@author: ererkka
"""
import matplotlib.pyplot as plt
import seaborn as sns

from src.const import CASES, CASE_LABELS
from src.results_reader import ResultsReader
from src.plotting import presentation_style

#%%
def get_state(result):
    return result.get_symbol('r_state', name='GWh',
                             index_names=['scenario', 'grid', 'node', 't'])

rr = ResultsReader('results/backbone')
rr.cases = CASES
data = rr.apply(get_state)

#%%

def plot_state(ax, case, data, label, **lineplot_kwargs):
    ts = data.xs('hydro', level='grid') / 1e3
    resampled = ts.unstack([0,1]).groupby(lambda t: int(t/24)).mean().unstack()
    resampled.name = ts.name
    resampled.index.names = ts.index.names[0:2] + ['day']
    df = resampled.reset_index()
    sns.lineplot(x='day', y='GWh', hue='node',
                 data=df[df['node'].str.contains('Reservoir')],
                 ax=ax, legend=False, ci='sd', err_kws={'alpha': 0.6}
                 )
    # Add case label
    ax.text(0.5, 0.85, label, horizontalalignment='center',
         verticalalignment='center', transform=ax.transAxes)

sns.set_style('ticks')
sns.set_palette('bright')
plt.style.use('dark_background')
presentation_style()
fig, axs = plt.subplots(1, len(CASES), sharex=True, sharey=True)
for ax, case, label in zip(axs, CASES, CASE_LABELS):
    plot_state(ax, case, data[case], label)
    ax.set_xticks([])
axs[0].set_ylabel('TWh')
axs[0].xaxis.set_visible(False)
fig.subplots_adjust()
fig.tight_layout()


#
################################################################################
# Original Author: A.J. Rubio-Montero (http://orcid.org/0000-0001-6497-753X),  #
#          CIEMAT - Sci-Track Group (http://rdgroups.ciemat.es/web/sci-track), #
#          for the EOSC-Synergy project (EU H2020 RI Grant No 857647).         #
# License (SPDX): BSD-3-Clause (https://opensource.org/licenses/BSD-3-Clause)  #
# Copyright (c): 2020-today, The LAGO Collaboration (http://lagoproject.net)   #
################################################################################

# note:
# command line, arti_params identical than do_sim.sh in ARTI.
# https://docs.python.org/3.3/library/argparse.html

import argparse, sys
#from builtins import int

CORSIKA_VER='75600'

def _get_arti_params_json_md(arti_dict):

    j= {"@graph" : [
    {
     "@id" : "/"+arti_dict['p']+"#artiParams",
     "@type" : "lago:ArtiParams",
     "lago:fluxTime" : "P"+str(arti_dict['p'])+"S",
     "lago:highEnergyIntModel" : arti_dict['h'],
     "lago:detectorSite" : "https://github.com/lagoproject/DMP/blob/0.0.1/defs/sitesLago.jsonld#"+arti_dict['s'],
     "lago:obsLev" : arti_dict['k'],
     "lago:flatArray" : not arti_dict['y'],
     "lago:cherenkov" : arti_dict['e'],
     "lago:debug" : arti_dict['k'],
     "lago:highEnergyCutsSecondaries" : arti_dict['a']
    }
    ]}
    return j

def get_sys_args():

    disclaimer='do_onedata: simulating LAGO sites and storing/publishing results in OneData'
    #epilog= "this can be ASCII art"
    parser = argparse.ArgumentParser(description=disclaimer, add_help=False)

    #parser.add_argument('-sum', dest='accumulate', default=max,
    #                   help='sum the integers (default: find the max)')

    #parser.add_argument('-sum', dest='accumulate', default=max,
    #                   help='sum the integers (default: find the max)')

    #showhelp() {
    #  echo
    #  echo -e "$0 version $VERSION"
    #  echo
    #  echo -e "USAGE $0:"
    #  echo
    #  echo -e "  -w <working dir>               : Working directory, where bin (run) files are located"
    #removed for OneData
    #  echo -e "  -p <project name>              : Project name (suggested format: NAMEXX)"
    #project name is set by the script as a fixed CODENAME
    #  echo -e "  -t <flux time>                 : Flux time (in seconds) for simulations"
    parser.add_argument('-t', dest='t', required=True, type=int,
                       help='Flux time (in seconds) for simulations')
    #  echo -e "  -v <CORSIKA version>           : CORSIKA version"
    # fixed, every docker image has their own CORSIKA version
    #  echo -e "  -h <HE Int Model (EPOS|QGSII)> : Define the high interaction model to be used"
    parser.add_argument('-h', dest='h', required=True, choices=['EPOS', 'QGSII'],
                       help='Define the high interaction model to be used <(EPOS|QGSII)>')
    #  echo -e "  -u <user name>                 : User Name."
    parser.add_argument('-u', dest='u', required=True,
                       help='ORCID code describing user, plain usernames are no allowed for publication')
    #  echo -e "  -s <site>                      : Location (several options)"
    parser.add_argument('-s', dest='s', required=True, choices=["hess","sac","etn","ber","lim","glr",
                                                                "mch","bga","mge","brc","and","mpc",
                                                                "cha","cid","mor","lsc","mbo","ccs"],
                       help='Predefined LAGO site')
    #  echo -e "  -j <procs>                     : Number of processors to use"
    parser.add_argument('-j', dest='j', default=1,
                       help='Number of processors to use')
    #  echo -e "  -y                             : Select volumetric detector mode (default=flat array)"
    parser.add_argument('-y', action='store_true', default=None,
                       help='Select volumetric detector mode (default=flat array)')
    #  echo -e "  -e                             : Enable CHERENKOV mode"
    parser.add_argument('-e', action='store_true', default=None,
                       help='Enable CHERENKOV mode')
    #  echo -e "  -d                             : Enable DEBUG mode"
    parser.add_argument('-d', action='store_true', default=None,
                       help='Enable DEBUG mode')
    #  echo -e "  -a                             : Enable high energy cuts for secondaries"
    parser.add_argument('-a', action='store_true', default=None,
                       help='Enable high energy cuts for secondaries')
    #  echo -e "  -k <altitude, in cm>           : Fix altitude, even for predefined sites"
    parser.add_argument('-k', dest='k', required=True, type=float,
                       help='Fix altitude, even for predefined sites, in cm, float and scientific notation allowed')
    #  echo -e "  -?                             : Shows this help and exit."
    parser.add_argument('-?', action='help', help='Shows this help and exit.')
    #  echo
    #}

    args = parser.parse_args()

    args_dict=vars(args)

    #add working dir, project, and corsika version...

    args_dict.update({'v':CORSIKA_VER})

    #project a.k.a codename
    #it shoulds describe a simulation

    codename=args_dict['s'] +'_'+ str(args_dict['t']) +'_'+ str(args_dict['k']) +'_'+ args_dict['v'] +'_'+ args_dict['h']
    if args_dict['y'] is True:
        codename+='_volu'
    else:
        codename+='_flat'

    if args_dict['e'] is True:
        codename+='_Cherenk'

    if args_dict['a'] is True:
        codename+='_HEcuts'

    args_dict.update({'p': codename})

    #working dir
    #args_dict.update({'w': '/opt/corsika-'+CORSIKA_VER+'-lago/run/'+str(args_dict['t'])})
    args_dict.update({'w': '/opt/corsika-'+CORSIKA_VER+'-lago/run/'})

    s=''
    for (key,value) in args_dict.items():
        if value is not None:
            s+=' -'+key
            if value is not True:
                s+=' '+str(value)

    return (s, args_dict, _get_arti_params_json_md(args_dict))


def cartesian_product(arr):
    points_index = [0 for i in range(len(arr))]
    bounds = range(len(arr))
    points = []
    coords = []
    incomplete = True
    while incomplete:
        for i in bounds:
            coords.append(arr[i][points_index[i]])
        points.append(coords)
        coords = []
        for i in [elem for elem in reversed(bounds)]:
            if points_index[i] < len(arr[i]) - 1:
                points_index[i]+=1
                for j in bounds:
                    if j>i:
                        points_index[j] = 0
                break
        else:
            incomplete = False
    return points


import json
from neo4j import GraphDatabase

conn = GraphDatabase.driver("bolt://54.160.119.107:35012", auth=("neo4j", "interpreters-flashes-soap"))

def createLink(name, fromNode, toNode, linker, liste, id="id", quantity=None):
    for objet in liste:
        if quantity:
            quantity = objet["quantity"]
        if objet[linker]:
            with conn.session() as session:
                session.write_transaction(requestCreateLink, name, fromNode, toNode, linker, objet, id, quantity)

def requestCreateLink(tx, name, fromNode, toNode, linker, objet, id, quantity):
    if quantity:
        requ = f"MATCH (a:{fromNode}{{id:'{objet[id]}'}}), (b:{toNode}{{id:'{objet[linker]}'}}) CREATE (a)-[r:{name}{{quantity:{quantity}}}]->(b)"
    else:
        requ = f"MATCH (a:{fromNode}{{id:'{objet[id]}'}}), (b:{toNode}{{id:'{objet[linker]}'}}) CREATE (a)-[r:{name}]->(b)"
    tx.run(requ)
    print(requ)

with open("datas.json", "r") as read_file:
    jsonTables = json.load(read_file)

for table, datas in jsonTables.items():
    if table == "Order":
        createLink("USE", "Order", "Coupon", "coupon_id", datas)

    if table == "Product":
        createLink("PART_OF", "Product", "Category", "category_id", datas)

    if table == "orders_orderitem":
        createLink("CONTAINS", "Order", "Product", "product_id", datas, id="order_id", quantity=1)


# -*- coding: utf-8 -*-
"""
Created v1 on Sun Mar 11 09:30:49 2018
Created v2 on Tue Jul 31 13:15:22 2018
Created v3 on 20181030

@author: bransonngai
"""
# import argparse
import talib as ta

import BATrader as ba
from BATrader.final.product import Stock
from BATrader.final.data_handler import MemoryDict
from BATrader.algo.algo import last_signal_bool, chg, chgp
from BATrader.utils.toolbox import timeit
from BATrader.market.MarketReader import convert

# import backtrader as bt
# import backtrader.feeds as btfeeds
# import backtrader.indicators as btind

import pandas as pd

from queue import Queue
from threading import Thread

from BATrader.logger import Logger
from BATrader.final.filters import Filters

from collections import OrderedDict
from copy import copy

import inspect

def divided_and_show_pct(a, b):
    """
    Two period of data change in pecentage (without %)
    """
    return round((float(a) / float(b)) * 100, 2)

"""
# 劵商異動:持股量佔比日變動達2% 或 周變動達4% 或 月變動達10%
# 過去X period 中, 升跌Y點數或以上的日子有多少

# 中午收市後, 找一找30mins Break out的股票, 並pus
# 找30分鐘圖,調整完反彈的股票-e.g. 1108
# 現價對於上市價之比例

# 941 20140718 - 60/200 平均線乖離不超5%, 放量上攻, 連破2條平均線,open\<ma60\<ma200\<close
# 992 週線下破60天
# 當日夜晚8點, 至第二朝8點, 呢12個鐘發報既最新公告 (特別留意沒有炒作焦點一段時間的日子, 如大跌之後牛皮的日子)

## 每日報告:
大市上日PE

from HKEx import PE_Report
pe = PE_Report()
pe.output_excel()

## 星期報告:
#### 過去一個星期:
藍籌入面, 10大升幅, 整體升跌比例, 恒指升跌
主板入面, 10大升幅, 整體升跌比例
0~50億市值主板股票, 10大升跌
50~200億市值主板股票, 10大升跌
板塊升跌幅Heatmap
"""

# Vector Scanner
# http://www.bossqiao.com/02%20coding/

# ------ Scanner ------
class Scanner(object):
    """
    Created on Sun May 15 23:37:02 2016
    PriceServer v1 2016-05
    RTQ update Robot

    scan_list : the scan target
    logic_dic : key-value pair, contain a callback function and params
    scan_result : list

    實現方法1:
        Dataframe, if condition true then return 股票 sym
        multiple conditions, set 左佢, return all matched

    實現方法2:
        由頭掃到落尾
    last_week_up_x_pct
    2018-10

        新增一個filter class, 當立即可以用symlist回答的, 就用filter 去實現, 非常快
        其次要做 dayback 計數的, 就用 scanner class.

        Scanner Class 的重新定義: 理應是Cerebro的設計, 像容器一般把所有東西放進去, 按
        個start就執行。 但scanner不同的地方是, 會有多於一個以上的condition, 而需要用到
        的data也不同, 故strategry class可以滿足經常改變的部份。

    2019-05

        因為 Strategy 終於成形, 發現只要將 s (Stock obj) 傳入一個獨立logic, 再return
        True False, 已經可以成為一個Condiction. 而且可以重用。

        Scanner Class的設計絶妙, 彈性很大。

        如果要位移 ( e.g. Close[0] -> Close[1] ) 可以在 s obj 做手腳腳

    Usage:
        scanner = Scanner()

        scanner.addsymlist(ba.fr.get_main_gem())
        # scanner.addfilter('and', filter_by_over_ma_60())
        scanner.addfilter('and', flt.filter_by_amount(10000000.0))
        scanner.addfilter('and', flt.filter_by_over_ma(60))
        # scanner.addfilter('not', filter_ot_list())
        scanner.addcondition(HighVolatilityStock)
        #    scanner.addstrategy(TestStrategy)

    """

    def __init__(self, running_mode=0, cond_method= 0, recalc= False, dayback= None):

        self.run_mode = {0: 'SingleThread', 1: 'Threading'}.get(running_mode)
        self.cond_apply_method = {0: 'recursive', 1: 'independent'}.get(cond_method)  # Recursive is one-and-one

        self.strats = list()  # From Cerebro, addstrategy will add to here
        self.datas = list()

        self.scan_list = []  # the scan target container of each loop
        self.scan_list_groups = {}  # Sometimes we needs "Sector" -> { Sector : [symlist] }
        self.scan_result = []  # the scan result
        self.filters_and = []  # match
        self.filters_not = []  # not match
        self.condition_list = {}

        self.last_scan_result = {'result': OrderedDict()}  # the last scan result

        self.flt = Filters()

        # Memory dict
        MemoryDict.recalc = recalc
        MemoryDict.dayback = dayback
        self.memory_db = MemoryDict()

        # For threading
        self.queue = Queue()

        # logger
        self.logger = None

    @timeit
    def run(self):
        """
        Run like cerebro
        """
        # Logger should init when running, not the class init, otherwise
        # the log will be hold
        self.logger = Logger(logname=ba.fr.path_output + 'logger\\%s_scanner.log' % ba.dk.tradeday(),
                             logger='main').getlog()

        # Every time to run, copy the loaded_full_list
        # self.scan_list = copy(self.loaded_full_list)

        # Clear the last scan result
        self.last_scan_result = {'steps': OrderedDict()}  # the last scan result
        self.logger.debug('Running')
        self.logger.info("Scanner mode activated : {}".format(self.run_mode))
        if not self.scan_list:
            self.logger.info('Scan list is empty')
            return
        else:
            self.logger.info('Scan list size : %s' % len(self.scan_list))

        # ---------- Filters ------------
        if self.filters_and:
            for f in self.filters_and:
                self.scan_list = set(f).intersection(self.scan_list)
            self.logger.info('Filtering and logic : %s' % len(self.scan_list))
        if self.filters_not:
            for f in self.filters_not:
                self.scan_list = list(set(self.scan_list) - set(f))
            self.logger.info('Filtering not logic : %s' % len(self.scan_list))

        # ---------- Condition ----------
        if self.condition_list:

            # Single Thread
            if self.run_mode == 'SingleThread':
                for name, func_tuple in list(self.condition_list.items()):

                    self.logger.info('Scanning cond: {} with {} stocks.'.format(name, self.scan_list()))

                    # condition
                    func        = func_tuple[0]
                    params_dict = func_tuple[1]

                    # loop tracker
                    steps_in_order_dic = self.last_scan_result['steps'][name] = {}  # last dic
                    true_hit_list = []

                    self.scan_list.sort(key= int)
                    for sym in self.scan_list:
                        self.logger.debug('Scanning {}'.format(sym))
                        try:
                            s = self.memory_db[sym]
                            if func(s, **params_dict):
                                true_hit_list.append(sym)
                                steps_in_order_dic.update({sym: True})
                                if name in s.scanner_meta_data:
                                    # If condition carry meta data, update to the key
                                    steps_in_order_dic.update({sym: s.scanner_meta_data[name]})
                        except Exception as e:
                            self.logger.info('Scanner Error : %s, %s' % (sym, str(e)))
                            pass

                    if self.cond_apply_method == 'recursive':
                        # 使用上一層結果
                        self.scan_list = true_hit_list

                # Last scan result
                self.last_scan_result['result'] = true_hit_list

            # Multi Thread
            elif self.run_mode == 'Threading':

                # First put symlist in queue
                self.scan_list.sort(key= int)
                for sym in self.scan_list:
                    self.queue.put(sym)

                def scan(func, params_dict):
                    while not self.queue.empty():
                        sym = self.queue.get()
                        self.logger.debug('Scanning {}'.format(sym))
                        s = self.memory_db[sym]
                        if func(s, **params_dict):
                            true_hit_list.append(sym)
                            steps_in_order_dic.update({sym: True})
                            if name in s.scanner_meta_data:
                                steps_in_order_dic.update({sym: s.scanner_meta_data[name]})

                for name, func_tuple in list(self.condition_list.items()):

                    self.logger.info('Scanning cond: {} with {} stocks.'.format(name, self.queue.qsize()))

                    # condition
                    func = func_tuple[0]
                    params_dict = func_tuple[1]

                    # loop tracker
                    steps_in_order_dic = self.last_scan_result['steps'][name] = {}  # last dic
                    true_hit_list = []

#                    t1 = Thread(target= scan, name='T1', args=(func, params_dict))
#                    t2 = Thread(target= scan, name='T2', args=(func, params_dict))
#                    t3 = Thread(target= scan, name='T3', args=(func, params_dict))
#                    t4 = Thread(target= scan, name='T4', args=(func, params_dict))
#
#                    t1.start()
#                    t2.start()
#                    t3.start()
#                    t4.start()
#
#                    t1.join()
#                    t2.join()
#                    t3.join()
#                    t4.join()

                    NUM_THREADS = 4
                    threads = [Thread(target= scan, args=(func, params_dict)) for i in range(NUM_THREADS)]  # Create 2 threads

                    list([th.start() for th in threads])  # Make all threads start their activities
                    list([th.join() for th in threads])  # block until all threads are terminated

                    if self.cond_apply_method == 'recursive':
                        for sym in true_hit_list:
                            self.queue.put(sym)
                    else:
                        # Put scan list back to queue
                        self.scan_list.sort(key= int)
                        for sym in self.scan_list:
                            self.queue.put(sym)

                # Last scan result
                self.last_scan_result['result'] = true_hit_list

        # ---------- Finish ----------
        # Write to TSCI, so we can view the scan result in tsci
        if self.cond_apply_method == 'recursive':
            ba.sm.tsci_write_symlist_to_file('scanner_result', self.scan_list)
            self.logger.info('>>> Scan complete. Result: %s' % len(self.scan_list))
        else:
            self.logger.info('>>> Scan complete.')
        return self.last_scan_result

    @timeit
    def run_groups_scan(self):
        result = {}
        if self.scan_list_groups:
            for gp_name, gp_symlist in self.scan_list_groups.items():
                self.setsymlist(gp_symlist)
                result[gp_name] = self.run()
        self.last_scan_result = result
        return result

    def result(self):
        # or simply the list(list(self.last_scan_result.items())[-1][1].keys())
        return self.scan_list

    def setsymlist(self, symlist: list):
        self.scan_list = symlist
        self.loaded_full_list = symlist

    def addsymlist(self, symlist: list):

        self.scan_list = list(set(self.scan_list + symlist))
        self.loaded_full_list = copy(self.scan_list)

    def addgroups(self, symlist_groups: dict):
        self.scan_list_groups = symlist_groups

    def addfilter(self, and_or_not, df):
        """
        'and' & df
        'not' & df
        """
        if and_or_not == 'and':
            self.filters_and.append(self.flt.get_the_symlist(df))
        elif and_or_not == 'not':
            self.filters_not.append(self.flt.get_the_symlist(df))

    def addcondition(self, condition, **kwargs):
        self.condition_list[condition.__name__] = (condition, kwargs)
        # self.condition_list.append(s: MemoryDict)

    def make(self, symlist):
        for s in symlist:
            yield s

    def make_queue(self, symlist):

        for sym in symlist:
            queue.put(sym)

        return queue

    def start_scan_hub(self):
        print("Scanner mode activated.")

        for i, scanner in enumerate(self.scanner_list):
            if i == 0:
                self.scan_symlist = self.symlist
            print('Start Scanning the %s criteria in %s stocks: %s' % (i, len(self.scan_symlist), scanner.name))

            scanner.symlist = self.scan_symlist
            result = scanner.start_scan()

            print('There is %s in result' % len(result))
            print()

            self.scan_symlist = result

        self.result = self.scan_symlist

        print('Here is the result: ', len(self.result))

    def today_have_signal(self, algo, symlist='', dayback=0):
        if not symlist:
            symlist = self.fr.getCleanlist()
        print(symlist)
        sig_list = []
        print('Scanning signal...')
        for sym in symlist:
            # print sym
            d = algo(self.tc.get_bar_ex_eod_db(sym))
            # print self.tradeday
            try:
                bool_ = d['Signal'].ix[self.tradeday]
            except KeyError:
                continue

            if bool_:
                sig_list.append(sym)
        print('Done.')
        return sig_list

# ------ Auxiliary related ------
def auxiliary_ground_rule(s: MemoryDict) -> bool:
    """
        9日平均成交額要大於3百萬
        成交量比前日大9倍
        股價必須大於2毫
        股價必須在 ma200 之上
    Args:
        s ():

    Returns:

    """
    daybar = s.daybar[-30:]

    total_turn_day = 9
    if len(daybar) < 9:
        total_turn_day = len(daybar)
    nine_day_turnover = ta.EMA(daybar.Turnover.values, total_turn_day)

    cond1 = nine_day_turnover[-1] >= 3000000  # 9日平均成交額要大於3百萬 或 (IPO) 平均成交大於3百萬
    cond2 = (daybar.Vol.iloc[-1] / daybar.Vol.iloc[-2]) > 9  # 成交量比前日大9倍
    cond3 = daybar.Close.iloc[-1] >= 0.20  # 股價必須大於2毫
    # cond4 = # 股價必須在 ma200 之上

    if (cond1 | cond2) & cond3:
        return True
    return False

def filter_production(s: MemoryDict, target_chgp: float) -> bool:
    """
    升幅％　大於　某個數值
    """
    if s.daybar_chgp > target_chgp:
        return True
    return False

# ------ Momentum related ------
def price_up_vol_up(s: MemoryDict, in_row=1):
    return last_signal_bool(ba.algo.Signal_p_up_vol_up(s.daybar, in_row))

def rainbow(s: MemoryDict, in_row=1):
    return last_signal_bool(ba.algo.Signal_MAHL_x_day_up(s.daybar, in_row))

def moving_average_condition(s: MemoryDict):
    """
    快線 高於 慢線
    """

    def __init__(self, *args):
        self.ma_period = args

    def data(self, sym):
        return ba.tc.get_bar_ex_eod_db(sym)

    def logic(self, sym):
        print('Moving Average (fast > slow) :', sym)
        ma_fast_p = self.ma_period[0][0]['fast_period']
        ma_slow_p = self.ma_period[0][0]['slow_period']

        ba.algo.set_data(self.data(sym))
        ba.algo.cmd_ema(ma_fast_p)
        ba.algo.cmd_ema(ma_slow_p)

        f = ba.algo.data['ema%s' % ma_fast_p].iloc[-1]
        s = ba.algo.data['ema%s' % ma_slow_p].iloc[-1]

        if f > s:
            result.append((sym, f, s))
            return True

def moving_average_golden_cross_condition(s: MemoryDict):
    """
    黃金交叉
    """

    def __init__(self, *args):
        self.ma_period = args

    def data(self, sym):
        return ba.tc.get_bar_ex_eod_db(sym)

    def logic(self, sym):
        print('Moving Average Golden Cross :', sym)
        ma_fast_p = self.ma_period[0][0]['fast_period']
        ma_slow_p = self.ma_period[0][0]['slow_period']

        ba.algo.set_data(self.data(sym))
        ba.algo.cmd_ema(ma_fast_p)
        ba.algo.cmd_ema(ma_slow_p)

        f0 = ba.algo.data['ema%s' % ma_fast_p].iloc[-1]
        s0 = ba.algo.data['ema%s' % ma_slow_p].iloc[-1]

        f1 = ba.algo.data['ema%s' % ma_fast_p].iloc[-2]
        s1 = ba.algo.data['ema%s' % ma_slow_p].iloc[-2]

        if f0 > s0 and f1 < s1:
            result.append((sym, f0, s0))
            return True

def moving_average_death_cross_condition(s: MemoryDict):
    """
    死亡交叉
    """

    def __init__(self, *args):
        self.ma_period = args

    def data(self, sym):
        return ba.tc.get_bar_ex_eod_db(sym)

    def logic(self, sym):
        print('Moving Average Death Cross :', sym)
        ma_fast_p = self.ma_period[0][0]['fast_period']
        ma_slow_p = self.ma_period[0][0]['slow_period']

        ba.algo.set_data(self.data(sym))
        ba.algo.cmd_ema(ma_fast_p)
        ba.algo.cmd_ema(ma_slow_p)

        f0 = ba.algo.data['ema%s' % ma_fast_p].iloc[-1]
        s0 = ba.algo.data['ema%s' % ma_slow_p].iloc[-1]

        f1 = ba.algo.data['ema%s' % ma_fast_p].iloc[-2]
        s1 = ba.algo.data['ema%s' % ma_slow_p].iloc[-2]

        if f0 < s0 and f1 > s1:
            result.append((sym, f0, s0))
            return True

def triple_ma_blazer(s: MemoryDict, *args) -> bool:
    """
    兩 set ma 交叉
    """
    g1 = (10, 20, 50)
    match = False
    d = ba.typical_ema_groups(s.daybar)
    if d['ema10'].iloc[-1] > d['ema10'].iloc[-2] > d['ema10'].iloc[-3]:
        if d['ema20'].iloc[-1] > d['ema20'].iloc[-2] > d['ema20'].iloc[-3]:
            if d['ema50'].iloc[-1] > d['ema50'].iloc[-2] > d['ema50'].iloc[-3]:
                match = True
    return match

# ------ Demand related ------
def sudden_vol_blowing(s: MemoryDict):
    """
    # 今日放量上升 (無消息, 無故大升)
    今日升 加上 放量倍數
    但是成交額都要跟得上, 起碼要1, 2千萬
    """

    d = s.daybar
    ratio = d.Vol.iat[-1] / d.Vol.iat[-2]
    match = False
    if ratio > self.times:
        try:
            turnover = ba.etnet.get_by_sym_sql(sym, exact_date=ba.dk.tradeday())['Turnover'].values[0]
            if turnover >= self.turnover:
                result.append((sym, ratio))
                match = True
        except:
            pass
    return match

def high_volatility_stock(s: MemoryDict):
    """
    ATR 對比 Close 大於 0.05 及
    ATR 對比 每格跳動 大於 30
    """

    match = False

    ba.algo.set_data(s.daybar)
    ba.algo.cmd_atr(10)
    d = ba.algo.data
    d['p'] = d.ATR / d.Close

    from BATrader.market.MarketReader import pricespread

    if d.p.iloc[-1] >= 0.05:
        if d.ATR.iloc[-1] / pricespread(d.Close.iloc[-1]) > 30:
            s.scanner_meta_data[inspect.stack()[0][3]] = (d.ATR.iloc[-1],  # ATR
             d.ATR.iloc[-1] / pricespread(d.Close.iloc[-1]),  # ATR / tick
             d.p.iloc[-1])  # ATR / Close]
        match = True

    return match

def unknown_func_last(s: MemoryDict):
    # Sector ond day momentum index

    sector = ba.etnet.etnet_concept_dic['內房股']
    # house_list = sm.fr.which_group(sector)

    etnet.get_by_sym_sql('1313', col_lst=['PE'])
    dic = etnet.etnet_concept_dic

    for sector, symlist in list(dic.items()):
        print(sector)

    #        path = self.path_input + 'sector.txt'
    #        with open(path,'a') as f:
    #            f.write(msg  + '\n')

    # 1 loops, best of 3: 37 s per loop
    ignore_sector = ['農業股']

    dayback = 1

    result_dic = {}
    df_dic = {}
    single_stock_df = pd.DataFrame()
    for chi_name, eng_name in list(etnet.etnet_concept_dic.items()):
        # print eng_name
        group = fr.which_group(eng_name)
        print(chi_name, eng_name, group)

        # Lv2 direct 做法
        #     d = rtq.get_aa_bunch_df(group, detail='lv2', col = ['Close','pClose'])
        #     d.Close = d.Close.astype('float')
        #     d.pClose = d.pClose.astype('float')

        # Lv1 + DB 做法
        d = rtq.get_aa_bunch_df(group, detail='lv1', col=['Close'])
        t = tc.get_bar_ex_eod_db(group, date=dk.tradeday(dayback), col=['Close'])
        t = t.reset_index().set_index('sym')[['Close']]
        t.columns = ['pClose']
        d = pd.concat([d, t], axis=1, sort=False)
        d.Close = d.Close.astype('float')
        d = d.dropna()
        df_dic[eng_name] = d

        d['SignleStockChgp'] = d.Close / d.pClose
        single_stock_df = single_stock_df.append(d)
        result_dic[chi_name] = d.Close.sum() / d.pClose.sum()

    r = {}
    for group, value in list(result_dic.items()):
        r[group] = (value - 1) * 100

    top20 = single_stock_df.sort_values('SignleStockChgp', ascending=False)[:20]

    df = pd.DataFrame(list(r.items()), columns=['Group', 'Value'])
    df.set_index('Group')
    top20_sector = pd.concat(
        [df.sort_values('Value', ascending=False)[:10], df.sort_values('Value', ascending=False)[-10:]], sort=False)

    print(top20)
    print(top20_sector)

    single_stock_df.sort_values('SignleStockChgp', ascending=False)[:20]

    for chi_name, eng_name in list(etnet.etnet_concept_dic.items()):
        # print eng_name
        group = fr.which_group(eng_name)
        print(eng_name, group)
        d = rtq.get_aa_bunch_df(group, detail='lv1', col=['Close'])
        t = tc.get_bar_ex_eod_db(group, date='20170524', col='Close')
        t = t.reset_index()
        t = t.set_index('sym')[['Close']]
        t.columns = ['pClose']
    d = pd.concat([d, t], axis=1, sort=False)

    sym = 2018

    from etnet import Etnet
    etnet = Etnet()
    etnet.get_tick(str(sym))[-20:]

def tick_is_over_5day_average(s: MemoryDict):
    def logic(self, sym):
        print('Tick is over 5days Average :', sym)
        df = ba.etnet.get_by_sym_sql(sym, col_lst=['tick'])

        import talib as ta
        df['ma5'] = ta.EMA(df['tick'].astype('float').values, 5)

        if df.ma5.iloc[-1] > df.ma5.iloc[-2]:
            return True

def inactive_and_thin(s: MemoryDict):
    """
    if last 5 days average trading amount is less than 1m, eliminate
    col name : inactive_or_thin
    """
    s.load_details(col_lst = ['tick','Turnover'], dayback= 11)
    data = s.details
    win = 5

    data['AvgT'] = data.Turnover.apply(lambda x: convert(x))
    data['AvgTick'] = data['tick'].apply(lambda x: convert(x)).rolling(window= win).mean()

    # if average turnover less than 1M and average Tick is less than 100 , kick out
    if data.AvgT[-1] <= 1000000 or data.AvgTick[-1] <= 100:
        return True
    return False

def low_liquidity(s: MemoryDict):
    """
    一隻股票在最新一天入面, 有超過5分鐘都冇成交, 代表liqudity低下, 或可剔除
    """
    s.load_min_bar('5') # It's simple mode
    lastday = s.data5min.reset_index()["Datetime"].map(lambda t: t.date().strftime('%Y%m%d')).unique()[-1]
    # ba.algo.make_min_bar(s.data1min.last('1D'), '5Min')

    # 壓縮成五分鐘燭 resample to 5 mins, 去掉唔要的時段
    data5m = s.data5min[lastday]

    # morning_session = data5m.between_time(start_time='09:30:00', end_time='11:59:00', include_end=True)
    # afternoon_session = data5m.between_time(start_time='13:00:00', end_time='15:59:00', include_end=True)
    # data5m = morning_session.append(afternoon_session)

    # 超過5分鐘都冇成交, 代表liqudity低下, 或可剔除
    # Adding half day filter : afternoon bar equal all zero bar
    #if len(afternoon_session) == len(afternoon_session.loc[afternoon_session.Vol == 0.0]):
        #if len(morning_session.loc[morning_session.Vol == 0.0]) > 2:
            #return True

    if len(data5m.loc[data5m.Vol == 0.0]) > 2:
        return True
    return False

# ------ Intraday related ------
def five_minutes_three_in_row(s: MemoryDict) -> bool:
    """
    Signal_3_in_row
    Args:
        s ():

    Returns:

    """
    if int(s.data1min.index[-1].strftime('%M')) % 5 != 0:
        return False

    s.load_min_bar('5')

    if last_signal_bool(ba.algo.Signal_3_in_row(s.data5min)):
        return True

    return False

def thirty_minues_blow_bbands(s: MemoryDict) -> bool:

    if int(s.data1min.index[-1].strftime('%M')) % 30 != 0:
        return False

    #ba.algo.set_data(ba.algo.make_min_bar(s.data1min, '30Min'))
    s.load_min_bar('30')
    if last_signal_bool(ba.algo.Signal_Blow_BBands(s.data30min)):
        return True

    return False

# ------ Swing Trade related ------
def is_today_t3b(s: MemoryDict):
    """
    # T3B2
    ba.algo.Signal_T3B2()
    generate_algo_msg('T3B2', ba.algo.data['Signal_T3B2'])
    """
    return last_signal_bool(ba.algo.Signal_T3B2(s.daybar))

def is_today_break_atr(s: MemoryDict):
    """
    # BreakATR
    ba.algo.Signal_BreakATR()
    generate_algo_msg('BreakATR', ba.algo.data['Signal_BreakATR'])

    """
    return last_signal_bool(ba.algo.Signal_BreakATR(s.daybar))

def is_today_over_ma_60_200(s: MemoryDict):
    """
    # >ma60/200
    ba.algo.Signal_Penetrate_ma60_ma200()
    generate_algo_msg('>ma60/200', ba.algo.data['Signal_penetrate_ma60_ma200'])
    """
    return last_signal_bool(ba.algo.Signal_Penetrate_ma60_ma200(s.daybar))

def is_today_break_bb_daybar(s: MemoryDict):
    """
    # breakBB_daybar
    ba.algo.Signal_Blow_BBands()
    generate_algo_msg('blowBB_daybar', ba.algo.data['Signal_Blow_BBands'])
    """
    return last_signal_bool(ba.algo.Signal_Blow_BBands(s.daybar))

# ------ Week related ------
def last_week_down_x_pct(s: MemoryDict, pct):
    """
    一個星期完結, 跌到 X % 以上
    燭身不能是長下影線
    """
    d = ba.algo.make_week_bar(s.daybar)
    d = chgp(d)

    if d['Chgp'].iloc[-1] < -pct:
        s.scanner_meta_data[inspect.stack()[0][3]] = d['Chgp'].iloc[-1]
        return True
    return False

def last_week_up_x_pct(s: MemoryDict, pct):
    """
    一個星期完結, 升到 X % 以上
    燭身不能是長上影線
    """
    d = ba.algo.make_week_bar(s.daybar)
    d = chgp(d)

    from BATrader.algo.algo import candle_wick

    d = candle_wick(d)

    if d['Chgp'].iloc[-1] > pct:
        if d.Wick.iloc[-1] != 1:
            s.scanner_meta_data[inspect.stack()[0][3]] = d['Chgp'].iloc[-1]
            return True
    return False

def scan_stock_relative_strong_weak():
    """
    # 大市走資比例, 如大市
    # 穿平均線比例 ( 10 , 20 , 60)
    """
    df[df.pClose > df.ma60]

# ------ Fundamental related ------
def pe_is_below(s: MemoryDict, pe=25):
    s.get_pe(s.sym)
    match = False
    if s.PE < pe:
        match = True
    return match

def ccass_trade_reach_certain_level_of_total_shares_condition(s: MemoryDict):
    def data(self, sym):
        day = ba.dk.tradeday()
        print(day)
        dic = ba.ccass.CCASS_flower_paper(sym, day)
        vol = ba.tc.get_bar_ex_eod_db(sym).loc[day]['Vol']  # e.g. 1853028.0
        dic.update({'vol': vol})
        return dic

    def logic(self, sym):
        print('CCASS condition for :', sym)
        d = self.data(sym)

        cond1 = d['top10%'] > 80  # 集中度大於80
        cond2 = divided_and_show_pct(d['total_in_ccass'], d['total_issued_shares']) < 30  # 在CCASS流通的股數非常少
        cond3 = divided_and_show_pct(d['vol'], d['total_in_ccass']) > 3  # 當日交易股數非常多

        if cond1 or cond2 or cond3:
            result.append((sym, d['top10%'],
                           divided_and_show_pct(d['total_in_ccass'], d['total_issued_shares']),
                           divided_and_show_pct(d['vol'], d['total_in_ccass'])))
            return True

def is_today_hkexnews_has_news(s: MemoryDict):
    s.load_hkexnews()

# ------ MMI related ------
def typical_mmi(s: MemoryDict):
    """
    (成交金融必須大於5百萬) 和 (Vol大於pVol 5倍) 和 ( A盤大於昨日A盤1.2倍 / 主動買入大於60 )
    ( A >= 55 和 A大過昨日 ) 和 (量 佔 昨天量比大於0.7) 和 (額 佔昨天額比大於0.7) 和 ( 額大於500萬 ) 或
    (額 大於 昨天額 2倍) 和 (A大於60) 和 (額大於9百萬) 或
    (額 大於 昨天額 6倍) 和 (額連續三日上升)

    (量 額 A 都大過昨日) 或
    (( d.Vol > d.Vol.shift(1)) & (d.Amount > d.Amount.shift(1)) & (d.A > d.A.shift(1))) |\
    """
    s.load_mmi(dayback= 10)
    return last_signal_bool(ba.algo.Signal_mmi(s.mmi))

if __name__ == '__main__':
    flt = Filters()
    #    sym = '2331'
    #    md = MemoryDict()
    #    s = md[sym]
    #    match = typical_mmi(s)

    #    self.run_mode = {0: 'SingleThread', 1: 'Threading'}.get(running_mode)
    #    self.cond_apply_method = {0: 'recursive', 1: 'independent'}  # Recursive is one-and-one

    scanner = Scanner(1, 1, dayback=30)

    #    scanner.addgroups({'內銀股': ['939','1398','2628','3988']})
    #    scanner.addcondition(last_week_up_x_pct, pct=5)
    #    scanner.addcondition(last_week_down_x_pct, pct=5)
    #    scanner.run_groups_scan()

    scanner.addsymlist(ba.fr.get_main_gem())
#    scanner.addfilter('and', filter_by_over_ma_60())
#    scanner.addfilter('and', flt.filter_by_amount(10000000.0))
#    scanner.addfilter('and', flt.filter_by_over_ma(60))
#    scanner.addfilter('not', filter_ot_list())
    scanner.addcondition(inactive_and_thin)
    scanner.addcondition(low_liquidity)
    result = scanner.run()


# Modules
from cs50 import get_int

# Condition for getting the input from the user
while True:
    answer = get_int("please, input the height desired between 1 and 8: ")
    if 1 <= answer <= 8:
        break

# Loop for print the pyramid
for n in range(0, answer):
    n += 1
    print(" " * (answer - n), end="")
    print("#" * n, end="")
    print()


from __future__ import print_function

import protos.grpc_service_pb2_grpc as grpc_service_pb2_grpc
import protos.server_status_pb2 as server_status_pb2
import protos.model_config_pb2 as model_config_pb2
import protos.request_status_pb2 as request_status_pb2

import grpc
import multiprocessing as mp
from concurrent import futures
import itertools

import argparse
parser = argparse.ArgumentParser(description='Xilin ML Suit gRPC Inference Proxy')
parser.add_argument("--address", metavar="<address>", type=str,
                    help="Server address to forward to", default="localhost")
parser.add_argument("-p", "--ports", metavar="<port>", type=int, default=[5001], nargs="+",
                    help="Server ports to forward to")
parser.add_argument("-l", "--listen", metavar="<port>", type=int, default=5000,
                    help="Server ports to listen on")
args = parser.parse_args()

ADDRESSES = ["{address}:{port}".format(port=port, address=args.address)
             for port in args.ports]
PORT = args.listen

def iterator_split(iterator, size):
    iterators = itertools.tee(iterator, size)
    result = []
    for i, it in enumerate(iterators):
        result.append(itertools.islice(it, i, None, size))
    return result

def roundrobin(*iterables):
    "roundrobin('ABC', 'D', 'EF') --> A D E B F C"
    # Recipe credited to George Sakkis
    pending = len(iterables)
    nexts = itertools.cycle(iter(it).next for it in iterables)
    while pending:
        try:
            for next in nexts:
                yield next()
        except StopIteration:
            pending -= 1
            nexts = itertools.cycle(itertools.islice(nexts, pending))

class InferenceServicer(grpc_service_pb2_grpc.GRPCServiceServicer):
    '''
    A proxy server that distributes requests
    '''
    def __init__(self, addresses):
        self.addresses = addresses
        self.next_address = 0

    def Status(self, request, context):
        master = self.addresses[0]
        with grpc.insecure_channel(master) as channel:
            stub = grpc_service_pb2_grpc.GRPCServiceStub(channel)
            return stub.Status(request)

    def Infer(self, request, context):
        master = self.addresses[self.next_address % len(self.addresses)]
        self.next_address += 1
        if self.next_address > 1000000:
            self.next_address = 0
        with grpc.insecure_channel(master) as channel:
            stub = grpc_service_pb2_grpc.GRPCServiceStub(channel)
            return stub.Infer(request)

    def StreamInfer(self, request_iterator, context):
        # request iterators
        request_its = iterator_split(request_iterator, len(self.addresses))
        channels = [grpc.insecure_channel(address)
                    for address in self.addresses]
        try:
            results = [grpc_service_pb2_grpc.GRPCServiceStub(channel).StreamInfer(it)
                       for channel, it in zip(channels, request_its)]
            for response in roundrobin(*results):
                yield response
        finally:
            for channel in channels:
                channel.close()

def main():
    print("Starting proxy on port", PORT)
    print("Addresses:", ADDRESSES)
    executor = futures.ThreadPoolExecutor(max_workers=64)
    executor.shutdown = lambda wait: None
    server = grpc.server(executor,
                         options=(('grpc.max_receive_message_length', 64*1024*1024),))
    servicer = InferenceServicer(ADDRESSES)
    grpc_service_pb2_grpc.add_GRPCServiceServicer_to_server(servicer,
                                                              server)

    # Bind port
    server.add_insecure_port('[::]:{port}'.format(port=PORT))

    # Start
    server.start()
    print("Server initialized")
    try:
        server.wait_for_termination()
    except KeyboardInterrupt:
        # Try to stop all threads
        print("Exiting due to keyboard interrupt")
        exit(0)

main()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Oct 10 22:38:25 2019

@author: xinlu
"""
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Oct 11 16:09:00 2019

@author: xinlu
"""

import string
from collections import Counter
import csv
import nltk
from nltk.tokenize import casual_tokenize
from nltk.util import ngrams
from nltk.util import bigrams
from nltk.lm import NgramCounter
from nltk.corpus import stopwords
import matplotlib.pyplot as plt
import re
import collections
import pandas as pd
import numpy as np
from nltk.corpus import stopwords
from nltk.sentiment.vader import SentimentIntensityAnalyzer
from nltk.classify import NaiveBayesClassifier
from sklearn import metrics
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report

DATA = 'INPUT.txt'

punctuation = list(string.punctuation) + ['…', '...','’']
stop_words = stopwords.words('english') + ['rt', 'via']

# The total number of distinct words (vocabulary)
# The total number of tokens corresponding to the top 10 most frequent words (types) in the vocabulary
def totals(text):
    token_count = 0
    counts = Counter(text)
    for key, val in counts.most_common(10):
        token_count += val
    return len(counts), counts.most_common(10), token_count

# The number of words, characters and av char count for each tweet
def tweet_counts(tweet):
    char_counts = [len(word) for word in tweet]
    total_chars = sum(char_counts)
    av_chars = total_chars / len(char_counts)
    return len(tweet), total_chars, av_chars

# calculate standard deviation of characters per token in a tweet
def standard_deviation(text):
    char_count = []
    words = text.split(' ')
    for word in words:
        char_count.append(len(word))

    diffs = 0
    average = sum(char_count)/len(char_count)
    for n in char_count:
        diffs += (n - average)**(2)
    return (diffs/(len(char_count)-1))**(0.5)

# The average number and standard deviation of characters per token
# The total number of characters
def token_counts(tweet_list):
    # sum the values with same keys for char_count and av_char_count
    char_counts = []
    word_counts = []

    for tweet in tweet_list:
        char_counts.append(tweet['char_count'])
        word_counts.append(tweet['word_count'])

    sd = standard_deviation(tweet['text'])
    total_chars = sum(char_counts)
    total_words = sum(word_counts)
    av_char_count = total_chars / total_words
    return total_chars, av_char_count, sd

# debugging
def write_counter_to_file(counters):
    filenames = ['bigrams','trigrams','fourgrams','fivegrams']
    num = 0
    for counter in counters:
        filename = filenames[num]
        with open(filename + '.csv', 'w') as csvfile:
            writer = csv.writer(csvfile)
            for key, value in counter.items():
                writer.writerow(list(key) + [value])
            num = num+1

# The total number of distinct n-grams (of words) that appear in the dataset for n=2,3
# You can use sets or Counters pretty easily
# https://dbader.org/blog/sets-and-multiset-in-python
# Check http://www.nltk.org/api/nltk.html#nltk.util.bigrams
# This also looks promising
# http://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html
def ngram_counts(tweet_list):
    bigram_counter = Counter()
    trigram_counter = Counter()
    fourgram_counter = Counter()
    fivegram_counter = Counter()
    list_counters = []

    for tweet in tweet_list:
        tokens = tweet['text'].split()
        bigrams = list(nltk.ngrams(tokens, 2))
        trigrams = list(nltk.ngrams(tokens, 3))
        fourgrams = list(nltk.ngrams(tokens, 4))
        fivegrams = list(nltk.ngrams(tokens, 5))
        bigram_counter.update(bigrams)
        trigram_counter.update(trigrams)
        fourgram_counter.update(fourgrams)
        fivegram_counter.update(fivegrams)
    distinct_bigram_counter = len(bigram_counter) # distinct counts
    distinct_trigram_counter = len(trigram_counter) # distinct counts
    distinct_fourgram_counter = len(fourgram_counter) # distinct counts
    distinct_fivegram_counter = len(fivegram_counter) # distinct counts

    # debug... trigrams count slightly higher??
    list_counters = [bigram_counter, trigram_counter, fourgram_counter, fivegram_counter]
    write_counter_to_file(list_counters)

    #total_counter = sum(ngram_counter.values()) # total counts
    return distinct_bigram_counter, distinct_trigram_counter, distinct_fourgram_counter, distinct_fivegram_counter

# Plot a token log frequency. Describe what this plot means and how to interpret it.
# A useful way to do this is by plotting log-frequency against the log-rank
# You could also plot Heap's law - types versus tokens
# You could also do a log frequency graph of the top n tokens
# Describe out it might help you understand coverage when training a model?

def token_log_freq(corpus):
    corpus_counts = Counter(corpus)
    plt.loglog([val for word, val in corpus_counts.most_common(1000)])
    plt.xlabel('rank')
    plt.ylabel('frequency')
    #plt.show()
    return

with open(DATA, 'r') as data:
    tweet_list = []
    corpus = []
    for line in data:
        #print(line.split('\t'))

        info = line.strip().split('\t')
        id, label, text = info[0], info[1], ' '.join(info[2:])

        #tokens = tokenize(text)
        tokens = casual_tokenize(text)
        tokens = [token.lower() for token in tokens]
        tokens = [token for token in tokens if not token in stop_words]
        tokens = [token for token in tokens if not token in punctuation]
        #tokens = [tokens for token in tokens if not token.isdigit()]

        corpus.extend(tokens) # one giant list
        word_count, char_count, av_char_count = tweet_counts(tokens)

        row = {
            'id': id,
            'label': label,
            'text': ' '.join(tokens),
            'word_count': word_count,
            'char_count': char_count,
        }
        tweet_list.append(row)

        total_tweets = len(tweet_list)

total_chars, av_char_count, sd = token_counts(tweet_list)

####################################Answers to questions#############################
print("total tweets = ", total_tweets)
print("total tokens = ", len(corpus))
print("total number of characters = ", total_chars)
print("av chars per token = ", av_char_count)
print("sd for chars per token = ", sd)

vocab, most_common, tokens_common = totals(corpus)

print("vocabulary size = ", vocab)
print("10 most common words = ", most_common)
print("token count of 10 most common words = ", tokens_common)

# The token/type ratio in the dataset
# Type-Token Ratio can be obtained by dividing the total type count by the total token count.
# The basic idea is that the higher the number, the more lexically diverse
print("token/type ratio = ", vocab / len(corpus))

bigram_counts, trigram_counts, fourgram_counts, fivegram_counts = ngram_counts(tweet_list)

print("unigram count = ", len(corpus))
print("bigram count = ", bigram_counts)
print("trigram count = ", trigram_counts)
print("four-gram count = ", fourgram_counts)
print("five-gram count = ", fivegram_counts)

#- The total number of distinct n-grams (of words) that appear in the dataset for n=2,3,4,5.
with open('INPUT.txt') as f:
    df = f.read()

##remove the user id 80xxx (first 18 digits) from each line
df = re.sub(r"(?<!\d)\d{18}(?!\d)","",df)
#remove url
def remove_url(txt):
    """Replace URLs found in a text string with nothing
    (i.e. it will remove the URL from the string).

    Parameters
    ----------
    txt : string
        A text string that you want to parse and remove urls.

    Returns
    -------
    The same txt string with url's removed.
    """
    return " ".join(re.sub("([^0-9A-Za-z \t])|(\w+:\/\/\S+)", " ", txt).split())

df1 = [remove_url(df)]
df2 = str(df1).split()
df3 = [word.lower() for word in df2]

stop_words = set(stopwords.words('english'))
filtered_sentence = [w for w in df3 if not w in stop_words]
list(stop_words)[0:20]
filtered_sentence = []

for w in df3:
    if w not in stop_words:
        filtered_sentence.append(w)
df9 = filtered_sentence

#n=5
s1 = re.sub(r'[^a-zA-Z0-9\s]', ' ', str(df9))
tokens1 = [token for token in s1.split(" ") if token != ""]
output1 = list(ngrams(tokens, 5))
ss1 = collections.Counter(output1)
ss1 = pd.DataFrame(ss1.most_common(),columns = ['ngrams','count'])
ss1

#n=4
s2 = re.sub(r'[^a-zA-Z0-9\s]', ' ', str(df9))
tokens2 = [token for token in s2.split(" ") if token != ""]
output2 = list(ngrams(tokens2, 4))
ss2 = collections.Counter(output2)
ss2 = pd.DataFrame(ss2.most_common(),columns = ['ngrams','count'])
ss2

#n=3
s3 = re.sub(r'[^a-zA-Z0-9\s]', ' ', str(df9))
tokens3 = [token for token in s3.split(" ") if token != ""]
output3 = list(ngrams(tokens3, 3))
ss3 = collections.Counter(output3)
ss3 = pd.DataFrame(ss3.most_common(),columns = ['ngrams','count'])
ss3

#n=2
s4 = re.sub(r'[^a-zA-Z0-9\s]', ' ', str(df9))
tokens4 = [token for token in s4.split(" ") if token != ""]
output4 = list(ngrams(tokens4, 2))
ss4 = collections.Counter(output4)
ss4 = pd.DataFrame(ss4.most_common(),columns = ['ngrams','count'])
ss4

# If the graph approximately accurate, it should look linear
# The idea here is that word count distributions follow a power law
# According to Zipf's law, the frequency of any word is inversely proportional
# to its rank in the frequency table.
token_log_freq(corpus)

######################################################################################
#############################Gold (Gold directory)####################################
######################################################################################

#What is the number of types that appear in the dev data but not the training data (OOV).?
#[Hint: compare vocabulary of all the words across your GOLD files with your INPUT.txt.
#Out-of-vocabulary words are those that appear in INPUT.txt that are not in your training data.]
with open('Gold/dev.txt') as dev:
    devdata = dev.read()
with open('Gold/train.txt') as train:
    traindata = train.read()
with open('Gold/devtest.txt') as devtest:
    devtest = devtest.read()
with open('Gold/test.txt') as test:
    test = test.read()

##remove the user id 80xxx (first 18 digits) from each line
devdata = re.sub(r"(?<!\d)\d{18}(?!\d)","",devdata)
traindata = re.sub(r"(?<!\d)\d{18}(?!\d)","",traindata)
devtest = re.sub(r"(?<!\d)\d{18}(?!\d)","",devtest)
test = re.sub(r"(?<!\d)\d{18}(?!\d)","",test)

#remove url
devdata = str([remove_url(devdata)]).split()
traindata = str([remove_url(traindata)]).split()
devtest = str([remove_url(devtest)]).split()
test = str([remove_url(test)]).split()

#combine 4 gold data sets

filenames = ['dev.txt', 'train.txt', 'devtest.txt','test.txt']
with open('Gold/golddata.txt', 'w') as outfile:
    for fname in filenames:
        with open(fname) as infile:
            outfile.write(infile.read())

with open('Gold/golddata.txt') as golddata:
    golddata = golddata.read()

#find types appear in the dev data but not the training data (OOV)
def notMatches(a,b):
    return [x for x in a if x not in b]

devNotTraining = pd.DataFrame(set(notMatches(devdata,traindata)), columns=['Types'])
print("The number of types that appear in the dev data but not the training data (OOV)")
print(len(devNotTraining))

#Look at the vocabulary growth (types) combining your four gold data sets against your input data.
#Plot vocabulary growth at difference sample sizes N. [Hint:The idea here is to take a value
#N around some percentage of your data such as 25%, 50%, 75%, 100% and look at your vocabulary size.]
##Combine all the input.

#remove nonsense words
golddata1 = str(golddata).split()
golddata2 = [word.lower() for word in golddata1]

N_tweet = len(golddata)
print('The total number of distinct words in gold dataset:', N_tweet)

#different sample sizes
N_tweet25 = round(.25 * N_tweet)
N_tweet50 = round(.5 * N_tweet)
N_tweet75 = round(.75 * N_tweet)

sample25 = golddata.splitlines()[0:N_tweet25]
sample50 = golddata.splitlines()[0:N_tweet50]
sample75 = golddata.splitlines()[0:N_tweet75]

#get list of words in different sample sizes
line25 = str(sample25).split()
word25 = [word.lower() for word in line25]

line50 = str(sample50).split()
word50 = [word.lower() for word in line50]

line75 = str(sample75).split()
word75 = [word.lower() for word in line75]

samplesizes = ('size = 25%', 'size = 50%', 'size = 75%', 'size = 100%')
y_pos = np.arange(len(samplesizes))
performance = [len(word25), len(word50), len(word75), len(golddata2)]

##please run the following 5 rows of code separately with parts of code of other questions
#otherwise graph will be crunched and looks weird
plt.bar(y_pos, performance, align='center', alpha=0.5)
plt.xticks(y_pos, samplesizes)
plt.ylabel('Vocab Size')
plt.title('vocabulary growth at difference sample sizes')

plt.show()

#What is the class distribution of the training data set - how many negative, neutral, positive tweets?
df_train = pd.read_csv('train.txt', sep = '\t', header = None)
df_train.columns = ['id', 'label', 'text']
df_train

df_pos_train = df_train[df_train['label'] == 'positive']
pos_tweets = df_pos_train['text'].tolist()

df_neg_train = df_train[df_train['label'] == 'negative']
neg_tweets = df_neg_train['text'].tolist()

df_neutral_train = df_train[df_train['label'] == 'neutral']
neutral_tweets = df_neutral_train['text'].tolist()

print('Class distribution of the training data set:')
print('Positive: ', len(pos_tweets))
print('Negative: ', len(neg_tweets))
print('Neutral: ', len(neutral_tweets))

#Look at the difference between the top word types across these three classes. [Recall these are your "unique words"
#- not tokens]
poswords1 = str(pos_tweets).split()
poswords2 = [word.lower() for word in poswords1]

stop_words = set(stopwords.words('english'))
filtered_sentence = [w for w in poswords2 if not w in stop_words]
list(stop_words)[0:20]

filtered_sentence = []

for w in poswords2:
    if w not in stop_words:
        filtered_sentence.append(w)
poswords3 = filtered_sentence
poswords4 = collections.Counter(poswords3)
poswords4.most_common(10)
poswords = pd.DataFrame(poswords4.most_common(10), columns=['words', 'count'])

negwords1 = str(neg_tweets).split()
negwords2 = [word.lower() for word in negwords1]

filtered_sentence2 = [w for w in negwords2 if not w in stop_words]

filtered_sentence2 = []

for w in negwords2:
    if w not in stop_words:
        filtered_sentence.append(w)
negwords3 = filtered_sentence
negwords4 = collections.Counter(negwords3)
negwords4.most_common(10)
negwords = pd.DataFrame(negwords4.most_common(10), columns=['words', 'count'])

neuwords1 = str(neutral_tweets).split()
neuwords2 = [word.lower() for word in neuwords1]

filtered_sentence3 = [w for w in neuwords2 if not w in stop_words]

filtered_sentence3 = []

for w in neuwords2:
    if w not in stop_words:
        filtered_sentence.append(w)
neuwords3 = filtered_sentence
neuwords4 = collections.Counter(neuwords3)
neuwords4.most_common(10)
neuwords = pd.DataFrame(neuwords4.most_common(10), columns=['words', 'count'])
neuwords

print('The top word types in positive tweets:')
print(poswords)

print('The top word types in negative tweets:')
print(negwords)

print('The top word types in neutral tweets:')
print(neuwords)
##Answer: there is not much difference between the top word types across these three classes

#What words are particularly characteristic of your training set and dev set? Are they the same?
#[Hint: we have not actually learned this concept yet -- just stick with most common words unless
#you've already read ahead and used something like PMI.]

df2 = str(traindata).split()
df3 = [word.lower() for word in df2]
df4 = collections.Counter(df3)
trainCommon = pd.DataFrame(df4.most_common(10), columns=['words', 'count'])

print('Most common words in the training data set:')
print(trainCommon)

df22 = str(devdata).split()
df33 = [word.lower() for word in df22]
df44 = collections.Counter(df33)
devCommon = pd.DataFrame(df44.most_common(10), columns=['words', 'count'])

print('Most common words in the dev data set:')
print(devCommon)

#The top 10 most common words of training set and dev set are exactly the same.

#######Part2
df_train = open('train.txt').readlines()
df_input = open('INPUT.txt').readlines()

# lowercase all the words and remove url
stop_words = set(stopwords.words('english'))
def preprocess_tweet(tweet):
    words = tweet.split(' ')
    temp = []
    for word in words:
        s = word.lower()
        if "http" in s or s in stop_words:
            continue
        else:
            temp.append(s)
    return " ".join(temp)

def split_files(file):
    split = []
    for line in file:
        s = line.split('\t',2)
        split.append((preprocess_tweet(s[2]),s[1]))
    return split

def features(sentence):
    words = sentence.lower().split()
    return dict(('contains(%s)' % w, True) for w in words)

df_train = split_files(df_train)
df_train

input_paper = []
input_tweets = []
num_of_tweets = 0
for line in df_input:
    num_of_tweets = num_of_tweets+1
    input_id = line.split('\t',2)[0]
    s = line.split('\t',2)[2]
    input_paper.append((input_id,preprocess_tweet(s)))
    input_tweets.append(preprocess_tweet(s))
#using vader
input_vader_ans = []

analyzer = SentimentIntensityAnalyzer()
#sentences = input_tweets
for sentence in input_tweets:
    vs = analyzer.polarity_scores(sentence)
    if vs['compound'] >= 0.10:
        input_vader_ans.append("positive")
    elif vs['compound'] <= - 0.05:
        input_vader_ans.append("negative")
    else :
        input_vader_ans.append("neutral")

#using naive bayes
classify=nltk.classify.apply_features(features,df_train)
trainer=NaiveBayesClassifier.train(classify)

input_list = [features(t) for t in input_tweets]
input_nb_ans = [trainer.classify(t) for t in input_list]

ids = [tw[0] for tw in input_paper]

df = pd.DataFrame(list(zip(ids, input_nb_ans, input_vader_ans)),
               columns =['ID', 'NB_ans', 'VD_ans'])
df.head(20)

df.to_csv('answer.csv')

######Part3
df_t = open('test.txt').readlines()
df_t = split_files(df_t)
input_tweets = [tw[0] for tw in df_t]
input_vader_ans1 = []
for sentence in input_tweets:
    vs = analyzer.polarity_scores(sentence)
    if vs['compound'] >= 0.10:
        input_vader_ans.append("positive")
    elif vs['compound'] <= - 0.05:
        input_vader_ans.append("negative")
    else :
        input_vader_ans.append("neutral")

classify=nltk.classify.apply_features(features,df_t)
trainer=NaiveBayesClassifier.train(classify)

input_list = [features(t) for t in input_tweets]
input_nb_ans = [trainer.classify(t) for t in input_list]

df = pd.DataFrame(list(zip(input_nb_ans, input_vader_ans)),
               columns =['NB_ans', 'VD_ans'])

df.head(20)

df_test = pd.read_csv('test.txt', sep = '\t', header = None)
df_test.columns = ['id', 'label', 'text','1']
NBlabels = df.iloc[:, 0].values
VDlabels = df.iloc[:, 1].values
TRlabels = df_test['label']
Text = df_test['text']

NBlabels = list(NBlabels)
VDlabels = list(VDlabels)
TRlabels = list(TRlabels)
Text = list(Text)
df1 = pd.DataFrame(zip(NBlabels,TRlabels, Text),
                   columns = ['NBlabels','TRlabels','Text'])
print(df1.loc[[11]].values)

### F1 for Vader model
confusion_matrix(y_true=TRlabels, y_pred=VDlabels)
target_names = ['positive', 'neutral', 'negative']
score = metrics.precision_recall_fscore_support(y_true=TRlabels, y_pred=VDlabels)
print(classification_report(y_true=TRlabels, y_pred=VDlabels, target_names=target_names))

### F1 for Bayes model
confusion_matrix(y_true=TRlabels, y_pred=NBlabels)
target_names = ['positive', 'neutral', 'negative']
score = metrics.precision_recall_fscore_support(y_true=TRlabels, y_pred=NBlabels)
print(classification_report(y_true=TRlabels, y_pred=NBlabels, target_names=target_names))


import os
import sys
import re
import readline
from classes.annotation import Annotation

"""
Script Purpose: Easily manage annotations for your interfaces
"""

#Constants
#Directory Information
maps_dir = "../../interfaces/"

#String parse with regex to construct list of annotation class objects
def get_annotations(map_data):
    annotation_list = []
    for line in map_data:
        if re.match('viewer.scene.annotations.add',line.strip('\t')):
            #print(line)
            pos_string = re.search('(?<=position:\[)(?s).*?(?=])',line).group(0).split(',')
            cam_pos_string = re.search('(?<=cameraPosition":\[)(?s).*?(?=])',line).group(0).split(',')
            cam_target_string = re.search('(?<=cameraTarget":\[)(?s).*?(?=])',line).group(0).split(',')
            anno = Annotation([float(x) for x in pos_string], [float(x) for x in cam_pos_string],[float(x) for x in cam_target_string], re.search('(?<="title":")(?s).*?(?=")', line).group(0), re.search('(?<="description":")(?s).*?(?=")', line).group(0),map_data.index(line))
            #print(anno.generate_js(1))
            annotation_list.append(anno)

    return annotation_list

def annotations_menu(annotations_list):
    print('---------------------------------------------------------------------\n')
    print('Enter 0 to add a new Annotation or Select an Annotation to Edit/Delete. Enter the Number Besides the Name\n')
    print('\t[0]\tAdd New Annotation')
    for anno in annotations_list:
        print('\t[' + str(annotations_list.index(anno)+1) + ']' + '\t' + anno.get_title())

#Construct new annotation object
def get_annotation():

    is_num=False

    anno_title = raw_input('Enter the title of your annotation: ')
    anno_desc = raw_input('Enter the description of your annotation: ')
    while not is_num:
        try:
            anno_X = float(input('Enter the X value of your annotation position: '))
            is_num = True
        except:
            print('Error: Invalid Number Choice\n')

    is_num=False
    while not is_num:
        try:
            anno_Y = float(input('Enter the Y value of your annotation position: '))
            is_num = True
        except:
            print('Error: Invalid Number Choice\n')

    is_num=False
    while not is_num:
        try:
            anno_Z = float(input('Enter the Z value of your annotation position: '))
            is_num = True
        except:
            print('Error: Invalid Number Choice\n')

    is_num=False
    while not is_num:
        try:
            cam_X = float(input('Enter the X value of your Target: '))
            is_num = True
        except:
            print('Error: Invalid Number Choice\n')

    is_num=False
    while not is_num:
        try:
            cam_Y = float(input('Enter the Y value of your Target: '))
            is_num = True
        except:
            print('Error: Invalid Number Choice\n')

    is_num=False
    while not is_num:
        try:
            cam_Z = float(input('Enter the Z value of your Target: '))
            is_num = True
        except:
            print('Error: Invalid Number Choice\n')

    """ is_num=False
    while not is_num:
        try:
            camTarg_X = float(input('Enter the X value of your Camera Target Position: '))
            is_num = True
        except:
            print('Error: Invalid Number Choice\n')

    is_num=False
    while not is_num:
        try:
            camTarg_Y = float(input('Enter the Y value of your Camera Target Position: '))
            is_num = True
        except:
            print('Error: Invalid Number Choice\n')

    is_num=False
    while not is_num:
        try:
            camTarg_Z = float(input('Enter the Z value of your Camera Target Position: '))
            is_num = True
        except:
            print('Error: Invalid Number Choice\n') """

    return Annotation([anno_X,anno_Y,anno_Z],[cam_X,cam_Y,cam_Z],[anno_X,anno_Y,anno_Z],anno_title,anno_desc,5)

#scan Directory info to get list of maps
def get_maps():
    exclude = set(['libs','pointclouds'])
    maps_list = []
    for root, dirs, files in os.walk(maps_dir):
        dirs[:] = [d for d in dirs if d not in exclude]
        for file in files:
            if file[len(file)-5:len(file)] == '.html':
                maps_list.append(os.path.join(root,file))

    return maps_list

def main():
    maps_list = get_maps()
    select_map = -1
    print('''
      ___                    _        _
     / _ \                  | |      | |
    / /_\ \_ __  _ __   ___ | |_ __ _| |_ ___  _ __
    |  _  | '_ \| '_ \ / _ \| __/ _` | __/ _ \| '__|
    | | | | | | | | | | (_) | || (_| | || (_) | |
    \_| |_/_| |_|_| |_|\___/ \__\__,_|\__\___/|_|   ''')

    print('\n')

    print('---------------------------------------------------------------------\n')
    print('Select a map to manage annotations for. Enter the Number Besides the Name\n')
    for map in maps_list:
        print('\t[' + str(maps_list.index(map)) + '].\t' + map)
    print('\n---------------------------------------------------------------------\n')

    while(select_map not in range(0,len(maps_list))):
        try:
            select_map = int(input('\tEnter Number: '))
            if select_map not in range(0,len(maps_list)):
                print('Error: Invalid Number Choice\n')
        except:
            print('Error: Invalid Number Choice\n')
    with open(maps_list[select_map],'r') as map_file:
        map_data = map_file.read()

    #Parse and read file to setup scenes
    split_data = map_data.split('\n')

    #print(split_data)

    annotations_list = get_annotations(split_data)

    #Generate List of Annotations
    select_anno = -2
    while(select_anno not in range(-1,len(annotations_list)+1)):
        try:
            annotations_menu(annotations_list)
            select_anno = int(input("Enter Selection (-1 to quit): " ))
            if select_anno not in range(-1,len(annotations_list)+1):
                print('Error: Invalid Number Choice\n')
        except:
            print('Error: Invalid Number Choice\n')

    if select_anno == 0:
        new_annotation = get_annotation()
        new_line = new_annotation.generate_js()
        split_data.insert(split_data.index("\t\t\tviewer.fitToScreen();")+1,new_line)
        new_file = '\n'.join([str(x) for x in split_data])
        with open(maps_list[select_map],'w') as map_file:
            map_file.write(new_file)
        print("Annotation added")
    elif select_anno == -1:
        quit()
    else:
        edit_annotation(annotations_list[select_anno-1],annotations_list,select_anno,maps_list,split_data,select_map)
        '''
        confirm = raw_input("Enter YES to DELETE this annotation. Any other input will cancel the operation: ")
        if confirm in ['yes','YES','Yes']:
            split_data.pop(annotations_list[select_anno-1].get_index())
            new_file = '\n'.join([str(x) for x in split_data])
            with open(maps_list[select_map],'w') as map_file:
                map_file.write(new_file)
            print("Annotation deleted")

        else:
            quit()
            '''

def edit_menu(anno):
    print('----------------------------------------')
    print('Select a choice (-1 to cancel without Saving)')
    print('')
    print('[0] Delete')
    print('[1] Edit Title (Current: ' + anno.get_title() + ')')
    print('[2] Edit Description (Current : ' + anno.get_desc() + ')')
    print('[3] Edit Position (Current: ' + str(anno.get_pos()) + ')')
    print('[4] Edit Target (Current: ' + str(anno.get_cpos()) + ')')
    print('[5] Save and Quit\n')
    #print()

#Pass annotation class obj as parameter
#Recursive!
def edit_annotation(anno,annotations_list,select_anno,maps_list,split_data,select_map):
    edit_menu(anno)

    choice = str(raw_input("Enter Choice: "))
    while( choice not in ['-1','0','1','2','3','4','5']):
        print('Invalid Choice')
        edit_menu(anno)
        choice = str(raw_input("Enter Choice: "))

    if choice == '-1':
        print('quitting')
        quit()
    elif choice =='0':
        confirm = raw_input("Enter YES to DELETE this annotation. Any other input will cancel the operation: ")
        if confirm in ['yes','YES','Yes']:
            split_data.pop(annotations_list[select_anno-1].get_index())
            new_file = '\n'.join([str(x) for x in split_data])
            with open(maps_list[select_map],'w') as map_file:
                map_file.write(new_file)
            print("Annotation deleted")

        else:
            quit()
    elif choice == '1':

        new_title = rlinput('Edit Title: ',anno.get_title())
        anno.set_title(new_title)
        edit_annotation(anno,annotations_list,select_anno,maps_list,split_data,select_map)
    elif choice == '2':

        new_title = rlinput('Edit Description: ',anno.get_desc())
        anno.set_desc(new_title)
        edit_annotation(anno,annotations_list,select_anno,maps_list,split_data,select_map)
    elif choice == '3':
        is_num=False
        while not is_num:
            try:
                #print('Current Position = ' + str(anno.get_pos()))
                anno_X = float(input('Enter the X value of your annotation position: '))
                is_num = True
            except:
                print('Error: Invalid Number Choice\n')

        is_num=False
        while not is_num:
            try:
                #print('Current Position = ' + str(anno.get_pos()))
                anno_Y = float(input('Enter the Y value of your annotation position: '))
                is_num = True
            except:
                print('Error: Invalid Number Choice\n')

        is_num=False
        while not is_num:
            try:
                #print('Current Position = ' + str(anno.get_pos()))
                anno_Z = float(input('Enter the Z value of your annotation position: '))
                is_num = True
            except:
                print('Error: Invalid Number Choice\n')
        anno.set_pos([anno_X,anno_Y,anno_Z])
        anno.set_target([anno_X,anno_Y,anno_Z])
        edit_annotation(anno,annotations_list,select_anno,maps_list,split_data,select_map)
    elif choice =='4':
        is_num=False
        while not is_num:
            try:
                #print('Current Camera Position = ' + str(anno.get_cpos()))
                cam_X = float(input('Enter the X value of your Camera Position: '))
                is_num = True
            except:
                print('Error: Invalid Number Choice\n')

        is_num=False
        while not is_num:
            try:
                #print('Current Camera Position = ' + str(anno.get_cpos()))
                cam_Y = float(input('Enter the Y value of your Camera Position: '))
                is_num = True
            except:
                print('Error: Invalid Number Choice\n')

        is_num=False
        while not is_num:
            try:
                #print('Current Camera Position = ' + str(anno.get_cpos()))
                cam_Z = float(input('Enter the Z value of your Camera Position: '))
                is_num = True
            except:
                print('Error: Invalid Number Choice\n')

        anno.set_cpos([cam_X,cam_Y,cam_Z])
        edit_annotation(anno,annotations_list,select_anno,maps_list,split_data,select_map)
    elif choice == '5':
        split_data[annotations_list[select_anno-1].get_index()] = anno.generate_js()
        new_file = '\n'.join([str(x) for x in split_data])
        with open(maps_list[select_map],'w') as map_file:
            map_file.write(new_file)
        print('Annotation Edited')

#Function to have default input value, ONLY WORKS ON LINUX
def rlinput(prompt, prefill=''):
    readline.set_startup_hook(lambda: readline.insert_text(prefill))
    try:
      return raw_input(prompt)
    finally:
      readline.set_startup_hook()

main()


#=====================================================================
# Program:      Homework Assignment #5B
# Programmer:   Teresa Potts
# Date:         Mar 14, 2013
# Abstract:     This program simulates a rock, paper, scissors game.
#               You play against a computer using the random import in
#               the Python library.
#=====================================================================

# Import the random library from Python.
import random

# Create global variables to use for the accumulator. Start the values out as zero.
PLAYER_ACCUMULATOR = 0
COMPUTER_ACCUMULATOR = 0
TIE_ACCUMULATOR = 0

# The main function to run the program.
def main():
    # Describe what the program is designed to do to the user.
    print("This is a old fashioned rock, paper, scissors game. It is you versus the computer.\nUse 1 for rock. Use 2 for paper. Use 3 for scissors. \nGood Luck!")
    # Create a variable to control the loop.
    again = 'y'
    # Play the game everytime the user wants to keep playing.
    while again == 'Y' or again =='y':
        # Call the play_game function.
        play_game()
        # Ask the user if they want to play the game again.
        again = input("Would you like to play again? (Enter Y for Yes): ")
    print("Thanks for playing.")

# Define the play_game function.
def play_game():

    # Get the players selection and show them what their choice is: rock, paper, or scissors.
    players_choice = get_players_choice()
    if players_choice == 1:
        print("You have chosen rock.")
    elif players_choice == 2:
        print("You have chosen paper.")
    else:
        print("You have chosen scissors.")

    # Get the computers selection and show the user what the computer selected: rock, paper, or scissors.
    computers_choice = get_computers_choice()
    if computers_choice == 1:
        print("Your competition has chosen rock.")
    elif computers_choice == 2:
        print("Your competition has chosen paper.")
    else:
        print("Your competition has chosen scissors.")

    # Call the function to determine the winner.
    winner = determine_the_winner(computers_choice, players_choice)
    # Return the output from the determine_the_winner function.
    return winner

# Define the get_computer_choice function.
def get_computers_choice():
    # Get a random integer between 1-3 for the computer's choice.
    computers_choice = random.randint(1,3)
    # Return the output from the computer_choice function.
    return computers_choice

# Define the get_players_choice function.
def get_players_choice():
    # Get the input from the user.
    players_choice = input("Enter value 1-Rock, 2-Paper, or 3-Scissors. ")

    # Display a error message if the user picks anything other than 1, 2, or 3.
    while players_choice not in ["1", "2", "3"]:
        print("Error: Please input value of 1, 2, or 3. ")
        players_choice = input("Enter value 1, 2, or 3. ")
        # Return the player's choice as an integer since it was looking for a string value.
    return int(players_choice)

# Define the determine_the winner function.
def determine_the_winner(computers_choice, players_choice):
    # If, elif, and else statements do comparisons on the values selected to determine who wins.
    if computers_choice == players_choice:
        # Show the user the outcome.
        print("Result is a tie!")
        # Use a global variable to start your accumulator.
        global TIE_ACCUMULATOR
        # Use an accumulator to keep track of ties. Add +1 if this condition is met.
        TIE_ACCUMULATOR += 1
    elif (computers_choice == 1 and players_choice == 3) or (computers_choice == 2 and players_choice == 1) or (computers_choice == 3 and players_choice == 2):
        # Show the user the outcome.
        print("Your competition wins!")
        # Use a global variable to start your accumulator.
        global COMPUTER_ACCUMULATOR
        # Use an accumulator to keep track of losses. Add +1 if this condition is met.
        COMPUTER_ACCUMULATOR += 1
    else:
        # Show the user the outcome.
        print("You win!")
        # Use a global variable to start your accumulator.
        global PLAYER_ACCUMULATOR
        # Use an accumulator to keep track of wins. Add +1 if this condition is met.
        PLAYER_ACCUMULATOR += 1

    # Print the users win-loss-tie record.
    print("Your W-L-T record is %d-%d-%d" % (PLAYER_ACCUMULATOR, COMPUTER_ACCUMULATOR, TIE_ACCUMULATOR))

# Call the main function.
main()

CITP 110/Chapter 6/Chapter 6 HW (good version w accum).py


import requests
import re
from lxml import etree

r1 = requests.get('http://www.zjwater.com/pages/category/0b1/index_1.htm')
selector = etree.HTML(r1.content)
div = selector.xpath('//table[@id="ctl00_cphBody_ctl00_dt_itemlist"]//a/@href')
urls = ['http://www.zjwater.com' + i for i in div]
titles = selector.xpath('//table[@id="ctl00_cphBody_ctl00_dt_itemlist"]//a/@title')
times = selector.xpath('//table[@id="ctl00_cphBody_ctl00_dt_itemlist"]//div [@class="SHORT_DIV2"]/text()')

for i, j, k in zip(urls, titles, times):
    content = j[:20] + '...' if len(j) > 20 else j
    item = '
 {} {}
'.format(i, j, k, content)
    print(item)

r2 = requests.get('http://www.nbwater.gov.cn/ztzl/slgcbzhgl/Default.aspx')
item2 = re.findall(
    '''.*?.*?\[(.*?)\]	''',
    r2.text, re.S)
for i in item2[:6]:
    content = i[1][:20] + '...' if len(i[1]) > 20 else i[1]
    item = '
 {} {}
'.format(
        'http://www.nbwater.gov.cn/ztzl/slgcbzhgl/Default.aspx/' + i[0], i[1], i[2].replace('.', '-'), content)
    print(item)

r3 = requests.get('http://www.xiangshan.gov.cn/col/col18303/index.html#')
r3.encoding = 'utf8'
item3 = re.findall(
    '''.*?(.*?)	''',
    r3.text)
for i in item3[:]:
    content = i[1][:20] + '...' if len(i[1]) > 20 else i[1]
    item = '
 {} {}
'.format(
        'http://www.xiangshan.gov.cn' + i[0], i[1], i[2].replace('2016-', ''), content)
    print(item)


# -*- coding: utf-8 -*-
"""
Created on Sat Oct 27 21:57:44 2018

@author: yyhaos
"""

url='https://www.google.com.hk/search?newwindow=1&safe=strict&as_st=y&hl=zh-CN&tbs=isz%3Alt%2Cislt%3Avga&tbm=isch&sa=1&ei=hZTUW4CeO8if0gLJsoyoDw&q=school+poster&oq=school+poster&gs_l=img.3..0i10i19k1j0i7i30i19k1l9.13967.16486.0.16702.8.8.0.0.0.0.434.1235.2-3j0j1.4.0....0...1c.1j4.64.img..4.3.799...0i7i30k1.0.Yegt4da9gMU'
# the target web, only google pics web is suitable

import os
import requests
from bs4 import BeautifulSoup as bs# need bs4
from urllib.request import Request,urlopen

os.makedirs('./image/', exist_ok=True)  #pics are saved in */image/

def pic_dl(IMAGE_URL,Name):
    try:
        r = requests.get(IMAGE_URL,timeout=5)
        with open(Name, 'wb') as f:
            f.write(r.content)
    except:
        print('Can not save ',Name,' ' )

headers= {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36 Edge/15.15063'
    }

def get_date(url):# from web get beautifulsoup
    request = Request(url,headers=headers)
    response = urlopen(request)
    print(response)
    html=bs(response)
    return html

html=get_date(url)
tag=html.find_all('a',class_='rg_l')
tag=html.find_all(class_="rg_meta notranslate")# this tag concludes the original weds of pics
print('tag:')
#print(tag)
cnt=1
Name='./image/img'+str(cnt)+'.png'
for i in tag:
    print('---')
    for j in i:
        j_z=eval(j)
        print(j_z["ou"])
        image_url=j_z["ou"] # Original Website of pic
        Name='./image/img'+str(cnt)+'.png'
        print(Name)
        pic_dl(image_url,Name)
        cnt=cnt+1

Poster Design/Data collect/pic collect (Web crawler)/main.py


import sys
import ast
from django import template
from django.template.defaultfilters import stringfilter

register = template.Library()

@register.filter
@stringfilter
def convert_note_for_ly(note):
    note = ast.literal_eval(note)

    pitch = convert_pitch(note)
    notes = convert_duration(pitch, note['duration'])

    return convert_start_beat(notes, note['measure'], note['beat'])

@register.filter
@stringfilter
def convert_drums_for_ly(drum):
    drum = ast.literal_eval(drum)
    notes = convert_duration(drum['note'], drum['duration'])
    ret = convert_start_beat(notes, drum['measure'], drum['beat'])
    return ret

def convert_pitch(note):
    temp_note = str(note['note']).replace("s", "is")
    temp_oct = int(note['octave']) - 4
    append = "'" if temp_oct >= 0 else ","
    dir = 1 if temp_oct >= 0 else -1

    for x in range(0, temp_oct, dir):
        temp_note += append

    if str(note['note']) == "x":
        temp_note = temp_note.replace("x", "c")
        if append == ",":
            temp_note = temp_note[:len(temp_note) - 1]
        else:
            temp_note += "'"

    return temp_note

def convert_start_beat(notes, measure, beat):
    ret = ''
    beat += (measure * 16)
    beat -= 1
    bars = beat / 16
    beat %= 16
    for x in range(0, bars):
        ret += 'r1 '

    halves = beat / 8
    beat %= 8
    for x in range(0, halves):
        ret += 'r2 '

    quarters = beat / 4
    beat %= 4
    for x in range(0, quarters):
        ret += 'r4 '

    eigths = beat / 2
    beat %= 2
    for x in range(0, eigths):
        ret += 'r8 '

    for x in range(0, beat):
        ret += 'r16 '

    ret += notes

    return ret

def convert_duration(notes, duration):
    value = "< " + notes

    if duration == 1:
        value += " >16"
    elif duration == 2:
        value += " >8"
    elif duration == 3:
        value += " >8."
    elif duration == 4:
        value += " >4"
    elif duration == 5:
        value += " >4~<" + notes + " >16"
    elif duration == 6:
        value += " >4."
    elif duration == 7:
        value += " >4~<" + notes + " >8."
    elif duration == 8:
        value += " >2"
    elif duration == 9:
        value += " >2~<" + notes + " >16"
    elif duration == 10:
        value += " >2~<" + notes + " >8"
    elif duration == 11:
        value += " >2~<" + notes + " >8."
    elif duration == 12:
        value += " >2."
    elif duration == 13:
        value += " >2.~<" + notes + " >16"
    elif duration == 14:
        value += " >2.~<" + notes + " >8"
    elif duration == 15:
        value += " >2.~<" + notes + " >8."
    elif duration == 16:
        value += " >1"

    return value


from steward_palantir.handlers import BaseHandler
from pyramid.security import unauthenticated_userid

class SmartMail(BaseHandler):
    """ A simple handler that sends pretty emails """
    name = 'smartmail'
    def __init__(self, mail_to):
        self.mail_to = mail_to

    def __call__(self, request, check, normalized_retcode, results, **kwargs):
        if normalized_retcode == 0:
            title = 'RESOLVED'
            action = 'succeeded'
        elif normalized_retcode == 1:
            title = 'WARNING'
            action = 'failed'
        else:
            title = 'ERROR'
            action = 'failed'

        if len(results) == 1:
            target = results[0].minion
        else:
            target = "%d minions" % len(results)

        subject = "[%s] %s on %s" % (title, check.name, target)

        if kwargs.get('marked_resolved'):
            minions = [result.minion for result in results]
            body = ("%s marked resolved by %s on %s" %
                    (check.name, unauthenticated_userid(request),
                     ', '.join(minions)))
        else:
            body = "%s %s on\n" % (check.name, action)
            for result in results:
                body += ("    %s %d time%s\n" % (result.minion, result.count,
                                                 's' if result.count > 1 else
                                                 ''))
                if result.stdout:
                    body += 'STDOUT:\n' + result.stdout
                if result.stderr:
                    body += 'STDERR:\n' + result.stderr

        request.subreq('mail', subject=subject, body=body, mail_to=self.mail_to)