import numpy as np
import pickle
import os

class OrnsteinUhlenbeckActionNoise:
	# from https://github.com/openai/baselines/blob/master/baselines/ddpg/noise.py
    def __init__(self, mu, sigma=0.3, theta=.15, dt=1e-2, x0=None):
        self.theta = theta
        self.mu = mu
        self.sigma = sigma
        self.dt = dt
        self.x0 = x0
        self.reset()

    def __call__(self):
        x = self.x_prev + self.theta * (self.mu - self.x_prev) * self.dt + \
                self.sigma * np.sqrt(self.dt) * np.random.normal(size=self.mu.shape)
        self.x_prev = x
        return x

    def reset(self):
        self.x_prev = self.x0 if self.x0 is not None else np.zeros_like(self.mu)

    def save(self, dir):
        file = os.path.join(dir, 'ounoise.pickle')
        with open(file, 'wb') as f:
            pickle.dump(self, f, pickle.HIGHEST_PROTOCOL)

    def load(self, dir):
        file = os.path.join(dir, 'ounoise.pickle')
        with open(file, 'rb') as f:
            noise = pickle.load(f)
        return noise


import yfinance as yf
import datetime
import pandas as pd

def get_dados(siglas, num_dias = 588, intervalo = '1wk', inicio = '',  fim = ''):

    """
        siglas -> []

        Retorna uma lista de DataFrames
        com os valores de fechamento das siglas passadas
    """

    if inicio == '':
        inicio = (datetime.date.today() - datetime.timedelta(num_dias))
    if fim == '':
        fim = datetime.date.today()

    dados = []
    for sigla in siglas:
        df = yf.download(sigla, start = inicio, end = fim, interval = intervalo)

        df.drop(['Open', 'High', 'Low', 'Adj Close', 'Volume'], axis = 1, inplace = True)
        df = df.transpose()
        df.dropna(axis = 1, inplace = True)
        df.index = [sigla]

        dados.append(df)

    dados = pd.concat(dados)

    return dados


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('spirit_user', '0004_auto_20150731_2351'),
    ]

    operations = [
        migrations.AddField(
            model_name='userprofile',
            name='given_likes_count',
            field=models.PositiveIntegerField(default=0, verbose_name='given likes count'),
        ),
        migrations.AddField(
            model_name='userprofile',
            name='received_likes_count',
            field=models.PositiveIntegerField(default=0, verbose_name='received likes count'),
        ),
    ]


# coding:utf8
# author:winton

import logging
import os
import datetime
import argparse

from conf import Config
from util import Util
from ConsumerManager import ConsumerManager

class Lams:
    '''
    控制数据收集的主要流程
    '''

    def init(self):
        '''
        读取配置并完成初始化
        '''
        loggerConfig = Config.logger
        # 检查日志文件夹是否存在
        logDir = os.path.split(loggerConfig['filename'])[0]
        if not os.path.exists(logDir):
            os.makedirs(logDir)
        # 日志配置
        logging.basicConfig(
            level=loggerConfig['level'],
            format=loggerConfig['format'],
            filename=loggerConfig['filename'],
            encoding=loggerConfig['encoding']
        )

        # 载入consumer
        self.cm = ConsumerManager(Config)
        logging.info('%d Consumers loaded' % len(self.cm.consumers))
        self.allFile = 0
        self.successFile = 0

        logging.info('Lams starting...')

    def startForNew(self, move_after_dispatch=True):
        '''
        开始处理指定目录下的数据
        '''
        dataDir = Config.datapool_new
        dataList = os.listdir(dataDir)
        if len(dataList) == 0:
            logging.info('New data not found, exiting...')
            os.system('exit 0')
        logging.info('New data found, dispatching...')
        for filename in dataList:
            self.dispatch(filename, dataDir, move_after_dispatch)
        logging.info('Dispatching finish, %d success, %d fail' % (self.successFile, self.allFile - self.successFile))
        os.system('exit 0')

    def startForAll(self, classInfo=None):
        '''
        重新处理所有数据
        '''
        if classInfo is not None:
            logging.info('dispatch for class [%s]' % classInfo)
        logging.info('dispatch all data...')
        for parent, dirnames, filenames in os.walk(Config.datapool):
            for filename in filenames:
                self.dispatch(filename, parent, False, classInfo=classInfo)
        logging.info('Dispatching finish, %d success, %d fail' % (self.successFile, self.allFile - self.successFile))
        os.system('exit 0')

    def dispatch(self, filename, dataDir, move_after_dispatch=True, classInfo=None):
        '''
        分发对应的文件列表
        '''
        filePath = os.path.join(dataDir, filename)
        try:
            event = Util.loadJsonFile(filePath)
            consumers = self.cm.getMapConsumer(event, classInfo)
            for csm in consumers:
                logging.info('event "%s" is sending to consumer "%s"' % (filePath, csm))
            self.cm.emitEvent(event, consumers)
        except Exception as e:
            logging.exception('Error when dispatching "%s" [%s]' % (filePath, str(e)))
        else:
            self.successFile += 1
            # 将已处理的文件移动到指定文件夹
            if move_after_dispatch:
                t = datetime.datetime.now()
                today = t.strftime('%Y-%m-%d')
                newDir = '%s/%s' % (Config.datapool, today)
                if not os.path.exists(newDir):
                    os.makedirs(newDir)
                newFilePath = '%s/%s' % (newDir, filename)
                logging.debug('moving [src=%s] [dst=%s]' % (filePath, newFilePath))
                os.rename(filePath, newFilePath)
        finally:
            self.allFile += 1

if __name__ == '__main__':
    ap = argparse.ArgumentParser(description='do dispatching jobs')
    ap.add_argument('-A', '--all', action='store_true', help='dispatch all history data and new data')
    ap.add_argument('-c', help='dispatch all but just dispatch to one class, use it in this form [moduleName:className]')
    args = ap.parse_args()

    test = Lams()
    test.init()
    if args.all:
        test.startForAll()
    elif args.c is not None:
        test.startForAll(args.c.split(':'))
    else:
        test.startForNew()


__all__ = [
    "JobItem",
]
import datetime

import top

class JobItem(top.Table):
    """job_item table ORM.
    """
    _job = top.Job()
    _agent_stocktake = top.AgentStocktake()

    def __init__(self):
        """Toll Outlet Portal job_item table initialiser.
        """
        super(JobItem, self).__init__('job_item')

    @property
    def schema(self):
        return ["id INTEGER PRIMARY KEY",
                "job_id INTEGER",
                "connote_nbr CHAR(30)",
                "item_nbr CHAR(32)",
                "consumer_name CHAR(30)",
                "email_addr CHAR(60)",
                "phone_nbr CHAR(20)",
                "pieces INTEGER",
                "status INTEGER",
                "created_ts TIMESTAMP",
                "pickup_ts TIMESTAMP",
                "pod_name CHAR(40)",
                "identity_type_id INTEGER",
                "identity_type_data CHAR(30)",
                "extract_ts TIMESTAMP",
                "reminder_ts TIMESTAMP",
                "notify_ts TIMESTAMP"]

    def collected_sql(self, business_unit, ignore_pe=False):
        """SQL wrapper to extract the collected items from the "jobitems"
        table.

        **Args:**
            *business_unit*: the id relating to the job.bu_id value.

        **Kwargs:**
            *ignore_pe*: ``boolean`` flag to ignore job items whose
            parent job is Primary Elect (default ``False``)

        **Returns:**
            the SQL string

        """
        sql = """SELECT ji.connote_nbr as 'REF1',
    ji.id as 'JOB_KEY',
    ji.pickup_ts as 'PICKUP_TIME',
    ji.pod_name as 'PICKUP_POD',
    it.description as 'IDENTITY_TYPE',
    ji.identity_type_data as 'IDENTITY_DATA',
    ji.item_nbr as 'ITEM_NBR',
    ag.code as 'AGENT_ID',
    ag.state as 'AGENT_STATE'
FROM job_item as ji, identity_type as it, job as j, agent as ag
WHERE pickup_ts IS NOT null
AND extract_ts IS null
AND ji.identity_type_id = it.id
AND j.agent_id = ag.id
AND (ji.job_id = j.id AND j.bu_id = %d)""" % business_unit

        if ignore_pe:
            sql += """
AND (j.service_code != 3 OR j.service_code IS NULL)"""

        return sql

    def upd_collected_sql(self, id, time):
        """SQL wrapper to update the collected items from the "jobitems"
        table.

        **Args:**
            *id*: the id relating to the ``jobitem.id`` value.

        **Returns:**
            the SQL string

        """
        sql = """UPDATE job_item
SET extract_ts = '%s'
WHERE id = %d""" % (time, id)

        return sql

    def upd_file_based_collected_sql(self, connote, item_nbr, time=None):
        """SQL wrapper to update the collected items from the "jobitems"
        table.

        This variant of the :meth:`upd_collected_sql` method is used
        to close of file-based extractions.

        **Args:**
            *connote*: connote value relating to the ``jobitem.connote_nbr``
            value

            *item_nbr*: connote value relating to the ``jobitem.item_nbr``
            value

        **Kwargs:**
            *time*: override the time to set from the current time

        **Returns:**
            the SQL string

        """
        if time is None:
            time = datetime.datetime.now().isoformat(' ').split('.', 1)[0]

        sql = """UPDATE %(name)s
SET extract_ts = '%(time)s', pickup_ts = '%(time)s'
WHERE connote_nbr = '%(connote)s'
AND item_nbr = '%(item_nbr)s'""" % {'name': self.name,
                                    'time': time,
                                    'connote': connote,
                                    'item_nbr': item_nbr}

        return sql

    def connote_sql(self, connote):
        """SQL wrapper to extract records where job_item.connote_nbr
        is equal to *connote*.

        **Args:**
            connote: Connote value relating to the job_item.connote_nbr
            column.

        **Returns:**
            the SQL string

        """
        sql = """SELECT id
FROM %s
WHERE connote_nbr = '%s'""" % (self.name, connote)

        return sql

    def connote_item_nbr_sql(self, connote, item_nbr):
        """SQL wrapper to extract records where job_item.connote_nbr
        is equal to *connote* and job_item.item_nbr equals *item_nbr*.

        **Args:**
            connote: Connote value relating to the job_item.connote_nbr
            column.

            item_nbr: Item Number value relating to the job_item.item_nbr
            column.

        **Returns:**
            the SQL string

        """
        sql = """SELECT id
FROM %s
WHERE connote_nbr = '%s'
AND item_nbr = '%s'
ORDER BY created_ts DESC""" % (self.name, connote, item_nbr)

        return sql

    def item_number_sql(self, item_nbr):
        """SQL wrapper to extract records where job_item.item_nbr
        is equal to *item_nbr*.

        **Args:**
            item_nbr: Item Number value relating to the job_item.item_nbr
            column.

        **Returns:**
            the SQL string

        """
        sql = """SELECT id
FROM %s
WHERE item_nbr = '%s'""" % (self.name, item_nbr)

        return sql

    def uncollected_sql(self, start_date, uncollected_period):
        """SQL wrapper to extract the job_item records which remain
        uncollected after *uncollected_period* has elapsed.

        **Args:**
            uncollected_period: job_item.notify_ts value that defines
            an uncollected parcel

        **Returns:**
            the SQL string

        """
        sql = """SELECT id
FROM job_item
WHERE (created_ts > '%s' AND notify_ts < '%s')
AND pickup_ts IS NULL
AND (email_addr != '' OR phone_nbr != '')
AND reminder_ts IS NULL""" % (start_date, uncollected_period)

        return sql

    def job_item_agent_details_sql(self, job_item_id):
        """SQL wrapper to extract the agent details against a *job_item_id*.

        SQL also returns additional information relating to the
        *job_item_id* such as:

        * ``jobitem.connote_nbr``
        * ``jobitem.item_nbr``
        * ``jobitem.notify_ts``
        * ``jobitem.created_ts``
        * ``jobitem.email_addr``
        * ``jobitem.phone_nbr``
        * ``jobitem.pickup_ts``

        and the *job.bu_id*.

        **Args:**
            job_item_id: the jobitem.id value to search against

        **Returns:**
            the SQL string

        """
        sql = """SELECT ag.name,
       ag.address,
       ag.suburb,
       ag.postcode,
       ji.connote_nbr,
       ji.item_nbr,
       ji.notify_ts,
       ji.created_ts,
       ji.email_addr,
       ji.phone_nbr,
       ji.pickup_ts,
       j.bu_id
FROM job_item as ji, job as j, agent as ag
WHERE ji.job_id = j.id
AND j.agent_id = ag.id
AND ji.id = %d""" % job_item_id

        return sql

    def update_reminder_ts_sql(self, id, ts=None):
        return self.update_timestamp_sql(id, column='reminder_ts', ts=ts)

    def update_notify_ts_sql(self, id, ts=None):
        return self.update_timestamp_sql(id, column='notify_ts', ts=ts)

    def update_timestamp_sql(self, id, column, ts=None):
        """SQL wrapper to update the ``job_item.reminder_ts`` to *ts*
        timestamp.

        **Args:**
            *id*: integer value relating to the ``job_item.id``

            *column*: the timestamp column to update

        **Kwargs:**
            *ts*: override the current time

        **Returns:**
            the SQL string

        """
        if ts is None:
            ts = datetime.datetime.now().isoformat(' ').split('.', 1)[0]

        sql = """UPDATE %s
SET %s = '%s'
WHERE id = %d
""" % (self.name, column, ts, id)

        return sql

    def connote_base_primary_elect_job(self, connote):
        """SQL wrapper to verify if a *connote* is associated with a primary
        elect job.

        Primary elect jobs are identified by a integer value 3 in the
        ``job.service_code`` column.

        **Args:**
            connote: the jobitem.connote value to search against.

        **Returns:**
            the SQL string

        """
        sql = """SELECT ji.id
FROM job as j, %s as ji
WHERE ji.job_id = j.id
AND ji.connote_nbr = '%s'
AND ji.notify_ts IS NULL
AND (ji.email_addr != '' OR ji.phone_nbr != '')
AND j.service_code = 3""" % (self.name, connote)

        return sql

    def uncollected_jobitems_sql(self,
                                 service_code=3,
                                 bu_ids=None,
                                 delivery_partners=None,
                                 day_range=14):
        """SQL wrapper to extract uncollected Service Code-based jobs.

        Service Code jobs are identified by a integer value in the
        ``job.service_code`` column.

        Query will ignore records that have either white space or
        a spurious ``.`` in the email or phone number columns.

        The *bu_ids* relate to the ``job.bu_id`` column.

        **Kwargs:**
            *service_code*: value relating to the ``job.service_code``
            column (default ``3`` for Primary Elect)

            *bu_ids*: integer based tuple of Business Unit ID's to search
            against (default ``None`` ignores all Business Units)

            *day_range*: number of days from current time to include
            in search (default 14.0 days)

        **Returns:**
            the SQL string

        """
        if bu_ids is None:
            bu_ids = tuple()
        if len(bu_ids) == 1:
            bu_ids = '(%d)' % bu_ids[0]

        if delivery_partners is None:
            delivery_partners = tuple()
        if len(delivery_partners) == 1:
            delivery_partners = "('%s')" % delivery_partners[0]

        now = datetime.datetime.now()
        start_ts = now - datetime.timedelta(days=day_range)
        start_date = start_ts.strftime('%Y-%m-%d %H:%M:%S')

        sql = """SELECT ji.id, ji.connote_nbr, ji.item_nbr
FROM job AS j, %(name)s AS ji, agent AS ag, delivery_partner AS dp
WHERE ji.job_id = j.id
AND j.agent_id = ag.id
AND ag.dp_id = dp.id
AND dp.name IN %(dps)s
AND ji.pickup_ts IS NULL
AND ji.notify_ts IS NULL
AND (ji.email_addr NOT IN ('', '.') OR ji.phone_nbr NOT IN ('', '.'))
AND j.bu_id IN %(bu_ids)s
AND j.service_code = %(sc)d
AND ji.created_ts > '%(start_date)s'""" % {'name': self.name,
                                           'dps': str(delivery_partners),
                                           'bu_ids': str(bu_ids),
                                           'sc': service_code,
                                           'start_date': start_date}

        return sql

    def reference_sql(self,
                      bu_ids,
                      reference_nbr=None,
                      picked_up=False,
                      delivery_partners=None,
                      columns=None,
                      alias='ji'):
        """Extract connote_nbr/item_nbr against *reference_nbr*.

        Query is an ``OR`` against both ``connote_nbr`` and ``item_nbr``.

        **Args:**
            *bu_ids*: integer based tuple of Business Unit ID's to search
            against (default ``None`` ignores all Business Units)

        **Kwargs:**
            *reference_nbr*: parcel ID number as scanned by the agent.  If
            ``None``, then the values from the ``agent_stocktake`` table
            will be used.

            *picked_up*: boolean flag that will extract ``job_items``
            that have been picked up if ``True``. Otherwise, will extract
            ``job_items`` that have not been picked up if ``False``.

            *delivery_partners*: string based list of Delivery Partner
            names to limit result set against.  For example,
            ``['Nparcel', 'Toll']``.  The values supported are as per
            the ``delivery_partner.name`` table set

            *columns*: string prepresentation of the columns to query
            against

            *alias*: table alias (default ``ji``)

        **Returns:**
            the SQL string

        """
        if columns is None:
            columns = self._select_columns(alias)

        if not picked_up:
            pickup_sql = 'IS NULL'
        else:
            pickup_sql = 'IS NOT NULL'

        if len(bu_ids) == 1:
            bu_ids = '(%d)' % bu_ids[0]

        ref = reference_nbr
        if reference_nbr is None:
            ref = self._agent_stocktake.reference_sql()

        dp_sql = str()
        if delivery_partners is not None:
            dps = ', '.join(["'%s'" % x for x in delivery_partners])
            dps = '(%s)' % dps
            if len(delivery_partners) == 1:
                dps = "('%s')" % delivery_partners[0]

            dp_sql = """AND dp.name IN %s AND ag.dp_id = dp.id""" % dps

        union_sql = self.job_based_reference_sql(bu_ids,
                                                 ref,
                                                 picked_up,
                                                 delivery_partners,
                                                 columns)

        sql = """SELECT DISTINCT %(columns)s
FROM %(name)s as %(alias)s,
     job AS j,
     agent AS ag,
     agent_stocktake AS st,
     delivery_partner AS dp
WHERE %(alias)s.job_id = j.id
AND ag.id = st.agent_id
AND j.bu_id IN %(bu_ids)s
AND j.agent_id = ag.id
AND (%(alias)s.connote_nbr IN (%(ref)s)
     OR %(alias)s.item_nbr IN (%(ref)s))
AND %(alias)s.pickup_ts %(pickup_sql)s
%(dp_sql)s
UNION
%(union)s""" % {'columns': columns,
                'bu_ids': str(bu_ids),
                'name': self.name,
                'ref': ref,
                'alias': alias,
                'union': union_sql,
                'pickup_sql': pickup_sql,
                'dp_sql': dp_sql}

        return sql

    def job_based_reference_sql(self,
                                bu_ids,
                                reference_nbr,
                                picked_up=False,
                                delivery_partners=None,
                                columns=None,
                                alias='ji'):
        """Extract connote_nbr/item_nbr against *reference_nbr* matched
        to the ``job.card_ref_nbr``.

        Query is an ``OR`` against both ``connote_nbr`` and ``item_nbr``.

        **Args:**
            *bu_ids*: integer based tuple of Business Unit ID's to search
            against (default ``None`` ignores all Business Units)

            *reference_nbr*: parcel ID number as scanned by the agent

        **Kwargs:**
            *picked_up*: boolean flag that will extract ``job_items``
            that have been picked up if ``True``. Otherwise, will extract
            ``job_items`` that have not been picked up if ``False``.

            *delivery_partners*: string based list of Delivery Partner
            names to limit result set against.  For example,
            ``['Nparcel', 'Toll']``.  The values supported are as per
            the ``delivery_partner.name`` table set

            *columns*: string prepresentation of the columns to query
            against

            *alias*: table alias

        **Returns:**
            the SQL string

        """
        if columns is None:
            columns = self._select_columns(alias)

        pickup_sql = 'AND %s.pickup_ts ' % alias
        if not picked_up:
            pickup_sql += 'IS NULL'
        else:
            pickup_sql += 'IS NOT NULL'

        if len(bu_ids) == 1:
            bu_ids = '(%d)' % bu_ids[0]

        dp_sql = str()
        if delivery_partners is not None:
            dps = ', '.join(["'%s'" % x for x in delivery_partners])
            dps = '(%s)' % dps
            if len(delivery_partners):
                dps = "('%s')" % delivery_partners[0]

            dp_sql = """AND dp.name IN %s AND ag.dp_id = dp.id""" % dps

        sql = """SELECT DISTINCT %(columns)s
FROM %(name)s AS %(alias)s,
     job AS j,
     agent AS ag,
     delivery_partner AS dp,
     agent_stocktake AS st
WHERE %(alias)s.job_id = j.id
AND j.bu_id IN %(bu_ids)s
AND j.agent_id = ag.id
%(dp_sql)s
AND %(alias)s.job_id IN
(
%(sql)s
)
%(pickup_sql)s""" % {'columns': columns,
                     'bu_ids': bu_ids,
                     'name': self.name,
                     'sql': self._job.reference_sql(reference_nbr),
                     'alias': alias,
                     'pickup_sql': pickup_sql,
                     'dp_sql': dp_sql}

        return sql

    def _select_columns(self, alias='ji'):
        """Helper method that captures required columns in the
        uncollected aged report query.

        **Kwargs:**
            *alias*: table alias

        **Returns:**
            the SQL string

        """
        columns = """%(alias)s.id as JOB_ITEM_ID,
       j.bu_id as JOB_BU_ID,
       %(alias)s.connote_nbr as CONNOTE_NBR,
       j.card_ref_nbr as BARCODE,
       %(alias)s.item_nbr as ITEM_NBR,
       j.job_ts as JOB_TS,
       %(alias)s.created_ts as CREATED_TS,
       %(alias)s.notify_ts as NOTIFY_TS,
       %(alias)s.pickup_ts as PICKUP_TS,
       %(alias)s.pieces as PIECES,
       %(alias)s.consumer_name as CONSUMER_NAME,
       ag.dp_code as DP_CODE,
       ag.code as AGENT_CODE,
       ag.name as AGENT_NAME,
       ag.address as AGENT_ADDRESS,
       ag.suburb as AGENT_SUBURB,
       ag.state as AGENT_STATE,
       ag.postcode as AGENT_POSTCODE,
       ag.phone_nbr as AGENT_PHONE_NBR,
       (SELECT DISTINCT ag.dp_code
        FROM agent_stocktake AS st, agent AS aag
        WHERE (%(alias)s.connote_nbr = st.reference_nbr
               OR j.card_ref_nbr = st.reference_nbr
               OR %(alias)s.item_nbr = st.reference_nbr)
        AND st.agent_id = aag.id) AS ST_DP_CODE,
       (SELECT DISTINCT ag.code
        FROM agent_stocktake AS st, agent AS aag
        WHERE (%(alias)s.connote_nbr = st.reference_nbr
               OR j.card_ref_nbr = st.reference_nbr
               OR %(alias)s.item_nbr = st.reference_nbr)
        AND st.agent_id = aag.id) AS ST_AGENT_CODE,
       (SELECT DISTINCT ag.name
        FROM agent_stocktake AS st, agent AS aag
        WHERE (%(alias)s.connote_nbr = st.reference_nbr
               OR j.card_ref_nbr = st.reference_nbr
               OR %(alias)s.item_nbr = st.reference_nbr)
        AND st.agent_id = aag.id) AS ST_AGENT_NAME""" % {'alias': alias}

        return columns

    def non_compliance_sql(self,
                           bu_ids,
                           picked_up=False,
                           delivery_partners=None,
                           alias='ji'):
        """Extract ``job_item`` detail of all items in the ``job_item``
        table that do not exist in the ``agent_stocktake`` table.

        Senarios are based on the *picked_up* flag.  For example, all
        parcels that *have* been picked up or *have not* been picked up.

        **Args:**
            *bu_ids*: integer based tuple of Business Unit ID's to search
            against (default ``None`` ignores all Business Units)

        **Kwargs:**
            *picked_up*: boolean flag that will extract ``job_items``
            that have been picked up if ``True``. Otherwise, will extract
            ``job_items`` that have not been picked up if ``False``.

            *delivery_partners*: string based list of Delivery Partner
            names to limit result set against.  For example,
            ``['Nparcel', 'Toll']``.  The values supported are as per
            the ``delivery_partner.name`` table set

            *alias*: table alias (default ``ji``)

        **Returns:**
            the SQL string

        """
        if bu_ids is None:
            bu_ids = tuple()

        if len(bu_ids) == 1:
            bu_ids = '(%d)' % bu_ids[0]

        columns = self._select_columns(alias)
        col = 'ji.id'

        if not picked_up:
            pickup_sql = 'IS NULL'
        else:
            pickup_sql = 'IS NOT NULL'

        dp_sql = str()
        if delivery_partners is not None:
            dps = ', '.join(["'%s'" % x for x in delivery_partners])
            dps = '(%s)' % dps
            if len(delivery_partners) == 1:
                dps = "('%s')" % delivery_partners[0]

            dp_sql = """AND dp.name IN %s AND ag.dp_id = dp.id""" % dps

        sql = """SELECT DISTINCT %(columns)s
FROM %(name)s AS %(alias)s, job AS j, agent AS ag, delivery_partner AS dp
WHERE %(alias)s.job_id = j.id
AND j.agent_id = ag.id
AND %(alias)s.pickup_ts %(pickup_sql)s
%(dp_sql)s
AND ji.id NOT IN
(%(sql)s)""" % {'columns': columns,
                'alias': alias,
                'name': self.name,
                'pickup_sql': pickup_sql,
                'dp_sql': dp_sql,
                'sql': self.reference_sql(bu_ids=bu_ids,
                                          picked_up=picked_up,
                                          delivery_partners=delivery_partners,
                                          columns=col)}

        return sql

    def total_agent_stocktake_parcel_count_sql(self,
                                               bu_ids,
                                               picked_up=False,
                                               delivery_partners=None,
                                               day_range=7,
                                               alias='ji'):
        """Sum ``agent_stocktake`` based parcel counts per ADP based on
        *picked_up*.

        Query is an ``OR`` against both ``connote_nbr`` and ``item_nbr``.

        **Args:**
            *bu_ids*: integer based tuple of Business Unit ID's to search
            against (default ``None`` ignores all Business Units)

        **Kwargs:**
            *picked_up*: boolean flag that will extract ``job_items``
            that have been picked up if ``True``. Otherwise, will extract
            ``job_items`` that have not been picked up if ``False``.

            *delivery_partners*: string based tuple of Delivery Partner
            names to limit result set against.  For example,
            ``['top', 'toll']``.  The values supported are as per
            the ``delivery_partner.name`` table set

            *day_range*: number of days from current time to include
            in the agent_stocktake table search (default 7 days)

            *alias*: table alias (default ``ji``)

        **Returns:**
            the SQL string

        """
        now = datetime.datetime.now()
        start_ts = now - datetime.timedelta(days=day_range)
        start_date = start_ts.strftime('%Y-%m-%d %H:%M:%S')

        if not picked_up:
            pickup_sql = 'IS NULL'
        else:
            pickup_sql = 'IS NOT NULL'

        if len(bu_ids) == 1:
            bu_ids = '(%d)' % bu_ids[0]

        col = 'st.reference_nbr'
        sql_ref = self.reference_sql(bu_ids=bu_ids,
                                     picked_up=picked_up,
                                     delivery_partners=delivery_partners,
                                     columns=col)

        sql = """SELECT DISTINCT agent.dp_code AS DP_CODE,
        agent.code AS AGENT_CODE,
        agent.name AS AGENT_NAME,
        (SELECT MAX(st.created_ts)
         FROM agent_stocktake AS st, agent AS ag
         WHERE ag.code = agent.code
         AND ag.id = st.agent_id) AS STOCKTAKE_CREATED_TS,
        (SELECT COUNT(DISTINCT ags.reference_nbr)
         FROM agent_stocktake AS ags, agent AS ag
         WHERE ags.agent_id = agent.id
         AND ags.created_ts > '%(start_date)s') AS AGENT_PIECES,
        (SELECT COUNT(ji.id)
         FROM job_item AS ji, job AS j, agent AS ag
         WHERE ag.id = agent.id
         AND j.agent_id = ag.id
         AND j.id = ji.job_id
         AND ji.pickup_ts %(pickup_sql)s) AS TPP_PIECES
FROM agent AS agent, agent_stocktake AS agent_stocktake
WHERE agent_stocktake.agent_id = agent.id
AND reference_nbr != ''
AND agent_stocktake.created_ts > '%(start_date)s'
AND agent_stocktake.reference_nbr IN (%(sql_ref)s)
GROUP BY agent.id,
         agent.dp_code,
         agent.code,
         agent.name,
         agent_stocktake.created_ts,
         agent_stocktake.reference_nbr""" % {'sql_ref': sql_ref,
                                             'start_date': start_date,
                                             'pickup_sql': pickup_sql}

        return sql

    def total_parcel_count_sql(self,
                               picked_up=False,
                               delivery_partners=None,
                               alias='ji'):
        """Sum parcel counts per ADP based on *picked_up*.

        **Kwargs:**
            *picked_up*: boolean flag that will extract ``job_items``
            that have been picked up if ``True``. Otherwise, will extract
            ``job_items`` that have not been picked up if ``False``.

            *delivery_partners*: string based tuple of Delivery Partner
            names to limit result set against.  For example,
            ``['Nparcel', 'Toll']``.  The values supported are as per
            the ``delivery_partner.name`` table set

            *alias*: table alias (default ``ji``)

        **Returns:**
            the SQL string

        """
        if not picked_up:
            pickup_sql = 'IS NULL'
        else:
            pickup_sql = 'IS NOT NULL'

        dp_sql = str()
        if delivery_partners is not None:
            dps = ', '.join(["'%s'" % x for x in delivery_partners])
            dps = '(%s)' % dps
            if len(delivery_partners) == 1:
                dps = "('%s')" % delivery_partners[0]

            dp_sql = """AND dp.name IN %s AND ag.dp_id = dp.id""" % dps

        sql = """SELECT SUM(%(alias)s.pieces)
FROM %(name)s AS %(alias)s, job AS j, agent AS ag, delivery_partner AS dp
WHERE %(alias)s.job_id = j.id
AND j.agent_id = ag.id
%(dp_sql)s
AND %(alias)s.pickup_ts %(pickup_sql)s""" % {'name': self.name,
                                             'dp_sql': dp_sql,
                                             'pickup_sql': pickup_sql,
                                             'alias': alias}

        return sql

    def agent_id_of_aged_parcels(self,
                                 period=7,
                                 delivery_partners=None,
                                 alias='ji'):
        """SQL to provide a distinct list of agents that have an
        aged parcel.

        **Kwargs:**
            *period*: time (in days) from now that is the cut off for
            agent compliance (default 7 days)

            *delivery_partners*: string based tuple of Delivery Partner
            names to limit result set against.  For example,
            ``['top', 'toll']``.  The values supported are as per
            the ``delivery_partner.name`` table set

            *alias*: table alias (default ``ji``)

        **Returns:**
            the SQL string

        """
        now = datetime.datetime.now()
        ts = now - datetime.timedelta(days=period)
        date = ts.strftime('%Y-%m-%d %H:%M:%S')

        dps = delivery_partners
        kwargs = {'period': period,
                  'delivery_partners': dps}
        compliance_sql = self._agent_stocktake.compliance_sql(**kwargs)

        sql = """%(compliance_sql)s
AND ag.id IN
(SELECT DISTINCT(j.agent_id)
 FROM job as j, %(name)s AS %(alias)s, agent AS ag
 WHERE %(alias)s.job_id = j.id
 AND %(alias)s.created_ts < '%(date)s'
 AND %(alias)s.pickup_ts IS NULL)""" % {'compliance_sql': compliance_sql,
                                        'name': self.name,
                                        'alias': alias,
                                        'date': date}

        return sql


inp = raw_input("Please enter a score [0.0 to 1.0]: ")
try:
	score = float(inp)
except:
	print("Error: We are expecting a number between 0.0 to 1.0.")
	quit()

grade = "Unknown"
if score < 0 or score > 1:
	print("Sorry, I can not grade the score, because it is out of range.")
	quit()
elif score >= 0.9:
	grade = "A"
elif score >= 0.8:
	grade = "B"
elif score >= 0.7:
	grade = "C"
elif score >= 0.6:
	grade = "D"
else:
	grade = "F"

print(grade)


#
# Facebook "Likes" per account.
#
# MISSING: EA to enumerate FB accounts.
#

import reqs

URL_ROOT = 'http://graph.facebook.com/'
KEYS = ['name', 'username', 'likes']

def get_likes(username):
    resp = reqs.get_data(URL_ROOT + username, {})
    return dict([(k, get_or_none(resp, k))
                 for k in KEYS])

#----------------

def get_or_none(dico, key):
    if key in dico:
        return dico[key]
    else:
        return None


import re

from .trees import Hierarchy

from .git_adapter import PythonGitAdapter

class HierarchyHandler(object):

    def __init__(self, path, git_adapter=None):
        if not git_adapter:
            git_adapter = PythonGitAdapter
        self.git = git_adapter(path)

    def get_branch_hierarchy(self, feature_id, branch_regexes):
        """
        Get a full branch hierarchy tree for the given feature.

        Each layer of the tree has its own list entry. Each layer is itself
        a list of 1 or more branches, each containing a tuple of:
          - Branch name
          - HEAD

        In time, we may want to support regex matching against other things
        than just feature_id.
        """
        hierarchy = Hierarchy()
        for branch, commit in self.git.get_branches_and_heads():
            for i, regex in enumerate(branch_regexes):
                if re.search(regex.format(feature_id=feature_id),
                             branch):
                    hierarchy[i].append((branch, commit))

        return hierarchy

    def get_branch_merge_statuses(self, hierarchy, branch_name):
        """
        Get the merge status of the given branch and its children from the
        given hierarchy.

        :returns:
            A list of dicts containing:
             - 'head'
             - 'merged_into_parent'
             - 'parent_name'
             - 'branch_name'
            for the given node and its children

        """
        node = hierarchy.get_node(branch_name)
        all_nodes = [node]
        all_nodes.extend(node.children)

        statuses = []
        for node in all_nodes:
            parent_name = node.parent.branch if node.parent else None
            merged_into_parent = False
            if parent_name:
                merged_into_parent = self.git.is_contained(
                    node.commit, parent_name)

            statuses.append({
                'head': node.commit,
                'parent_name': parent_name,
                'merged_into_parent': merged_into_parent,
                'branch_name': node.branch,
            })

        return statuses

def match_features(features, regexes, branch_name):
    """
    Get a list of features that match the branch_name using the regex given
    """
    matched = []
    for feature in features:
        feature_id = feature['feature_id']
        for regex in regexes:
            if re.search(regex.format(feature_id=feature_id), branch_name):
                matched.append(feature)
    return matched


# Read text from a file, and count the occurence of words in that text
# Example:
# count_words("The cake is done. It is a big cake!")
# --> {"cake":2, "big":1, "is":2, "the":1, "a":1, "it":1}

def read_file_content(filename):
    #opening the file
    with open("./story.txt", "r") as openingfile:
        read_file_content = openingfile.read()
    return read_file_content

def count_words():
    text = read_file_content("./story.txt")
    # dividing sentences into words
    split_text = text.split()

    #creating a dictionary to store the words
    count = {}
    #looping through each word and counting each appearance
    for word in split_text:
        if word in count:
            count[word] += 1
        else:
            count[word] = 1
    return count

print(count_words())


# -*- coding: utf-8 -*-
"""
Created on Fri Jan 20 20:40:00 2023

@author: basti
"""

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

opt = ""
opt2 = ""

#IMPORT DU FICHIER
data = pd.read_csv("bdd/data/data_ml_"+str(opt)+"22-23.csv", sep= ";", index_col = 0)
nb_top = pd.read_csv("bdd/data/nb_top_joueurs"+str(opt2)+".csv",sep=";", index_col = 0)
nb_top_precis = pd.read_csv("bdd/data/nb_top_joueurs_poste_precis.csv",sep=";", index_col = 0)
opp_score = pd.read_csv("bdd/data/score_dis_opp.csv",sep=";", index_col = 0)

nb_top_precis = nb_top_precis.fillna(0)
nb_top_precis = nb_top_precis.rename(
    columns = {
        "DF":"DF2",
        "DM":"DM2",
        "GK":"GK2"
        })
data = data.dropna()

data.team.value_counts()

data = data.rename( columns = {
    "Cmp%" : "Cmp"})

desc = data.describe()
desc.to_csv("Sorties/base_descript.csv",sep=";")

data["FDD"] = (data["Int"]/(data.Int.quantile([0.75]).values)) - ((data["opp_Sh"]/(data.opp_Sh.quantile([0.75]).values)) +
                                                (data["Fls"]/(data.Fls.quantile([0.75]).values)) +
                                                (data["coup_arret"]/(data.coup_arret.quantile([0.75]).values)))

data["FDA"] = ((data["Sh"]/(data.Sh.quantile([0.75]).values)) +
               (data["opp_fls"]/(data.opp_fls.quantile([0.75]).values)) +
               (data["Cmp"]/(data.Cmp.quantile([0.75]).values))) - (data["Dist"]/(data.Dist.quantile([0.75]).values))

Att = data.groupby("team").mean("FDA").reset_index()
Att = Att[["team","FDA"]].sort_values(by = ["FDA"],ascending=False)
Att.head

Def = data.groupby("team").mean("FDD").reset_index()
Def = Def[["team","FDD"]].sort_values(by = ["FDD"],ascending=True)
Def.head

# nb_top['Squad'] = nb_top['Squad'].replace('Hellas Verona','Hellas-Verona')
# nb_top['Squad'] = nb_top['Squad'].replace('Saint-Étienne','Saint-Etienne')
# nb_top['Squad'] = nb_top['Squad'].replace('Paris S-G','Paris-Saint-Germain')

data = data.merge(nb_top, how = "left", left_on = "team", right_on = "Squad")
data = data.merge(nb_top_precis, how = "left", left_on = "team", right_on = "Squad")
data = data.merge(opp_score, how = "left", on = "Opponent")
data.to_csv("bdd/data/data_temp_2122.csv", sep= ";")

df_reg = data.copy()
#df_reg = df_reg.loc[df_reg["team"] == "Milan"]
df_reg=df_reg.dropna()

df_reg.columns
df_reg.info()
a = df_reg.opp_Formation.value_counts()
a = a.reset_index()
a = a.rename(columns={"index":"opp_Formation", "opp_Formation":"n_form"})

df_reg = df_reg.merge(a, how="left", on="opp_Formation")
df_reg = df_reg.loc[df_reg["n_form"] >= 30]

b = df_reg.Formation.value_counts()
b = b.reset_index()
b = b.rename(columns={"index":"Formation", "Formation":"n"})

df_reg = df_reg.merge(b, how="left", on="Formation")
df_reg = df_reg.loc[df_reg["n"] >= 30]

corr = df_reg.corr()

df_reg = df_reg.loc[df_reg["PK"] < 3]

df_reg["top_GK"] = 0
df_reg["top_MF"] = 0
df_reg["top_DM"] = 0
df_reg["top_DF"] = 0
df_reg["top_FW"] = 0
df_reg["top_MO"] = 0
df_reg["top_AIL"] = 0
df_reg["top_MIL"] = 0

for i in df_reg.index :
    if df_reg["GK"][i] > 0:
        df_reg["top_GK"][i] = 1
for i in df_reg.index :
    if df_reg["MF"][i] > 0:
        df_reg["top_MF"][i] = 1
for i in df_reg.index :
    if df_reg["DM"][i] > 0:
        df_reg["top_DM"][i] = 1
for i in df_reg.index :
    if df_reg["DF"][i] > 0:
        df_reg["top_DF"][i] = 1
for i in df_reg.index :
    if df_reg["FW"][i] > 0:
        df_reg["top_FW"][i] = 1
for i in df_reg.index :
    if df_reg["MO"][i] > 0:
        df_reg["top_MO"][i] = 1
for i in df_reg.index :
    if df_reg["AIL"][i] > 0:
        df_reg["top_AIL"][i] = 1
for i in df_reg.index :
    if df_reg["MIL"][i] > 0:
        df_reg["top_MIL"][i] = 1

df_reg['nb_def'] = pd.Series(df_reg["Formation"].str[0:1])
df_reg['nb_opp_def'] = pd.Series(df_reg["opp_Formation"].str[0:1])

df_reg['nb_att'] = pd.Series(df_reg["Formation"].str[4:5])
df_reg['nb_opp_att'] = pd.Series(df_reg["opp_Formation"].str[4:5])
df_reg['nb_mil'] = pd.Series(df_reg["Formation"].str[2:3])
df_reg['nb_opp_mil'] = pd.Series(df_reg["opp_Formation"].str[2:3])

df_reg['nb_def'] = pd.to_numeric(df_reg['nb_def'], errors='coerce').convert_dtypes(int)
df_reg['nb_opp_def'] = pd.to_numeric(df_reg['nb_opp_def'], errors='coerce').convert_dtypes(int)
df_reg['nb_att'] = pd.to_numeric(df_reg['nb_att'], errors='coerce').convert_dtypes(int)
df_reg['nb_opp_att'] = pd.to_numeric(df_reg['nb_opp_att'], errors='coerce').convert_dtypes(int)
df_reg['nb_mil'] = pd.to_numeric(df_reg['nb_mil'], errors='coerce').convert_dtypes(int)
df_reg['nb_opp_mil'] = pd.to_numeric(df_reg['nb_opp_mil'], errors='coerce').convert_dtypes(int)

df_reg["diff_avant"] = df_reg["nb_att"] - df_reg["nb_opp_def"]
df_reg["diff_def"] = df_reg["nb_def"] - df_reg["nb_opp_att"]
df_reg["diff_mil"] = df_reg["nb_mil"] - df_reg["nb_opp_mil"]

df_reg["nb_def"] = df_reg["nb_def"].astype("int64")
df_reg["nb_opp_att"] = df_reg["nb_opp_att"].astype("int64")

df_reg["diff_avant"] = df_reg["diff_avant"].astype("int64")
df_reg["diff_def"] = df_reg["diff_def"].astype("int64")
df_reg["diff_mil"] = df_reg["diff_mil"].astype("int64")

df_reg["sup_def"] = 0

for i in df_reg.index :
    if df_reg["diff_def"][i] > 0:
        df_reg["sup_def"][i] = 1

df_reg["diff_def"].value_counts()
df_reg.info()

df_reg['opp_Formation'] = pd.Categorical(df_reg['opp_Formation'])

df_reg.to_csv("bdd/data/df_reg_22_23.csv", sep= ";")


from django.urls import path
from .views import MovieList,MovieDetail,MovieCategory,MovieLanguage,MovieSearch,MovieYear,MostWatch
from django.conf.urls import url

app_name='movie'
urlpatterns = [
    path('', MovieList.as_view(),name='Movie_List'),
    path('category/<str:category>', MovieCategory.as_view(),name='MovieCategory'),
    path('language/<str:lang>', MovieLanguage.as_view(),name='MovieLanguage'),
    path('search/', MovieSearch.as_view(),name='MovieSearch'),
    path('<slug:slug>', MovieDetail.as_view(),name='Movie_Detail'),
    path('year/<int:year>', MovieYear.as_view(),name='MovieYear'),
    path('mostwatch/',MostWatch.as_view(),name='MostWatch')
]


# python3
import sys
class Query:

    def __init__(self, query):
        self.type = query[0]
        if self.type == 'check':
            self.ind = int(query[1])
        else:
            self.s = query[1]

class QueryProcessor:
    _multiplier = 263
    _prime = 1000000007

    def __init__(self, bucket_count):
        self.bucket_count = bucket_count
        # store all strings in one list
        self.elems = [ [] for _ in range(0,bucket_count)]

    def _hash_func(self, s):
        ans = 0
        for c in reversed(s):
            ans = (ans * self._multiplier + ord(c)) % self._prime
        return ans % self.bucket_count

    def write_search_result(self, was_found):
        print('yes' if was_found else 'no')

    def write_chain(self, chain):
        print(' '.join(chain))

    def read_query(self):
        return Query(sys.stdin.readline().split())

    def process_query(self, query):
        if query.type == "check":
            # use reverse order, because we append strings to the end
            # self.write_chain(cur for cur in reversed(self.elems)
            #             if self._hash_func(cur) == query.ind)
            self.write_chain(reversed(self.elems[query.ind]))
        else:
            hash_ = self._hash_func(query.s)
            try:
                ind = self.elems[hash_].index(query.s)
            except ValueError:
                ind = -1
            if query.type == 'find':
                self.write_search_result(ind!=-1)
            elif query.type == 'add':
                if ind == -1:
                    self.elems[hash_].append(query.s)
            else:
                if ind != -1:
                    self.elems[hash_].pop(ind)

    def process_queries(self):
        n = int(input())
        for i in range(n):
            self.process_query(self.read_query())

if __name__ == '__main__':
    bucket_count = int(input())
    proc = QueryProcessor(bucket_count)
    proc.process_queries()


#!flask/bin/python
from flask import Flask, jsonify, request, json
import requests
import time
from core import test_class
from BeautifulSoup import BeautifulSoup

app = Flask(__name__)

@app.route('/ytc/search', methods=['GET'])
def get_tasks():
    start_time = time.time()
    q1 = request.args.get('q')
    url = 'https://www.google.co.in/search?q='+q1
    response = requests.get(url)
    html = response.content
    soup = BeautifulSoup(html)
    div = soup.find('div', attrs={'id': 'res'})
    jsonArr = []
    for row in div.findAll('cite'):
        jsonArr.append(row.text)

    print("--- %s seconds ---" % (time.time() - start_time))
    return jsonify({'links': jsonArr})

@app.route('/ytc/redosearch', methods=['GET'])
def get_tasks_redo():
    w = 'wiki'
    yt = 'youtuber'
    start_time = time.time()
    q1 = request.args.get('q')
    jsonArr = []
    url = 'https://www.google.co.in/search?q='+q1+w
    response = requests.get(url)
    html = response.content
    soup = BeautifulSoup(html)
    div = soup.find('div', attrs={'id': 'res'})
    for row in div.findAll('cite'):
        jsonArr.append(row.text)

    url = 'https://www.google.co.in/search?q='+q1+yt
    response = requests.get(url)
    html = response.content
    soup = BeautifulSoup(html)
    div = soup.find('div', attrs={'id': 'res'})
    for row in div.findAll('cite'):
        jsonArr.append(row.text)

    jsonArr =  [k for k in jsonArr if 'wikipedia' in k]
    print("--- %s seconds ---" % (time.time() - start_time))
    return jsonify({'links': jsonArr})

tasks = [
    {
        'Approve': 72,
        'Spam': 18.5,
        'Hate Speech': 2.3,
        'Far Right': 0,
        'Harassment' : 7.2
    }
]

@app.route('/pace/predict', methods=['GET'])
def get_pace_data():
    return jsonify(tasks)

@app.route('/pace/message', methods = ['POST'])
def api_message():

    if request.headers['Content-Type'] == 'text/plain':
        return "Text Message: " + request.data

    elif request.headers['Content-Type'] == 'application/json':
        return "JSON Message: " + json.dumps(request.json)

    else:
        return "415 Unsupported Media Type ;)"

@app.route('/test', methods=['GET'])
def get_test():
    f = test_class.Fridge()
    return jsonify(f.in_fridge('apples'))

if __name__ == '__main__':
    app.run(port=5001, debug=True)


# -*- coding: utf-8 -*-
'''
Created on 2015年9月29日

@author: michael
'''
from openerp import models, fields, api, _

class move_transfer_wizard(models.TransientModel):
    _name='move.transfer.wizard'

    stock_quant_ids = fields.Many2many('stock.quant','stock_quant_move_transfter_ref','wizard_id','stock_quant_id', string='庫存明細', help='')

    @api.multi
    def action_addline(self):
        mt_obj=self.env["move.transfer"]
        move_id=self.env.context.get("active_id",False)
        if move_id:
           mt = mt_obj.browse(move_id)
           lines= []
           for x in self["stock_quant_ids"]:
               if x.location_id.id == mt.location_id.id:
                  lines.append((0,0,{"lot_id":x.lot_id.id,
                                     "product_id":x.product_id.id,
                                     "product_uom_id":x.product_id.uom_id.id,
                                     "product_qty":x.qty}))
           if lines:
               mt.write({"line_ids":lines})
        return {'type': 'ir.actions.act_window_close'}


import threading

import mock
import pytest

from blazingmq import BasicHealthMonitor
from blazingmq import Error
from blazingmq import QueueOptions
from blazingmq import Session
from blazingmq import session_events
from blazingmq.testing import HostHealth

def test_receiving_host_health_events():
    # GIVEN
    spy = mock.MagicMock()
    host_health = BasicHealthMonitor()
    toggled_health_twice = threading.Event()
    host_healthy_events = []

    def callback(*args):
        spy(*args)

        if isinstance(args[0], session_events.HostHealthRestored):
            host_healthy_events.append(args[0])

        if len(host_healthy_events) == 2:
            toggled_health_twice.set()

    # WHEN
    session = Session(callback, host_health_monitor=host_health)
    host_health.set_unhealthy()
    host_health.set_healthy()
    host_health.set_unhealthy()
    host_health.set_healthy()
    toggled_health_twice.wait()
    session.stop()

    # THEN
    assert spy.call_args_list == [
        mock.call(session_events.Connected(None)),
        mock.call(session_events.HostUnhealthy(None)),
        mock.call(session_events.HostHealthRestored(None)),
        mock.call(session_events.HostUnhealthy(None)),
        mock.call(session_events.HostHealthRestored(None)),
        mock.call(session_events.Disconnected(None)),
    ]

def test_enabling_real_host_health_monitoring():
    # GIVEN
    spy = mock.MagicMock()

    def callback(*args):
        spy(*args)

    # WHEN
    session = Session(callback)
    session.stop()

    # THEN
    assert spy.call_args_list == [
        mock.call(session_events.Connected(None)),
        mock.call(session_events.Disconnected(None)),
    ]

def test_disabling_host_health_monitoring():
    # GIVEN
    spy = mock.MagicMock()

    def callback(*args):
        spy(*args)

    # WHEN
    session = Session(callback, host_health_monitor=None)
    session.stop()

    # THEN
    assert spy.call_args_list == [
        mock.call(session_events.Connected(None)),
        mock.call(session_events.Disconnected(None)),
    ]

def test_queue_suspension(unique_queue):
    # GIVEN
    host_health = HostHealth()
    queue_suspended_event_received = threading.Event()

    def on_session_event(event):
        print(event)
        if isinstance(event, session_events.QueueSuspended):
            queue_suspended_event_received.set()

    session = Session(on_session_event, host_health_monitor=host_health)

    session.open_queue(
        unique_queue,
        read=False,
        write=True,
        options=QueueOptions(suspends_on_bad_host_health=True),
    )

    # WHEN
    session.post(unique_queue, b"blah")

    host_health.set_unhealthy()
    queue_suspended_event_received.wait()

    with pytest.raises(Exception) as exc:
        session.post(unique_queue, b"blah")

    session.stop()

    # THEN
    assert exc.type is Error
    assert exc.match("QUEUE_SUSPENDED")


import os
import sys
import warnings
import platform
import datetime as dt
import numpy as np
import matplotlib.pyplot as mp
import matplotlib.dates as md

def dmy2ymd(dmy):
    return dt.datetime.strptime(
        str(dmy, encoding='utf-8'),
        '%d-%m-%Y').date().strftime('%Y-%m-%d')

def read_data(filename):
    dates, closing_prices = np.loadtxt(
        filename, delimiter=',', usecols=(1, 6),
        unpack=True, dtype=np.dtype('M8[D], f8'),
        converters={1: dmy2ymd})
    return dates, closing_prices

def calc_returns(N, closing_prices):
    returns = np.diff(closing_prices) / closing_prices[:-1]
    weights = np.hanning(N)
    smr = np.convolve(weights, returns, 'valid')
    return returns, smr

def fit_polys(fit_x, fit_y, fit_d, poly_x):
    fit_p = np.polyfit(fit_x, fit_y, fit_d)
    poly_y = np.polyval(fit_p, poly_x)
    return poly_y

def find_inters(fit_x, fit_y1, fit_y2,
                fit_d, min_x, max_x):
    fit_p1 = np.polyfit(fit_x, fit_y1, fit_d)
    fit_p2 = np.polyfit(fit_x, fit_y2, fit_d)
    fit_p3 = np.polysub(fit_p1, fit_p2)
    roots = np.roots(fit_p3)
    reals = roots[np.isreal(roots)].real
    inters = []
    for real in reals:
        if min_x < real and real < max_x:
            inters.append([real,
                           np.polyval(fit_p1, real)])
    inters.sort()
    inters = np.array(inters)
    return inters

def init_chart(N, first_day, last_day):
    mp.gcf().set_facecolor(np.ones(3) * 240 / 255)
    mp.title('Smoothing Returns (%d Days)' % N, fontsize=20)
    mp.xlabel('Trading Days From %s To %s' % (
        first_day.astype(md.datetime.datetime).strftime(
            '%d %b %Y'),
        last_day.astype(md.datetime.datetime).strftime(
            '%d %b %Y')), fontsize=14)
    mp.ylabel('Returns Of Stock Price', fontsize=14)
    ax = mp.gca()
    ax.xaxis.set_major_locator(
        md.WeekdayLocator(byweekday=md.MO))
    ax.xaxis.set_minor_locator(md.DayLocator())
    ax.xaxis.set_major_formatter(
        md.DateFormatter('%d %b %Y'))

    mp.tick_params(which='both', top=True, right=True,
                   labelright=True, labelsize=10)
    mp.grid(linestyle=':')

def draw_returns(dates, bhp_returns, vale_returns):
    dates = dates.astype(md.datetime.datetime)
    mp.plot(dates, bhp_returns, '-', c='orangered',
            alpha=0.25, label='BHP Returns')
    mp.plot(dates, vale_returns, '-', c='dodgerblue',
            alpha=0.25, label='VALE Returns')

def draw_smrs(N, dates, bhp_smrs, vale_smrs):
    dates = dates.astype(md.datetime.datetime)
    mp.plot(dates, bhp_smrs, '-', c='orangered',
            alpha=0.75, label='BHP SMR-%d' % N)
    mp.plot(dates, vale_smrs, '-', c='dodgerblue',
            alpha=0.75, label='VALE SMR-%d)' % N)

def draw_polys(N, dates, bhp_polys, vale_polys, degree):
    dates = dates.astype(md.datetime.datetime)
    mp.plot(dates, bhp_polys, '-', c='orangered',
            label='BHP SMR-%d Polynomial (%d)' % (
                N, degree))
    mp.plot(dates, vale_polys, '-', c='dodgerblue',
            label='VALE SMR-%d Polynomal (%d)' % (
                N, degree))

def draw_inters(inters):
    dates, inters = np.hsplit(inters, 2)
    dates = dates.astype(int).astype(
        'M8[D]').astype(md.datetime.datetime)
    mp.scatter(dates, inters, marker='X', s=120,
               c='firebrick',
               label='Intersection of SMRs', zorder=3)

    mp.gcf().autofmt_xdate()
    mp.legend()

def show_chart():
    mng = mp.get_current_fig_manager()
    if 'Windows' in platform.system():
        mng.window.state('zoomed')
    else:
        mng.resize(*mng.window.maxsize())
    mp.show()

def main(argc, argv, envp):
    warnings.filterwarnings('ignore',
                            category=np.RankWarning)
    dates, bhp_closing_prices = read_data('bhp.csv')
    dates, vale_closing_prices = read_data('vale.csv')
    N = 8
    bhp_returns, bhp_smrs = calc_returns(
        N, bhp_closing_prices)
    vale_returns, vale_smrs = calc_returns(
        N, vale_closing_prices)
    days = dates[N - 1:-1].astype(int)
    degree = 5
    bhp_ploys = fit_polys(days, bhp_smrs, degree, days)
    vale_ploys = fit_polys(days, vale_smrs, degree, days)
    inters = find_inters(days, bhp_smrs, vale_smrs,
                         degree, days[0], days[-1])
    init_chart(N, dates[0], dates[-2])
    draw_returns(dates[:-1], bhp_returns, vale_returns)
    draw_smrs(N, dates[N - 1:-1], bhp_smrs, vale_smrs)
    draw_polys(N, dates[N - 1:-1], bhp_ploys, vale_ploys,
               degree)
    draw_inters(inters)
    show_chart()
    return 0

if __name__ == '__main__':
    sys.exit(main(len(sys.argv), sys.argv, os.environ))


import ssl
import re
import traceback
from urllib.request import Request, urlopen
import requests
import json
from careerjet_api import CareerjetAPIClient
from bs4 import BeautifulSoup, SoupStrainer
from concurrent.futures import ThreadPoolExecutor

# ---------------------------------------------------------------------------- #
def getFullJobDesc(jobsDict):
    descriptions = []
    urls = []
    try:
        jobsArr = jobsDict['jobs']

        for element in jobsArr:
            urls.append(str(element['url']))

        print("\nUrls:", urls[0:4])

        def get_url(url):
            return requests.get(url).text

        with ThreadPoolExecutor(max_workers=50) as pool:
            resList = list(pool.map(get_url, urls))
            print("Threads done")

        for element in resList:
            soup = BeautifulSoup(element, 'html.parser')
            print(soup.prettify())
            pageNumberArr = soup.find_all("script")[1]
            pageNumberArr = str(pageNumberArr).replace('",
                                                                                "")
            res = json.loads(pageNumberArr)['description']
            res = res.split('
')
            print("\nInitial parsing finished")
            for part in res:
                if part == ' ' or part == '':
                    res.remove(part)

            res = "".join(res)
            description = res.replace("", "").replace("", "").replace("
", "").replace("
",
                                                                                "").replace(
                "
", "").replace("
", "").replace("Posting Notes:", "")
            print("\nDescription is cleaned")
            descriptions.append(description)
        return descriptions

    except Exception as err:
        traceback.print_exc()
        return " "

# ---------------------------------------------------------------------------- #
if __name__ == "__main__":

    try:
        cj = CareerjetAPIClient("en_US")
        location = 'san francisco'
        keywords = 'data scientist'
        pageNum = '0'

        if keywords is not None or location is not None:
            result_json = cj.search({
                'location': location,
                'keywords': keywords,
                'affid': '5f41b19494be1f6eaab8a755b612e343',
                'user_ip': '11.22.33.44',
                'url': 'http://www.example.com/jobsearch?q=python&l=london',
                'user_agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:31.0) Gecko/20100101 Firefox/31.0',
                'pagesize': '99',
                'page': pageNum,
            })

    except Exception as e:
        print(f'API query failed\nFailed to retrieve jobs from CareerJet API with info: loc={location}, keywords={keywords}, pageNum={pageNum}')
        print(f'Error:\n{e}')

    if len(result_json['jobs']) != 99 or type(result_json) is not dict:
        raise Exception('CareerJet API response did not pass testing. Response:', result_json)

    print('Size of jobs:', len(result_json['jobs']))

    descriptions = getFullJobDesc(result_json)

    print(descriptions)


import logging as logger
import os

from pyspark.sql import DataFrame

from ddataflow.exceptions import BiggerThanMaxSize
from ddataflow.sampling.default import filter_function
from ddataflow.utils import get_or_create_spark

class DataSource:
    """
    Utility functions at data source level
    """

    def __init__(
        self,
        *,
        name: str,
        config: dict,
        local_data_folder: str,
        snapshot_path: str,
        size_limit,
    ):
        self._name = name
        self._local_data_folder = local_data_folder
        self._snapshot_path = snapshot_path
        self._size_limit = size_limit
        self._config = config
        self._filter = None
        self._source = None

        if "source" in self._config:
            self._source = config["source"]
        else:
            if self._config.get("file-type") == "parquet":
                self._source = lambda spark: spark.read.parquet(self._name)
            else:
                self._source = lambda spark: spark.table(self._name)

        if "filter" in self._config:
            self._filter = self._config["filter"]
        else:
            if self._config.get("default_sampling"):
                self._filter = lambda df: filter_function(df)

    def query(self):
        """
        query with filter unless none is present
        """
        df = self.query_without_filter()

        if self._filter is not None:
            print(f"Filter set for {self._name}, applying it")
            df = self._filter(df)
        else:
            print(f"No filter set for {self._name}")

        return df

    def has_filter(self) -> bool:
        return self._filter is not None

    def query_without_filter(self):
        """
        Go to the raw data source without any filtering
        """
        spark = get_or_create_spark()
        logger.debug(f"Querying without filter source: '{self._name}'")
        return self._source(spark)

    def query_locally(self):
        logger.info(f"Querying locally {self._name}")

        path = self.get_local_path()
        if not os.path.exists(path):
            raise Exception(
                f"""Data source '{self.get_name()}' does not have data in {path}.
            Consider downloading using  the following command:
            ddataflow current_project download_data_sources"""
            )
        spark = get_or_create_spark()
        df = spark.read.parquet(path)

        return df

    def get_dbfs_sample_path(self) -> str:
        return os.path.join(self._snapshot_path, self._get_name_as_path())

    def get_local_path(self) -> str:
        return os.path.join(self._local_data_folder, self._get_name_as_path())

    def _get_name_as_path(self):
        """
        converts the name when it has "/mnt/envents" in the name to a single file in a (flat structure) _mnt_events
        """
        return self.get_name().replace("/", "_")

    def get_name(self) -> str:
        return self._name

    def get_parquet_filename(self) -> str:
        return self._name + ".parquet"

    def estimate_size_and_fail_if_too_big(self):
        """
        Estimate the size of the data source use the _name used in the _config
        It will throw an exception if the estimated size is bigger than the maximum allowed in the configuration
        """

        print("Estimating size of data source: ", self.get_name())
        df = self.query()
        size_estimation = self._estimate_size(df)

        print("Estimated size of the Dataset in GB: ", size_estimation)

        if size_estimation > self._size_limit:
            raise BiggerThanMaxSize(self._name, size_estimation, self._size_limit)

        return df

    def _estimate_size(self, df: DataFrame) -> float:
        """
        Estimates the size of a dataframe in Gigabytes

        Formula:
            number of gigabytes = (N*V*W) / 1024^3
        """

        print(f"Amount of rows in dataframe to estimate size: {df.count()}")
        average_variable_size_bytes = 50
        return (df.count() * len(df.columns) * average_variable_size_bytes) / (
            1024**3
        )


from abc import abstractmethod

class URLFormat:
    @abstractmethod
    def build_url(self, qs):
        # type: (dict) -> str
        ...

class URLFormats:
    class _Detail(URLFormat):
        URL_FORMAT = (
            "https://play.google.com/store/apps/details?id={id}&hl={hl}&gl={gl}"
        )

        def build_url(self, qs):
            return self.URL_FORMAT.format(**qs)

    Detail = _Detail()


import numpy as np
import cv2
import glob
import os
import argparse

parser = argparse.ArgumentParser()
parser.add_argument("--rgb_path",     type=str,   help="raw image path", default='./data_augmentation/raw_data/bg_img/select/total/')
# parser.add_argument("--mask_path",     type=str,   help="raw image path", default='./data_augmentation/raw_data/bg_img/select/select_rgb/')
# parser.add_argument("--obj_mask_path",     type=str,   help="raw image path", default='./data_augmentation/raw_data/obj_mask/')
# parser.add_argument("--bg_path",     type=str,   help="raw image path", default='./data_augmentation/raw_data/bg_img/select/select_rgb/')
parser.add_argument("--output_path",     type=str,   help="raw image path", default='./data_augmentation/raw_data/bg_img/save_bg/')

args = parser.parse_args()

if __name__ == '__main__':

    os.makedirs(args.output_path, exist_ok=True)

    rgb_path = os.path.join(args.rgb_path, '*.jpg')
    rgb_list = glob.glob(rgb_path)

    idx = 0
    for rgb_idx in rgb_list:
        print(rgb_idx)
        idx += 1
        file_name = rgb_idx.split('/')[5]
        file_name = file_name.split('.')[0]

        # if not os.path.isfile(args.obj_mask_path + file_name + '.png'):
        rgb_img = cv2.imread(rgb_idx)
        zero_mask = np.zeros(rgb_img.shape[:2], dtype=np.uint8)
        zero_mask = np.expand_dims(zero_mask, axis=-1)

        cv2.imwrite(args.output_path + 'rgb/' + 'bg2_image_idx_{0}_'.format(idx) + '_rgb.jpg', rgb_img)
        cv2.imwrite(args.output_path + 'gt/' + ' bg2_image_idx_{0}_'.format(idx) + '_mask.png', zero_mask)
        # cv2.imwrite(args.obj_mask_path + file_name + '.png', zero_mask)


"""
Code for running experiments for
"Asymmetric Tri-training for Debiasing Missing-Not-At-Random Explicit Feedback".
"""
import argparse
import yaml
import warnings

import tensorflow as tf

from trainer import Trainer, Tuner

possible_model_names = ['uniform', 'uniform-at', 'user', 'user-at', 'item', 'item-at',
                        'both', 'both-at', 'nb', 'nb-at', 'nb_true', 'nb_true-at']

parser = argparse.ArgumentParser()
parser.add_argument('--data', '-d', type=str, required=True)
parser.add_argument('--model_name', '-m', type=str,
                    choices=possible_model_names, required=True)
parser.add_argument('--tuning', '-t', action='store_true')

if __name__ == "__main__":
    warnings.filterwarnings("ignore")
    tf.get_logger().setLevel("ERROR")
    args = parser.parse_args()

    # hyper-parameters
    config = yaml.safe_load(open('../config.yaml', 'r'))
    eta = config['eta']
    batch_size = config['batch_size']
    max_iters = config['max_iters']
    pre_iters = config['pre_iters']
    post_steps = config['post_steps']
    post_iters = config['post_iters']
    num_sims = config['num_sims']
    n_trials = config['n_trials']
    model_name = args.model_name
    tuning = args.tuning
    data = args.data

    if tuning:
        tuner = Tuner(data=data, model_name=model_name)
        tuner.tune(n_trials=n_trials)

        print('\n', '=' * 25, '\n')
        print(f'Finished Tuning of {model_name}!')
        print('\n', '=' * 25, '\n')

    trainer = Trainer(data=data, batch_size=batch_size, max_iters=max_iters,
                      pre_iters=pre_iters, post_steps=post_steps,
                      post_iters=post_iters, eta=eta, model_name=model_name)
    trainer.run_simulations(num_sims=num_sims)

    print('\n', '=' * 25, '\n')
    print(f'Finished Running {model_name}!')
    print('\n', '=' * 25, '\n')


import os
import glob
import shutil
import re
import gi
gi.require_version('Limba', '1.0')
gi.require_version('AppStream', '1.0')
from gi.repository import Limba
from gi.repository import AppStream
from hashlib import sha256

from ..utils import build_cpt_path
from ..repository.models import *
from ..user import User
from ..extensions import db
from ..utils import get_current_time
from .dscfile import DSCFile

class IPKImporter():
    def __init__(self, search_dir):
        self._import_dir = search_dir
        self._asmdata = AppStream.Metadata()
        self._asmdata.set_locale("C")

        self._xdg_cat_map = { 'AudioVideo': self._category_by_id("multimedia"),
                        'Audio': self._category_by_id("multimedia"),
                        'Video': self._category_by_id("multimedia"),
                        'Development': self._category_by_id("development"),
                        'Education': self._category_by_id("education"),
                        'Game': self._category_by_id("games"),
                        'Graphics': self._category_by_id("graphics"),
                        'Network': self._category_by_id("network"),
                        'Office': self._category_by_id("customization"),
                        'Science': self._category_by_id("science"),
                        'Settings': self._category_by_id("tools"),
                        'System': self._category_by_id("system"),
                        'Utility': self._category_by_id("tools"),

                        'Arcade': self._category_by_id("arcade")
                      }

    def _category_by_id(self, cat_name):
        return Category.query.filter_by(idname=cat_name).one()

    def _map_categories(self, cpt):
        xdg_cats = cpt.get_categories()
        if cpt.get_kind() != AppStream.ComponentKind.DESKTOP:
            return [Category.query.filter_by(idname="components").one()]
        if not xdg_cats:
            return [Category.query.filter_by(idname="other").one()]

        cats = list()
        for xcat in xdg_cats:
            cat = self._xdg_cat_map.get(xcat)
            if cat:
                cats.append(cat)
        if len(cats) == 0:
            return [Category.query.filter_by(idname="other").one()]

        return cats

    def _import_package(self, pkg_fname, sha256sum, dsc):
        pkg = Limba.Package()
        pkg.open_file(pkg_fname)

        if pkg.has_embedded_packages():
            self._reject_dsc("Package contains embedded packages. This is not allowed in repositories.", dsc)
            return

        pki = pkg.get_info()

        cpt_xml = pkg.get_appstream_data()

        self._asmdata.clear_components()
        self._asmdata.parse_data(cpt_xml)
        cpt = self._asmdata.get_component()

        pkgid = pkg.get_id()
        cptname = pki.get_name()
        arch = pki.get_architecture()

        dest_pkgfname = "%s_%s.ipk" % (pkgid.replace("/", "-"), arch)

        cpt_desc = cpt.get_description()
        if not cpt_desc:
            cpt_desc = "
A software component
"

        # we just accept packages for the master repository for now
        repo_name = dsc.get_val('Target')
        repo = None
        try:
            repo = Repository.query.filter_by(name=repo_name).one()
        except:
            self._reject_dsc("Could not find target repository: %s" % (repo_name), dsc)
            return

        repo_pool_path = os.path.join(repo.root_dir, "pool", build_cpt_path (cptname))
        repo_icons_path = os.path.join(repo.root_dir, "assets", build_cpt_path (cptname), pki.get_version(), "icons")
        pkg_dest = os.path.join(repo_pool_path, dest_pkgfname)
        repo_location = os.path.join(build_cpt_path (cptname), dest_pkgfname)

        dbcpt = Component(
            cid=cpt.get_id(),
            kind=AppStream.ComponentKind.to_string(cpt.get_kind()),
            sdk=True if pki.get_kind() == Limba.PackageKind.DEVEL else False,
            name=cpt.get_name(),
            summary=cpt.get_summary(),
            description=cpt_desc,
            developer_name=cpt.get_developer_name(),
            url=cpt.get_url(AppStream.UrlKind.HOMEPAGE),
            xml=cpt_xml,
            repository=repo
            )
        dbcpt.categories = self._map_categories(cpt)
        db.session.add(dbcpt)

        dbpkg = Package(
            name=pki.get_name(),
            version=pki.get_version(),
            kind=PackageKind.SDK if pki.get_kind() == Limba.PackageKind.DEVEL else PackageKind.COMMON,
            fname=repo_location,
            architecture=arch,
            sha256sum=sha256sum,
            dependencies=pki.get_dependencies(),
            component=dbcpt,
            repository=repo
            )
        db.session.add(dbpkg)

        pkg.extract_appstream_icons(repo_icons_path)

        if not os.path.exists(repo_pool_path):
            os.makedirs(repo_pool_path)
        shutil.copyfile(pkg_fname, pkg_dest)

    def _reject_dsc(self, reason, dsc):
        print("REJECT: %s => %s" % (reason, str(dsc)))
        # TODO: Actually reject the package and move it to the morgue

    def _process_dsc(self, dscfile):
        dsc = DSCFile()
        dsc.open(dscfile)

        uploader = dsc.get_val('Uploader')
        if not uploader:
            self._reject_dsc("Uploader field was not set.", dsc)
            return
        m = re.findall(r'<(.*?)>', uploader)
        if not m:
            self._reject_dsc("Unable to get uploader email address.", dsc)
            return

        user = None
        try:
            user = User.query.filter_by(email=m[0]).one()
        except:
            self._reject_dsc("Could not find user '%s'" % (uploader), dsc)
            return

        key = None
        try:
            key = dsc.validate(user.gpghome)
            key = key.replace(' ', '')
        except Exception as e:
            self._reject_dsc("Validation failed: %s" % (str(e)), dsc)
            return

        if key != user.pgpfpr:
            self._reject_dsc("Validation failed: Fingerprint does not match user", dsc)
            return

        # if we are here, everything is fine - we can import the packages if their checksums match
        for sha256sum, fname in dsc.get_files().items():
            real_sha256 = None
            fname_full = os.path.join(self._import_dir, fname)
            with open(fname_full, 'rb') as f:
                real_sha256 = sha256(f.read()).hexdigest()
            if real_sha256 != sha256sum:
                self._reject_dsc("Validation failed: Checksum mismatch for '%s'" % (fname), dsc)
                return
            self._import_package(fname_full, sha256sum, dsc)

    def import_packages(self):
        for fname in glob.glob(self._import_dir+"/*.dsc"):
            self._process_dsc(fname)
        db.session.commit()


#encoding=utf-8
import sktensor
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import sys
import os
import scipy.io as scio
import numpy as np
np.seterr(divide='ignore', invalid='ignore')
from random import random
from scipy.sparse import rand as sprand
from compt_methods import *
from road_similar import *
import seaborn as sns

#根据确定的数据维度大小获取数据，对于缺失部分补平均数
def get_tensor(mat_file,ori_file,size_=0):
    ori_data = scio.loadmat(mat_file)['Speed']
    if size_:
        p,q,r = size_
        ori_data = ori_data[:p,:q,:r]
    scio.savemat(ori_file,{'Speed':ori_data})
    return ori_data

#根据缺失率和原始数据进行随机缺失并保存缺少数据
def gene_rand_sparse(ori_data, miss_ratio,miss_file):
    data = ori_data.copy()
    dshape = np.shape(data)
    rand_ts = np.random.random_sample(dshape)
    zero_ts = np.zeros(dshape)
    data = data*(rand_ts>miss_ratio) + zero_ts*(rand_ts<=miss_ratio)
    #if not os.path.exists(miss_file):
    scio.savemat(miss_file,{'Speed':data})
    return data

#根据缺失率和原始数据进行连续缺失并保存缺失数据
def gene_cont_sparse(ori_data,miss_ratio,miss_file):
    data = ori_data.copy()
    dshape = data.shape
    rand_ts = np.random.random_sample(dshape[:-1])
    S = np.rint(rand_ts+0.5-miss_ratio)
    W_cont = np.zeros(dshape)
    for k in range(dshape[2]):
        W_cont[:,:,k] = S[:,:]
    data = data*W_cont
    scio.savemat(miss_file,{'Speed':data})
    return data

gene_sparse = {'rand':gene_rand_sparse,'cont':gene_cont_sparse}

#获取缺失数据，缺失位置和真实缺失率
def get_sparsedata(miss_file):
    #miss_loc = []
    sparse_data = scio.loadmat(miss_file)['Speed']
    dshape = np.shape(sparse_data)
    W_miss = sparse_data==0
    true_miss_ratio = W_miss.sum()/sparse_data.size
    return sparse_data,W_miss,true_miss_ratio

#处理原始的缺失数据
def deal_orimiss(ori_data,shorten = False):
    p,q,r = ori_data.shape
    zero_mat = np.zeros((p,q,r))
    sparse_data = (ori_data>1)*ori_data+(ori_data<=1)*zero_mat
    W = sparse_data>0
    W_miss = np.where(sparse_data<=1)
    M_pos = np.where(W_miss)
    if not shorten:
        for i in range(len(W_miss[0])):
            pos1,pos2,pos3 = W_miss[0][i],W_miss[1][i],W_miss[2][i]
            neigh_info = []
            for n2 in (1,-1):
                for n3 in (1,-1):
                    try:
                        temp = sparse_data[pos1,pos2+n2,pos3+n3]
                        neigh_info.append(temp)
                    except:
                        pass
            if sum(neigh_info)>0:
                sparse_data[pos1,pos2,pos3] = sum(neigh_info)/(np.array(neigh_info)>0).sum()
        return sparse_data,W

    Arr = [set(arr.tolist()) for arr in M_pos]
    Arr_len = [len(arr) for arr in Arr]
    Arr_short = Arr_len.index(min(Arr_len))
    sparse_data = np.delete(sparse_data,list(Arr[Arr_short]),Arr_short)
    return sparse_data,W

#用奇异值比例来确定秩
def tk_rank(data,thre = 0.9):
    rank_set = [0,0,0]
    for i in range(3):
        if np.isnan(data).all():
            print(np.isnan(data).sum())
            sys.exit()
        if np.isnan(data).all():
            print(np.isinf(data).sum())
            sys.exit()
        U,sigma,VT = scipy.linalg.svd(dtensor(data).unfold(i),0)
        for r in range(len(sigma)):
            if sum(sigma[:r])/sum(sigma) > thre:
                rank_set[i] = r
                break
    return rank_set

#预填充，这里采用同一线圈同一时段不同日期的平均数填充
def pre_impute(sparse_data,W,day_axis=1,bias_bool = False):
    if not bias_bool:
        pos = np.where(W==False)
        for p in range(len(pos[0])):
            i,j,k = pos[0][p],pos[1][p],pos[2][p]
            if day_axis == 0:
                if (sparse_data[:,j,k]>0).sum()>0:
                    sparse_data[i,j,k] = np.sum(sparse_data[:,j,k])/(sparse_data[:,j,k]>0).sum()
                else:
                    sparse_data[i,j,k] = np.sum(sparse_data[i,:,:])/(sparse_data[i,:,:]>0).sum()
            elif day_axis == 1:
                if (sparse_data[i,:,k]>0).sum()>0:
                    sparse_data[i,j,k] = np.sum(sparse_data[i,:,k])/(sparse_data[i,:,k]>0).sum()
                else:
                    sparse_data[i,j,k] = np.sum(sparse_data[:,j,:])/(sparse_data[:,j,:]>0).sum()
        return sparse_data
    b,b1 = {},{}
    sp = np.shape(sparse_data)
    impute_data = sparse_data.copy()
    mean = np.sum(impute_data)/(impute_data>0).sum()
    for n in range(3):
        b[n] = np.random.uniform(0,0.1,sp[n])
        b1[n] = np.zeros(sp[n])
    seta,miu = 1e-4,800
    sum_F = sum([np.sum(b[n]**2) for n in range(3)])
    J = 1/2*np.sum(W*(sparse_data-impute_data)**2)+miu/2*sum_F
    ite = 0
    while ite < 100:
        ite += 1
        J_pre = J
        for i in range(sp[0]):
            b1[0][i] = (1-seta*miu)*b[0][i]+seta*np.sum(W[i,:,:]*(sparse_data-impute_data)[i,:,:])
        for j in range(sp[1]):
            b1[1][j] = (1-seta*miu)*b[1][j]+seta*np.sum(W[:,j,:]*(sparse_data-impute_data)[:,j,:])
        for k in range(sp[2]):
            b1[2][k] = (1-seta*miu)*b[2][k]+seta*np.sum(W[:,:,k]*(sparse_data-impute_data)[:,:,k])
        if sum([np.sum((b1[n]-b[n])**2)**0.5 for n in range(3)]) < 0.001:
            pass
        for n in range(3):
            b[n] = b1[n].copy()
        for i in range(sp[0]):
            for j in range(sp[1]):
                for k in range(sp[2]):
                    impute_data[i,j,k] = mean+b[0][i]+b[1][j]+b[2][k]
        sum_F = sum([np.sum(b[n]**2) for n in range(3)])
        J = 1/2*np.sum(W*(sparse_data-impute_data)**2)+miu/2*sum_F
        if abs(J-J_pre) < 1:
            break
    print(np.sum(W*(sparse_data-impute_data)**2))
    sparse_data[W==False] = mean+b[0][i]+b[1][j]+b[2][k]
    return sparse_data

#求rmse,mape,mae,rse
def rmse_mape_rse(est_data,ori_data,W):
    S = np.shape(est_data)
    W_miss = (W==False)
    diff_data = np.zeros(S)+W_miss*(est_data-ori_data)
    #diff_data = ori_data-est_data
    rmse = float((np.sum(diff_data**2)/W_miss.sum())**0.5)
    mre_mat=np.zeros_like(est_data)
    mre_mat[W_miss]=np.abs((est_data[W_miss]-ori_data[W_miss])/ori_data[W_miss])
    mape = float(np.sum(mre_mat)/W_miss.sum())*100
    rse = float(np.sum(diff_data**2)**0.5/np.sum(ori_data[W_miss]**2)**0.5)
    mae = float(np.sum(np.abs(diff_data))/W_miss.sum())
    return round(rmse,4),round(mape,2),round(rse,4),round(mae,4)

def show_img(X,RMSE_list,MAE_list,name_list):
    for i in range(len(name_list)):
        plt.plot(X,RMSE_list[i],'--o')
        plt.savefig(img_dir+'rmse_'+name_list[i])
        plt.close()
        plt.plot(X,MAE_list[i],'--o')
        plt.savefig(img_dir+'mae_'+name_list[i])
        plt.close()
    return 0

def compare_iter(ori_speeddata,miss_data,miss_pos,W):
    sp = np.shape(miss_data)
    rank_set = [0,0,0]
    main_rate = 0.9
    alpha = [1/3,1/3,1/3]
    beta = [0.1,0.1,0.1]
    beta1 = beta.copy()
    gama = [2,2,2]
    lou = 1e-3
    K = 100
    conv = 1e-4
    conv_list = np.arange(1e-4,1e-3,5e-5)
    K_list = [50+10*count for count in range(16)]

    RMSE_lrtc_list,MAE_lrtc_list = [],[]
    RMSE_silrtc_list,MAE_silrtc_list = [],[]
    RMSE_halrtc_list,MAE_halrtc_list = [],[]
    range_list = conv_list
    for conv in conv_list:
        est_lrtc = lrtc_cpt(miss_data,beta,alpha,gama,conv,K,W)
        RMSE_lrtc,MAPE_lrtc,RSE_lrtc,MAE_lrtc = rmse_mape_rse(est_lrtc,ori_speeddata,W)
        RMSE_lrtc_list.append(RMSE_lrtc)
        MAE_lrtc_list.append(MAE_lrtc)
        est_silrtc = silrtc_cpt(miss_data,alpha,beta1,conv,K)
        RMSE_silrtc,MAPE_silrtc,RSE_silrtc,MAE_silrtc = rmse_mape_rse(est_silrtc,ori_speeddata,W)
        RMSE_silrtc_list.append(RMSE_silrtc)
        MAE_silrtc_list.append(MAE_silrtc)
        est_halrtc = halrtc_cpt(miss_data,lou,conv,K,W)
        RMSE_halrtc,MAPE_halrtc,RSE_halrtc,MAE_halrtc = rmse_mape_rse(est_halrtc,ori_speeddata,W)
        RMSE_halrtc_list.append(RMSE_halrtc)
        MAE_halrtc_list.append(MAE_halrtc)
    fig = plt.figure()
    ax = fig.add_subplot(1,1,1)
    ax.plot(range_list,RMSE_lrtc_list,'r--^',label='lrtc')
    ax.plot(range_list,RMSE_silrtc_list,'r--s',label='silrtc')
    ax.plot(range_list,RMSE_halrtc_list,'r--D',label='halrtc')
    ax.legend(loc='best')
    plt.savefig(img_dir+'compare_conv_rmse.png')
    plt.close()
    fig = plt.figure()
    ax = fig.add_subplot(1,1,1)
    ax.plot(range_list,MAE_lrtc_list,'r--^',label='lrtc')
    ax.plot(range_list,MAE_silrtc_list,'r--s',label='silrtc')
    ax.plot(range_list,MAE_halrtc_list,'r--D',label='halrtc')
    ax.legend(loc='best')
    plt.savefig(img_dir+'compare_conv_mae.png')
    plt.close()

    #show_img(K_list,RMSE_tc,MAE_tc,['lrtc','silrtc','halrtc'])
    return 0

def compare_methods(ori_speeddata,ori_W):
    RM_dict,MA_dict,RS_dict,MP_dict = {},{},{},{}
    rt_dict = {}
    miss_list = []
    eva_dict = {'rmse':RM_dict,'mae':MA_dict,'mape':MP_dict}
    for i in range(8):
        print('----'+str(i)+'----')
        miss_ratio = round(0.1*(i+1),2)

        miss_path = data_dir+'miss_'+str(miss_ratio)+''.join(['_'+str(ch) for ch in data_size])+'.mat'
        if not os.path.exists(miss_path):
            gene_rand_sparse(ori_speeddata,miss_ratio,miss_path)
        miss_data,W_miss,tm_ratio = get_sparsedata(miss_path)
        W = (W_miss==False)
        rW = W | (ori_W == False)

        miss_list.append(round(tm_ratio * 100,1))
        #预填充
        time_s = time.time()
        miss_data = pre_impute(miss_data, W)
        time_e = time.time()
        rm, mp, rs, ma = rmse_mape_rse(miss_data, ori_speeddata, rW)
        km = 'pre-impute'
        if km not in RM_dict:
            RM_dict[km], MP_dict[km], RS_dict[km], MA_dict[km] = [], [], [], []
            rt_dict[km] = []
        RM_dict[km].append(rm)
        MA_dict[km].append(ma)
        MP_dict[km].append(mp)
        RS_dict[km].append(rs)
        rt_dict[km].append(round(time_e - time_s, 1))
        """
        #参数
        p = 0.7
        K = 100     #iterations
        F_thre = 1e-4  #F_norm convergence threshold

        #halrtc
        time_s = time.time()
        lou = 1 / T_SVD(miss_data, p)[0][0]
        est_halrtc = halrtc_cpt(miss_data,lou,F_thre,K,W,alpha=[1/3,1/3,1/3])
        time_e = time.time()
        rm, mp, rs, ma = rmse_mape_rse(est_halrtc,ori_speeddata,rW)
        km = 'HaLRTC'
        if km not in RM_dict:
            RM_dict[km], MP_dict[km], RS_dict[km], MA_dict[km] = [], [], [], []
            rt_dict[km] = []
        RM_dict[km].append(rm)
        MA_dict[km].append(ma)
        MP_dict[km].append(mp)
        RS_dict[km].append(rs)
        rt_dict[km].append(round(time_e - time_s,1))

        #Kmeans+halrtc
        time_s = time.time()
        K_n = 4   #cluster_num
        est_kmeans = Kmeans_ha(miss_data, W, K_n, K, F_thre, p)
        time_e = time.time()
        rm,mp,rs,ma = rmse_mape_rse(est_kmeans,ori_speeddata,rW)
        km = 'HaLRTC-CSP'
        if km not in RM_dict:
            RM_dict[km],MP_dict[km],RS_dict[km],MA_dict[km] = [],[],[],[]
            rt_dict[km] = []
        RM_dict[km].append(rm)
        MA_dict[km].append(ma)
        MP_dict[km].append(mp)
        RS_dict[km].append(rs)
        rt_dict[km].append(round(time_e - time_s,1))
        '''
        #STD
        time_s = time.time()
        ap,lm,thre = 2e-10,0.05,0.1
        est_STD = STD_cpt(miss_data, W, thre, ap, lm, p)
        time_e = time.time()
        rm, mp, rs, ma = rmse_mape_rse(est_STD, ori_speeddata, rW)
        km = 'STD'
        if km not in RM_dict:
            RM_dict[km], MP_dict[km], RS_dict[km], MA_dict[km] = [], [], [], []
            rt_dict[km] = []
        RM_dict[km].append(rm)
        MA_dict[km].append(ma)
        MP_dict[km].append(mp)
        RS_dict[km].append(rs)
        rt_dict[km].append(round(time_e - time_s,1))
        '''
        #BPCA
        time_s = time.time()
        est_BPCA = BPCA_cpt(miss_data, p)
        time_e = time.time()
        rm, mp, rs, ma = rmse_mape_rse(est_BPCA, ori_speeddata, rW)
        km = 'BPCA'
        if km not in RM_dict:
            RM_dict[km], MP_dict[km], RS_dict[km], MA_dict[km] = [], [], [], []
            rt_dict[km] = []
        RM_dict[km].append(rm)
        MA_dict[km].append(ma)
        MP_dict[km].append(mp)
        RS_dict[km].append(rs)
        rt_dict[km].append(round(time_e - time_s,1))
        """

    eva_dict = {'RMSE':RM_dict,'MAE':MA_dict,'MRE':MP_dict,'Run_Time':rt_dict}
    metric_dict = {'RMSE':'km/h', 'MAE':'km/h', 'MRE':'%', 'Run_Time':'s'}
    eva_Ylim = {'RMSE':[2,10],'MAE':[0,5],'MRE':[5,20],'Run_Time':[0,5000]}
    shape = ['r--o','r--*','r--x','r--^','r--s','r--D']
    MK = ['o','o','*','*','x','x']
    CR = ['r','b','y','r','b','y']

    fw = open('compare_methods' + '.txt', 'w')
    fw.write('methods:'+','.join(list(eva_dict['RMSE'].keys()))+'\n')
    fw.write('Missing Rate (%):' + ','.join(list(map(str, miss_list))) + '\n')
    for eva in eva_dict:
        #fig = plt.figure()
        #ax = fig.add_subplot(1,1,1)
        plt.xlabel('Missing Rate (%)')
        plt.ylabel(eva+' ('+metric_dict[eva]+')')
        # xticks([-np.pi, -np.pi / 2, 0, np.pi / 2, np.pi],
        #        [r'$-pi$', r'$-pi/2$', r'$0$', r'$+pi/2$', r'$+pi$'])
        nl = 0
        fw.write(eva+':\n')
        for method in eva_dict[eva]:
            plt.plot(miss_list,eva_dict[eva][method],color=CR[nl],marker=MK[nl],label='$'+method+'$')
            fw.write(','.join(list(map(str, eva_dict[eva][method]))) + '\n')
            nl += 1
        plt.legend(loc='best')
        plt.savefig(img_dir+'compare_mr_'+'_'+eva+'.png')
        plt.close()
    fw.close()

    return 0

def svd_vary(sparse_data):
    ds = sparse_data.shape
    SG = np.zeros((max(ds),3))
    plt.xlabel('Missing Rate (%)')
    for i in range(len(ds)):
        A = dtensor(sparse_data).unfold(i)
        # U,sigma,VT = np.linalg.svd(A,0)
        # SG[:len(sigma),i] = sigma
        for j in range(ds[i]):
            SG[j,i] = np.var(A[j])
        plt.plot(list(range(max(ds))),SG[:,i],'r--o',label='$'+str(i)+'$')

    plt.legend(loc=1)
    plt.savefig(img_dir+'svd_vary.png')
    plt.close()
    return

#比较三维张量和四维张量的填充结果
def compare_3d_4d(ori_speeddata,miss_data):
    time1 = time.time()
    lou = 1 / T_SVD(miss_data, 0.7)[0][0]
    #lou = 0.1
    print(lou)
    est_halrtc = halrtc_cpt(miss_data, lou, 1e-4, 100, W)
    time2 = time.time()
    print('3d_halrtc:', rmse_mape_rse(est_halrtc, ori_speeddata, (W | (ori_W == False))))
    print('3d_time', str(time2 - time1) + 's')
    return

def compare_PI(ori_speeddata,ori_W):
    RM_dict, MA_dict, RS_dict, MP_dict = {}, {}, {}, {}
    rt_dict = {}
    miss_list = []
    # 参数
    p = 0.7
    K = 100  # iterations
    F_thre = 1e-4  # F_norm convergence threshold
    for i in range(8):
        print('----'+str(i)+'----')
        miss_ratio = round(0.1*(i+1),2)
        miss_path = data_dir+'miss_'+str(miss_ratio)+''.join(['_'+str(ch) for ch in data_size])+'.mat'
        if not os.path.exists(miss_path):
            gene_rand_sparse(ori_speeddata,miss_ratio,miss_path)
        miss_data,W_miss,tm_ratio = get_sparsedata(miss_path)
        W = (W_miss==False)
        miss_list.append(round(tm_ratio * 100,1))
        #预填充
        #miss_data = pre_impute(miss_data,W,False)
        rW = W|(ori_W==False)
        time_s = time.time()
        K_n = 4  # cluster_num
        #est_kmeans = Kmeans_ha(miss_data, W, K_n, K, F_thre, p)
        est_halrtc = halrtc_cpt(miss_data, 1e-3, 1e-4, 100, W,[1/3,1/3,1/3])
        time_e = time.time()
        rm, mp, rs, ma = rmse_mape_rse(est_halrtc, ori_speeddata, rW)
        km = 'Without-PI'
        if km not in RM_dict:
            RM_dict[km], MP_dict[km], RS_dict[km], MA_dict[km] = [], [], [], []
            rt_dict[km] = []
        RM_dict[km].append(rm)
        MA_dict[km].append(ma)
        MP_dict[km].append(mp)
        RS_dict[km].append(rs)
        rt_dict[km].append(round(time_e - time_s,1))

        miss_data = pre_impute(miss_data, W, False)
        rW = W | (ori_W == False)
        time_s = time.time()
        K_n = 4  # cluster_num
        #est_kmeans = Kmeans_ha(miss_data, W, K_n, K, F_thre, p)
        est_halrtc = halrtc_cpt(miss_data, 1e-3, 1e-4, 100, W,[1/3,1/3,1/3])
        time_e = time.time()
        rm, mp, rs, ma = rmse_mape_rse(est_halrtc, ori_speeddata, rW)
        km = 'With-PI'
        if km not in RM_dict:
            RM_dict[km], MP_dict[km], RS_dict[km], MA_dict[km] = [], [], [], []
            rt_dict[km] = []
        RM_dict[km].append(rm)
        MA_dict[km].append(ma)
        MP_dict[km].append(mp)
        RS_dict[km].append(rs)
        rt_dict[km].append(round(time_e - time_s,1))

    eva_dict = {'RMSE': RM_dict, 'MAE': MA_dict, 'MRE': MP_dict, 'Run_Time': rt_dict}
    metric_dict = {'RMSE': 'km/h', 'MAE': 'km/h', 'MRE': '%', 'Run_Time': 's'}
    eva_Ylim = {'RMSE': [2, 10], 'MAE': [0, 5], 'MRE': [5, 20], 'Run_Time': [0, 5000]}
    shape = ['r--o', 'r--*', 'r--x', 'r--^', 'r--s', 'r--D']
    MK = ['o', 'o', '*', '*', 'x', 'x']
    CR = ['r', 'b', 'y', 'r', 'b', 'y']
    fw = open('compare_PI_' + '.txt', 'w')
    fw.write('Missing Rate (%):' + ','.join(list(map(str, miss_list))) + '\n')
    for eva in eva_dict:
        # fig = plt.figure()
        # ax = fig.add_subplot(1,1,1)

        plt.xlabel('Missing Rate (%)')
        plt.ylabel(eva + ' (' + metric_dict[eva] + ')')
        # xticks([-np.pi, -np.pi / 2, 0, np.pi / 2, np.pi],
        #        [r'$-pi$', r'$-pi/2$', r'$0$', r'$+pi/2$', r'$+pi$'])
        nl = 0
        fw.write(eva + ':' + '\n')
        for method in eva_dict[eva]:
            plt.plot(miss_list, eva_dict[eva][method], color=CR[nl], marker=MK[nl], label='$' + method + '$')
            fw.write(','.join(list(map(str, eva_dict[eva][method]))) + '\n')
            nl += 1

        plt.legend(loc='best')
        plt.savefig(img_dir + 'compare_PI_' + '_' + eva + '.png')
        plt.close()
    fw.close()
    return 0

def ori_imputation(miss_data,W,ori_speeddata,ori_W):
    ds = miss_data.shape
    rW = W|(ori_W==False)
    est_Km = Kmeans_ha(miss_data, W, 4, 100, 1e-4, 0.7)
    plt.xlim(0,50)
    plt.ylim(0,50)
    plt.plot([0,50],[0,50],color='b',linewidth=2)
    #plt.scatter(1, 2, s=10, c='r')
    r = 1
    for j in range(ds[1]):
            for k in range(ds[2]):
                if not rW[r,j,k]:
                    plt.scatter(ori_speeddata[r,j,k],est_Km[r,j,k],s=10,c='r')
    plt.savefig('填充真实对比图.png')
    plt.close()
    return

if __name__ == '__main__':
    data_dir = './data/'
    img_dir = './img_test/'
    mat_path = '/home/qiushye/2013_east/2013_east_speed.mat'
    #数据：日期数，线圈数量，时间间隔数
    #data_size = (60,80,144)
    #data_size = (15,35,288)
    data_size = (30,20,72)
    #广州数据

    ori_speeddata = scio.loadmat('../GZ_data/60days_tensor.mat')['tensor']
    #train_sim(ori_speeddata[:,:30,:])
    #assign_group(ori_speeddata[:,:30,:])
    #simMat = multi_sim(ori_speeddata[:,:30,:])
    #scio.savemat('road_sim.mat',{'sim':simMat})
    #sys.exit()
    #print(np.var(ori_speeddata))
    shorten = False
    print((ori_speeddata==0).sum()/ori_speeddata.size)
    ori_speeddata,ori_W = deal_orimiss(ori_speeddata,shorten)
    data_size = np.shape(ori_speeddata)
    print(data_size)

    # compare_methods(ori_speeddata,ori_W)
    # compare_PI(ori_speeddata,ori_W)
    #sys.exit()
    miss_ratio = 0.2
    miss_path = data_dir+'miss_'+str(round(miss_ratio,1))+'_'+'_'.join([str(ch) for ch in data_size])+'.mat'
    #miss_path = data_dir+'cont_miss_'+'_'.join([str(ch) for ch in data_size])+'.mat'
    if not os.path.exists(miss_path):
        gene_rand_sparse(ori_speeddata,miss_ratio,miss_path)
    miss_data,W_miss,tm_ratio = get_sparsedata(miss_path)
    # f, ax = plt.subplots()
    # sns.heatmap(miss_data[0], cmap='RdBu', linewidths=0.05, ax=ax)
    # ax.invert_yaxis()
    # ax.set_xlabel('time interval')
    # ax.set_ylabel('days')
    # plt.savefig(img_dir+'miss_heatmap.png')
    # plt.close()
    # sys.exit()
    W = miss_data>0
    #W1 = miss_data==0
    data_shape = np.shape(ori_speeddata)
    miss_data = pre_impute(miss_data,W)
    print('pre_impute:', rmse_mape_rse(miss_data, ori_speeddata, W | (ori_W == False)))
    #compare_3d_4d(ori_speeddata,miss_data)
    ori_imputation(miss_data, W, ori_speeddata, ori_W)
    #svd_vary(miss_data)

    sys.exit()
    #est_partmiss(ori_speeddata, ori_W, miss_data, W, 0.04)
    #u,sigma,vt = np.linalg.svd(dtensor(miss_data).unfold(1),0)
    #print(sigma[0],sigma[0]/sum(sigma))
    #sys.exit()
    lou = 1e-3
    K = 100
    conv = 1e-4

    halrtc_para = [3e-3,100,1e-4]
    [lou,K,conv_thre] = halrtc_para
    time0 = time.time()
    #est_halrtc = halrtc_cpt(miss_data, 1.3e-3, 1e-4, 100, W, 0)
    time1 = time.time()
    #print('ori_halrtc:', rmse_mape_rse(est_halrtc, ori_speeddata, (W | (ori_W == False))))
    print('ori_time', str(time1- time0) + 's')
    K_n = 2
    labels = SC_1(miss_data, 6, K_n, axis=0)
    est_SC = cluster_ha(labels, miss_data, W, K_n, halrtc_para, axis=0)
    time_e = time.time()
    print('sc_est:',rmse_mape_rse(est_SC, ori_speeddata, W|(ori_W==False)))
    clr_assign,K_n = road_Kmeans(miss_data,ori_W,K_n,W,axis=0,method='cos')
    est_kmeans = cluster_ha(clr_assign,miss_data,W,K_n,halrtc_para,axis=0)
    print('kmeans_est:',rmse_mape_rse(est_kmeans,ori_speeddata,W|(ori_W==False)))
    time2 = time.time()
    print('kmeans_time:',time2-time1,'s')
    sys.exit()
    cr = range(30)