# -*- coding: utf-8 -*-

from flask import (
    Blueprint, flash, g, redirect, render_template, request, session, url_for, jsonify,current_app,send_file,Response,make_response
)
import json,time
from myolap.dataquery import dbhelper
from myolap.handler import json_result
from myolap.utils.funcutils import get_sql_field,get_pandas_express
from myolap.utils.const import _const

import pandas as pd

handler = Blueprint('olapconfig', __name__)

@handler.route("/olap/olap/query", methods=['POST'])
def olapconfig_query():
    #目前不支持分頁
    result = 'fail'
    code = 0
    message = '查询数据失败';
    success = False
    try:
        formdata = json.loads(request.get_data())
        #根据formdata 拼装sql
        pdresult = get_data_mysql(formdata )
        result = {
            "data": json.loads(pdresult.to_json(orient='records', date_format='iso', double_precision=2))
        }
        message ='查询数据成功'
        success = True

    except Exception as e:
        message = 'error: %s' %(e)
        current_app.logger.error('error: %s' %(e))
    return json_result(result=result, code=0, message=message,success=success)

@handler.route("/olap/olap/download", methods=['POST'])
def olapconfig_download():
    #目前不支持分頁
    formdata = json.loads(request.get_data())
    print(formdata)
    # from myolap.dataquery.dbhelper import get_data_by_dsandsql

    #根据formdata 拼装sql
    pdresult = get_data_mysql(formdata )
    #处理流下载

    from io import BytesIO
    outfile = BytesIO()
    pdresult.to_excel(outfile, sheet_name='Sheet1',columns=formdata['columns'],header=formdata['header'],index=False)
    outfile.seek(0)
    return send_file(outfile, mimetype='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',as_attachment=True, attachment_filename='download.xlsx')

def get_bracket_string(items):
    #根据列表返回小括号（）包括的字符串
    str =''
    for item in items:
        str = str + "'" + item + "',"
    if len(str)>1:
        str="(" + str[0:len(str)-1] + ")"
    return str

def get_where_field(all_conditional_field):
    # 对条件进行分析，找出维度条件 和 度量条件 拼装成列表形势
    where_dim_field = []
    where_metric_field = []

    for sqlfield_key in all_conditional_field:
        print(sqlfield_key)
        obj = all_conditional_field[sqlfield_key]

        if obj['column_type'] == _const.COLUMN_TYPE_DIM:
            # 维度条件
            if obj['sql_field_type'] == _const.SQL_FIELD_TYPE_STR:
                if len(obj['value']) == 1 and obj['value'][0] =='~' :
                    pass
                else:
                    if obj['sql_field'] == 'appkey':
                        where_dim_field.append(" {}='{}' ".format(obj['sql_express'], obj['value']))
                    else:
                        where_dim_field.append(' {} in {} '.format(obj['sql_express'], get_bracket_string(obj['value'])))
            elif obj['sql_field_type'] == _const.SQL_FIELD_TYPE_DATE:
                where_dim_field.append(
                    " {} >= '{}' AND {} <= '{}'".format(obj['sql_express'], obj['from'], obj['sql_express'], obj['to']))
            elif obj['sql_field_type'] == _const.SQL_FIELD_TYPE_NUM :
                where_dim_field.append(
                    " {} >= '{}' AND {} <= '{}'".format(obj['sql_express'], obj['from'], obj['sql_express'], obj['to']))
            else:
                current_app.logger.error('未定义的数据类型:{}'.format(obj['sql_field_type']))

        elif obj['column_type'] == _const.COLUMN_TYPE_METRIC:
            # 度量条件
            if obj['sql_field_type'] == _const.SQL_FIELD_TYPE_NUM:
                if str(obj['from']).strip() != '':
                    where_metric_field.append( " {} >= {} ".format(obj['sql_field'], obj['from']))
                if str(obj['to']).strip() != '':
                    where_metric_field.append( " {} <= {} ".format(obj['sql_field'], obj['to']))
                # where_metric_field.append(
                #     " {} >= {} & {} <= {}".format( obj['sql_field'], obj['from'], obj['sql_field'], obj['to']))
            else:
                current_app.logger.error('未定义的类型:{}'.format(obj['sql_field_type']))

    print('where==============')
    print(where_dim_field)
    print('where==============')
    print(where_metric_field)
    print('where==============')

    return where_dim_field,where_metric_field

def get_data_mysql(formdata ):
    dimension_field = formdata['dimension_field']
    query_field =formdata['query_field']
    all_query_field=formdata['all_query_field']
    #条件字段合并到一个对象中  数据结构是一样的
    conditional_field=formdata['fixed_conditional_field']
    if formdata['other_conditional_field'] is not None:
        conditional_field.update(formdata['other_conditional_field'])

    #对维度列进行加工
    sql_col_dim_fields=[]
    sql_col_groupby_fields=[]
    sql_merge_keys=[]
    #求汇总或者平均数据的时候，维度dim定义为固定常数维度   方便后面dataframe合并
    # sql_avg_dim_fields =[]
    # sql_sum_dim_fields = []
    for field in dimension_field:
        sql_col_dim_fields.append(' {} as {} '.format(field['sql_express'], field['sql_field']))
        sql_col_groupby_fields.append(' {} '.format(field['sql_express']))
        sql_merge_keys.append(field['sql_field'])

    #对条件进行分析，找出维度条件 和 度量条件
    where_dim_field, where_metric_field = get_where_field(conditional_field)

    print('where==============')
    #对跨模型计算字段进行分析，找出依赖字段
    calc_field ={}
    print('calc field   ~=======')
    if query_field.__contains__('~'):
        for f in query_field['~']:
            calc_field.update(get_sql_field(f['sql_express']))

    print(calc_field)
    #从query_field中找出已经正常选中的字段，
    for qmodel in query_field:
        for field in query_field[qmodel]:
            calc_field[field['sql_field']] = 1

    #
    print('zhuijia hou....')
    print(calc_field)

    #对查询字段进行加工
    from myolap.model.sysdbmodel import OlapModelConfig, OlapModelConfigMetadata

    per = get_permission(formdata['olap_id'])

    sqls={}
    for qmodel in all_query_field:
        if qmodel =='~':
            print('qmodel==~ continue...................')
            continue

        sql_col_v_fields = []

        for field in all_query_field[qmodel]:
            #包含在calc_field 则进行计算
            if calc_field.__contains__(field['sql_field']):
                sql_col_v_fields.append(' {} as {} '.format(field['sql_express'], field['sql_field']))

        print(sql_col_v_fields)
        if len(sql_col_v_fields)> 0 :
            #长度大于0才进行计算
            u = g.session.query(OlapModelConfig).filter(OlapModelConfig.model_name == qmodel).first()
            strsql = ' SELECT '
            if len(sql_col_dim_fields)>0:
                strsql = strsql + ','.join(sql_col_dim_fields) + ','
            strsql = strsql + ','.join(sql_col_v_fields)
            strsql = strsql + ' FROM ( ' + u.model_sql + ' ) x WHERE 1=1 '
            if len( where_dim_field ) > 0:
                strsql = strsql + ' AND '  + ' AND '.join(where_dim_field)
            #行权限限制
            if len(per['row_permission']) >0:
                strsql = strsql + ' AND ' + ' AND '.join(per['row_permission'])

            if len(sql_col_groupby_fields) > 0:
                strsql = strsql + ' GROUP BY ' + ','.join(sql_col_groupby_fields)

            print('==========the sql is==============')
            print(strsql)

            df= dbhelper.get_data_by_dsandsql(u.datasource_name,  strsql)
            #加 __rowid__  用于没有groupby字段的合并
            if len(sql_merge_keys) == 0:
                #如果没有维度，则是真个记录集groupby ，出来只有一行数据 此时pandas合并的时候，需要一个mergekey
                #这里默认加一个行号作为mergekey
                df['__rowid__'] = range(1, len(df) + 1)
            sqls[qmodel] = df

    #合并数据
    if len(sql_merge_keys) > 0:
        pd_all = dbhelper.merge_dataframe(sql_merge_keys, [ df for df in sqls.values() ])
    else:
        pd_all = dbhelper.merge_dataframe(['__rowid__'], [df for df in sqls.values()])
        pass
    # pd_all_sum = dbhelper.merge_dataframe_rowid(sql_merge_keys,[ df for df in sqls_sum.values()],'合计')
    # pd_all_avg = dbhelper.merge_dataframe_rowid(sql_merge_keys,[ df for df in sqls_avg.values()],'平均')

    # 跨模型计算字段处理
    if query_field.__contains__('~'):
        for f in query_field['~']:
            pandas_express = get_pandas_express(f['sql_field'], f['sql_express'])
            print('pandas express=')
            print(pandas_express)
            # 跨字段计算
            pd_all.eval(pandas_express, inplace=True)

    # 合并后筛选维度数据
    if len(where_metric_field) > 0:
        str_where = ' & '.join(where_metric_field)
        print('where tiaojian ============')
        print(str_where)
        pd_all = pd_all.query(str_where)

    #找到所有维度字段
    all_columns = pd_all.columns
    sql_value_fields = set(pd_all.columns).difference(sql_merge_keys,['__rowid__'])

    #最后处理汇总和平均
    pd_all_sum = pd_all[sql_value_fields].sum()
    pd_all_avg = pd_all[sql_value_fields].mean()

    pd_all_sum['__rowid__'] ='合计'
    pd_all_avg['__rowid__'] = '均值'
    for col in sql_merge_keys:
        pd_all_sum[col] = '-'
        pd_all_avg[col] = '-'
    #处理不展示合计和平均的字段
    for qmodel in query_field:
        for field in query_field[qmodel]:
            #是否展示均值
            if not field['is_average']:
                pd_all_avg[field['sql_field']] ='-'
            #是否展示总计
            if not field['is_total']:
                pd_all_sum[field['sql_field']] = '-'

    pd_all = pd_all.append(pd_all_sum,ignore_index=True)
    pd_all = pd_all.append(pd_all_avg,ignore_index=True)

    #列权限限制
    if len(per['col_permission'])>0:
        set_col = set(per['col_permission'])
        for col in all_columns:
            if col =='__rowid__' or col in set_col:
                pass
            else:
                pd_all.drop(col, axis=1, inplace=True)

    # result = {
    #     "data": json.loads(pd_all.to_json(orient='records', date_format='iso', double_precision=2))
    # }
    # return json_result(result=result,code=200,message='ok')
    # 页面数据和下载数据公用查询，  返回不同类型
    return pd_all

def get_permission(olap_id):
    #根据权限设置，限制数据以及维度行权限
    result =[]

    from myolap.utils.authutils import get_current_username
    current_username = get_current_username()
    print(current_username)

    row_permission = []    #空列表，表示不做限制
    col_permission = []    #空列表，表示不做限制
    if current_username =='admin' :
        #管理员或者创建者自己，新增的时候不做限制
        pass
    else:
        # olap_id = formdata['olap_id']
        from myolap.model.sysdbmodel import OlapDimension,OlapDimensionPermission, multi_to_list_by_columns
        #查询多维表对象
        olapobj = g.session.query(OlapDimension).filter(OlapDimension.olap_id == olap_id).first()

        #db没有记录 表示多维表是新增的状态下，还没有保存到db
        if olapobj is None:
            pass
        else:
            permission = None
            #找到当前用户的权限设置
            for per in olapobj.permission:
                if per.username == current_username:
                    permission = per
                    break
            if permission is None:
                #没赋予权限
                row_permission.append(' {}  =  {} '.format(1,2))   #限制不能查看 给一个不可能==true的表达式
                # col_permission=['__rowid__']                       #限制不能查看，给一个无用的序号列
            else:
                #有记录 表示有权限，需要进一步判断行、列权限
                rowper = json.loads( permission.row_permission)
                for p in rowper:
                    if p['op'].upper() == 'EQ':
                        row_permission.append(" {} = '{}' ".format( p['sql_express'] , p['val'] ))
                    elif p['op'].upper() == 'IN':
                        row_permission.append(' {} in {} '.format(p['sql_express'], get_bracket_string(p['val'].split(','))))
                    else:
                        pass

                col_permission = json.loads( permission.col_permission)
                # col_permission.append('__rowid__')

    result ={
        'row_permission': row_permission,
        'col_permission': col_permission
    }

    #问题 ：缺少olap_id  无法获取用户权限限制

    return result

def get_dimension_items_mysql(formdata ):
    model_name = formdata['model_names']
    sql_field =formdata['sql_field'].strip()
    sql_express=formdata['sql_express'].strip()

    print('where==============')
    print(formdata)
    #对查询字段进行加工
    from myolap.model.sysdbmodel import OlapModelConfig, OlapModelConfigMetadata

    per = get_permission(formdata['olap_id'])

    keys_of_dim = {}
    list_dim = []
    for qmodel in model_name:
        print(qmodel)
        u = g.session.query(OlapModelConfig).filter(OlapModelConfig.model_name == qmodel).first()

        strsql =' SELECT DISTINCT ' + sql_express + ' AS ' + sql_field
        strsql = strsql + ' FROM ( ' + u.model_sql + ' ) x WHERE 1=1 '

        # 行权限限制
        if len(per['row_permission']) > 0:
            strsql = strsql + ' AND ' + ' AND '.join(per['row_permission'])

        print('==========the sql is==============')
        print(strsql)
        df= dbhelper.get_data_by_dsandsql(u.datasource_name,  strsql)
        for v in df[sql_field].drop_duplicates().values.tolist():
            if v not in keys_of_dim and len(v.strip()) > 0:
                keys_of_dim[v.strip()] = 1
                list_dim.append({"id":v.strip(), "name":v.strip()})

    result = list_dim
    return result

    # return json_result(result=result,code=200,message='ok')

# def get_dimension_metricdata_mysql(formdata ):
#     model_name = formdata['model_name']
#     sql_field =formdata['sql_field'].strip(' ')
#     sql_express=formdata['sql_express'].strip(' ')
#
#     print('where==============')
#     print(formdata)
#     #对查询字段进行加工
#     from myolap.model.sysdbmodel import OlapModelConfig, OlapModelConfigMetadata
#
#     keys_of_dim = {}
#
#     u = g.session.query(OlapModelConfig).filter(OlapModelConfig.model_name == model_name).first()
#
#     strsql =' SELECT MAX( ' + sql_express + ') AS ' + 'max_' + sql_field + ','
#     strsql = strsql + ' MIN( ' + sql_express +') AS min_' + sql_field
#     strsql = strsql + ' FROM ( ' + u.model_sql + ' ) x WHERE 1=1 '
#     print('==========the sql is==============')
#     print(strsql)
#     df= dbhelper.get_data_by_dsandsql(u.datasource_name,  strsql)
#     for v in df[sql_field].drop_duplicates().values.tolist():
#         keys_of_dim[v]=1
#
#         # dim_values.append(df[sql_field].drop_duplicates().values.tolist())
#     list_dim = list(keys_of_dim.keys())
#     result = {
#         "data": list_dim
#     }
#
#     return json_result(result=result,code=200,message='ok')

@handler.route("/olap/olapconfig/checkfield", methods=['POST'])
def olapconfig_checkfield():
    #检查olap多维表的字段配置是否正确

    result = 'fail'
    code = 0
    message = '检查字段失败';
    success = False
    try:
        formdata = json.loads(request.get_data())
        from myolap.model.sysdbmodel import OlapModelConfig, OlapModelConfigMetadata
        u = g.session.query(OlapModelConfig).filter(OlapModelConfig.model_name == formdata['model_name']).first()

        if u is None:
            # 校验失败 模型没找到
            message = '模型校验失败，没有找到模型'
        else:
            #提交请求到对应数据源，查看是否执行成功
            sql = 'select {} from ( {} limit 1 ) x '.format(formdata['sql_express'], u.model_sql)

            from myolap.dataquery.dbhelper import get_data_by_dsandsql
            r = dbhelper.get_data_by_dsandsql(u.datasource_name,sql)
            if r is not None:
                code = 200
                message='校验成功'
                result ='ok'
                success = True
            else:
                message = '模型校验失败'

    except Exception as e:
        message = 'error: %s' %(e)
        current_app.logger.error('error: %s' %(e))
    return json_result(result=result, code=0, message=message,success=success)

@handler.route("/olap/olapconfig/querydimension", methods=['POST'])
def olapconfig_querydimension():
    result = 'fail'
    code = 0
    message = '查询维度信息失败';
    success = False
    try:
        formdata = json.loads(request.get_data())
        print(formdata)
        result = get_dimension_items_mysql(formdata)
        message = '查询维度信息成功';
        success = True
    except Exception as e:
        message = 'error: %s' %(e)
        current_app.logger.error('error: %s' %(e))
    return json_result(result=result, code=0, message=message,success=success)
    # return result

# @handler.route("/olap/olapconfig/querymetric", methods=['POST'])
# def olapconfig_querymetric():
#     formdata = json.loads(request.get_data())
#     print(formdata)
#     result = get_dimension_metricdata_mysql(formdata)
#
#     return result

@handler.route("/olap/olapconfig/save", methods=['POST'])
def olapconfig_save():
    #检查olap多维表的字段配置是否正确
    result = 'fail'
    code = 0
    message = '保存多维表信息失败';
    success = False
    try:
        from myolap.utils.authutils import get_current_username
        from myolap.model.sysdbmodel import OlapDimension
        username = get_current_username()
        formdata = json.loads(request.get_data())

        u = g.session.query(OlapDimension).filter(OlapDimension.olap_id == formdata['olap_id']).first()
        if u is None:
            # 新增
            u = OlapDimension()
            u.olap_id = formdata['olap_id']
            g.session.add(u)

        #更新字段内容
        u.dimension_name = formdata['dimension_name']
        u.create_by = username
        u.version = formdata['version']
        str_checked_list = json.dumps(formdata['checked_list'])
        str_filter_list = json.dumps(formdata['filter_list'])

        u.checked_list = str_checked_list
        u.filter_list = str_filter_list
        g.session.commit()
        result ='ok'
        message ='保存多维表信息成功'
        success = True
    except Exception as e:
        message = 'error: %s' %(e)
        current_app.logger.error('error: %s' %(e))
    return json_result(result=result, code=0, message=message,success=success)

@handler.route("/olap/olapconfig/shortcut/save", methods=['POST'])
def olapconfig_shortcut_save():
    #检查olap多维表的字段配置是否正确
    result = 'fail'
    code = 0
    message = '保存快捷键信息失败';
    success = False
    try:
        from myolap.utils.authutils import get_current_username
        from myolap.model.sysdbmodel import OlapDimensionShortcut
        username = get_current_username()
        formdata = json.loads(request.get_data())
        u = g.session.query(OlapDimensionShortcut).filter(OlapDimensionShortcut.id == formdata['shortcut_id']).first()
        if u is None:
            # 新增
            u = OlapDimensionShortcut()
            u.id = formdata['shortcut_id']
            g.session.add(u)
        #更新字段内容
        u.shortcut_name = formdata['shortcut_name']
        u.olap_id = formdata['olap_id']
        u.create_by = username
        u.version = formdata['version']
        str_checked_list = json.dumps(formdata['checked_list'])
        str_filter_list = json.dumps(formdata['filter_list'])

        u.checked_list = str_checked_list
        u.filter_list = str_filter_list

        g.session.commit()
        result ='ok'
        message ='保存快捷键信息成功'
        success = True

    except Exception as e:
        message = 'error: %s' %(e)
        current_app.logger.error('error: %s' %(e))
    return json_result(result=result, code=0, message=message,success=success)

@handler.route("/olap/olapconfig/get", methods=['GET'])
def olapconfig_get():
    # 检查olap多维表的字段配置是否正确
    result = 'fail'
    code = 0
    message = '获取多维表信息失败';
    success = False
    try:
        olap_id = request.args.get("olap_id")
        per = get_permission(olap_id)
        from myolap.model.sysdbmodel import OlapDimension
        u = g.session.query(OlapDimension).filter(OlapDimension.olap_id == olap_id).first()

        if u is None:
            result = {
                "checked_list": [],
                "filter_list": {},
                "dimension_name": '',
                "olap_id": olap_id
            }
            message = '未查询到相关信息'
            success = True
        else:
            #进行列限制
            allow_cols =per['col_permission']
            checked_list = json.loads(u.checked_list)
            filter_list = json.loads(u.filter_list)
            remove_not_allow_field(allow_cols,checked_list,filter_list)

            result = {
                "checked_list": checked_list,
                "dimension_name":u.dimension_name,
                "filter_list": filter_list,
                "olap_id": u.olap_id
            }
            message = '查询信息成功'
            success = True
    except Exception as e:
        message = 'error: %s' %(e)
        current_app.logger.error('error: %s' %(e))
    return json_result(result=result, code=0, message=message,success=success)

def remove_not_allow_field(allow_cols, checked_list,filter_list):
    if len(allow_cols) > 0:
        for item in checked_list:
            for child in item['children'][:: -1]:
                if child['sql_field'] == '__rowid__' or child['sql_field'] in allow_cols:
                    pass
                else:
                    # 不在允许的列范围内
                    item['children'].remove(child)

        for child in filter_list['add_no_must_filter_list'][:: -1]:
            if  child['sql_field'] == '__rowid__' or child['sql_field'] in allow_cols:
                pass
            else:
                # 不在允许的列范围内
                filter_list['add_no_must_filter_list'].remove(child)

        for child in filter_list['no_must_filter_list'][:: -1]:
            print('====================no_must_filter_list')
            print(child['sql_field'])
            if  child['sql_field'] == '__rowid__' or child['sql_field'] in allow_cols:
                pass
            else:
                # 不在允许的列范围内
                filter_list['no_must_filter_list'].remove(child)
                print('yichu............')
                print(child['sql_field'])

        for child in filter_list['must_filter_list'][:: -1]:
            if  child['sql_field'] == '__rowid__' or child['sql_field'] in allow_cols:
                pass
            else:
                # 不在允许的列范围内
                filter_list['must_filter_list'].remove(child)
    else:
        pass

@handler.route("/olap/olapconfig/shortcut/get", methods=['GET'])
def olapconfig_shortcut_get():
    # 检查olap多维表的字段配置是否正确
    result = 'fail'
    code = 0
    message = '获取快捷键信息失败';
    success = False
    try:
        shortcut_id = request.args.get("shortcut_id")
        from myolap.model.sysdbmodel import OlapDimensionShortcut
        u = g.session.query(OlapDimensionShortcut).filter(OlapDimensionShortcut.id == shortcut_id).first()

        if u is None:
            result = {
                "checked_list": [],
                "filter_list": {},
                "shortcut_name": '',
                "olap_id": '',
                "shortcut_id":''
            }
            message = '未查询到快捷键信息'
            success = True
        else:
            result = {
                "checked_list": u.checked_list,
                "shortcut_name":u.shortcut_name,
                "filter_list":u.filter_list,
                "olap_id": u.olap_id,
                "shortcut_id":u.id
            }
            message = '查询信息成功'
            success = True
    except Exception as e:
        message = 'error: %s' %(e)
        current_app.logger.error('error: %s' %(e))
    return json_result(result=result, code=0, message=message,success=success)

@handler.route("/olap/olapconfig/list", methods=['GET'])
def olapconfig_list():
    # 检查olap多维表的字段配置是否正确

    result = 'fail'
    code = 0
    message = '获取快捷键信息失败';
    success = False
    try:
        from myolap.utils.authutils import get_current_username
        #当前登录用户
        username = get_current_username()
        from myolap.model.sysdbmodel import OlapDimension,OlapDimensionPermission, multi_to_list, multi_to_list_by_columns2
        # 只查询指定的字段
        # create_by = request.args.get("create_by")
        if username == 'admin':
            print('admin........')
            modellist = g.session.query(OlapDimension).all()
        else:
            print('普通用户')
            modellist = g.session.query(OlapDimension).filter(OlapDimension.create_by == username).all()

        list = multi_to_list_by_columns2(modellist,['olap_id','dimension_name','create_by'],'permission')
        result = {
            "current": 0,
            "pages": 0,
            "records": list,
            "size": 0,
            "total": 0
        }
        message = '查询信息成功'
        success = True
    except Exception as e:
        message = 'error: %s' %(e)
        current_app.logger.error('error: %s' %(e))
    return json_result(result=result, code=0, message=message,success=success)

@handler.route("/olap/olapconfig/shortcut/list", methods=['GET'])
def olapconfig_shortcut_list():
    # 检查olap多维表的字段配置是否正确

    result = 'fail'
    code = 0
    message = '获取快捷键信息失败';
    success = False
    try:

        from myolap.utils.authutils import get_current_username
        username = get_current_username()
        olap_id = request.args.get("olap_id")

        from myolap.model.sysdbmodel import OlapDimensionShortcut, multi_to_list, multi_to_list_by_columns

        modellist = g.session.query(OlapDimensionShortcut)\
            .with_entities(OlapDimensionShortcut.id,OlapDimensionShortcut.olap_id,OlapDimensionShortcut.shortcut_name)\
            .filter(OlapDimensionShortcut.create_by == username).filter(OlapDimensionShortcut.olap_id == olap_id).all()

        list = multi_to_list_by_columns(modellist,['id','olap_id','shortcut_name'])
        result = {
            "current": 0,
            "pages": 0,
            "records": list,
            "size": 0,
            "total": 0
        }
        message = '查询信息成功'
        success = True
    except Exception as e:
        message = 'error: %s' % (e)
        current_app.logger.error('error: %s' % (e))
    return json_result(result=result, code=0, message=message, success=success)

@handler.route("/olap/olapconfig/delete", methods=['DELETE'])
def olapconfig_delete():
    # 删除某张多维表
    result = 'fail'
    code = 0
    message = '获取快捷键信息失败';
    success = False
    try:
        olap_id = request.args.get("id")
        print('delete................')
        print(olap_id)

        # from myolap.app import sysdb
        from myolap.model.sysdbmodel import OlapDimension
        g.session.query(OlapDimension).filter(OlapDimension.olap_id == olap_id).delete()
        g.session.commit()
        result ='OK'
        message = '删除成功'
        success = True

    except Exception as e:
        message = 'error: %s' % (e)
        current_app.logger.error('error: %s' % (e))
    return json_result(result=result, code=0, message=message, success=success)

@handler.route("/olap/olapconfig/shortcut/delete", methods=['DELETE'])
def olapconfig_shortcut_delete():
    #删除某个快捷键
    result = 'fail'
    code = 0
    message = '删除信息失败';
    success = False
    try:
        shortcut_id = request.args.get("shortcut_id")
        from myolap.model.sysdbmodel import OlapDimensionShortcut
        g.session.query(OlapDimensionShortcut).filter(OlapDimensionShortcut.id==shortcut_id).delete()
        g.session.commit()
        result = 'OK'
        message ='删除数据成功'
        success = True
    except Exception as e:
        message = 'error: %s' % (e)
        current_app.logger.error('error: %s' % (e))
    return json_result(result=result, code=0, message=message, success=success)


from __future__ import absolute_import, division, print_function

from fractions import Fraction
import sys
import subprocess
sys.path.insert(0, '../')

from compiler import *
from constructs import *

def test_harris_corner():

    R = Parameter(Int, "R")
    C = Parameter(Int, "C")
    x = Variable(Int, "x")
    y = Variable(Int, "y")

    row = Interval(Int, 0, R+1)
    col = Interval(Int, 0, C+1)

    cond = Condition(x, '>=', 1) & Condition(x, '<=', R) & \
           Condition(y, '<=', C) & Condition(y, '>=', 1)

    condRed = Condition(x, '>=', 2) & Condition(x, '<=', R-1) & \
              Condition(y, '<=', C-1) & Condition(y, '>=', 2)

    zeroCond = Condition(x, '==', 0) & Condition(y, '==', 0)
    endCond = Condition(x, '==', R) & Condition(y, '==', C)

    img = Image(Float, "img", [R+2, C+2])

    Iy = Function(([x, y], [row, col]), Float, "Iy")
    Iy.defn = [ Case(cond, img(x-1, y-1)*(-1.0/12) + img(x-1, y+1)*(1.0/12) + \
                           img(x, y-1)*(-2.0/12) + img(x, y+1)*(2.0/12) + \
                           img(x+1, y-1)*(-1.0/12) + img(x+1, y+1)*(1.0/12)) ]

    Ix = Function(([x, y], [row, col]), Float, "Ix")
    Ix.defn = [ Case(cond, img(x-1, y-1)*(-1.0/12) + img(x+1, y-1)*(1.0/12) + \
                           img(x-1, y)*(-2.0/12) + img(x+1, y)*(2.0/12) + \
                           img(x-1, y+1)*(-1.0/12) + img(x+1, y+1)*(1.0/12)) ]

    Ixx = Function(([x, y], [row, col]), Float, "Ixx")
    Ixx.defn = [ Case(cond, Ix(x, y) * Ix(x, y)) ]

    Iyy = Function(([x, y], [row, col]), Float, "Iyy")
    Iyy.defn = [ Case(cond, Iy(x, y) * Iy(x, y)) ]

    Ixy = Function(([x, y], [row, col]), Float, "Ixy")
    Ixy.defn = [ Case(cond, Ix(x, y) * Iy(x, y)) ]

    Sxx = Function(([x, y], [row, col]), Float, "Sxx")
    Sxx.defn = [ Case(condRed, Ixx(x-1, y-1) + Ixx(x-1, y) + Ixx(x-1, y+1) +\
                               Ixx(x, y-1)   + Ixx(x, y)   + Ixx(x, y+1) +\
                               Ixx(x+1, y-1) + Ixx(x+1, y) + Ixx(x+1, y+1)) ]

    Syy = Function(([x, y], [row, col]), Float, "Syy")
    Syy.defn = [ Case(condRed, Iyy(x-1, y-1) + Iyy(x-1, y) + Iyy(x-1, y+1) +\
                               Iyy(x, y-1)   + Iyy(x, y)   + Iyy(x, y+1) +\
                               Iyy(x+1, y-1) + Iyy(x+1, y) + Iyy(x+1, y+1)) ]

    Sxy = Function(([x, y], [row, col]), Float, "Sxy")
    Sxy.defn = [ Case(condRed, Ixy(x-1, y-1) + Ixy(x-1, y) + Ixy(x-1, y+1) +\
                               Ixy(x, y-1)   + Ixy(x, y)   + Ixy(x, y+1) +\
                               Ixy(x+1, y-1) + Ixy(x+1, y) + Ixy(x+1, y+1)) ]

    det = Function(([x, y], [row, col]), Float, "det")
    det.defn = [ Case(condRed, Sxx(x, y) * Syy(x, y) - Sxy(x, y) * Sxy(x, y)) ]

    trace = Function(([x, y], [row, col]), Float, "trace")
    trace.defn = [ Case(condRed, Sxx(x, y) + Syy(x, y)) ]

    harris = Function(([x, y], [row, col]), Float, "harris")
    harris.defn = [ Case(condRed,
                         det(x, y) - 0.04 * trace(x, y) * trace(x, y)) ]

    # test boundary case
    harris.defn.append(Case(zeroCond, 0.0))

    groups = [[Ix, Iy, Ixx, Iyy, Ixy], \
              [Sxx, Sxy, Syy], \
              [det, trace, harris]]
    '''
    groups = [[Ix, Iy], \
              [Sxx, Sxy, Syy], \
              [harris]]
    '''

    p_est = [ (R, 1024), (C, 1024) ]

    # build the pipeline
    pipeline = buildPipeline([harris],
                             #grouping = groups,
                             inline_directives = [Ixx, Iyy, Ixy, det, trace],
                             param_estimates = p_est,
                             pipe_name = "harris")

    filename = "harris_graph"
    dot_file = filename+".dot"
    png_file = filename+".png"
    g = pipeline.pipeline_graph
    g.write(filename+".dot")
    dotty_str = "dot -Tpng "+dot_file+" -o "+png_file
    subprocess.check_output(dotty_str, shell=True)

    filename = 'harris_naive.cpp'
    c_file = open(filename, 'w')
    c_file.write(pipeline.generate_code().__str__())
    c_file.close()


import sys

from pyndn.security.policy import ConfigPolicyManager
from pyndn import Name

from pyndn.security.security_exception import SecurityException
from pyndn.util.boost_info_parser import BoostInfoParser

import os

"""
This module implements a simple hierarchical trust model that uses certificate
data to determine whether another signature/name can be trusted.

The policy manager enforces an environment, which corresponds to the network
prefix, i.e. the root of the network namespace.
All command interests must be signed with a certificate in this environment
to be trusted.

There is a root name and public key which must be the top authority in the environment
for the certificate to be trusted.
"""

class IotPolicyManager(ConfigPolicyManager):
    def __init__(self, identityStorage, configFilename=None):
        """
        :param pyndn.IdentityStorage: A class that stores signing identities and certificates.
        :param str configFilename: A configuration file specifying validation rules and network
            name settings.
        """

        # use the default configuration where possible
        # TODO: use environment variable for this, fall back to default
        path = os.path.dirname(__file__)
        templateFilename = os.path.join(path, '.default.conf')
        self._configTemplate = BoostInfoParser()
        self._configTemplate.read(templateFilename)

        if configFilename is None:
            configFilename = templateFilename

        super(IotPolicyManager, self).__init__(identityStorage, configFilename)
        self.setEnvironmentPrefix(None)
        self.setTrustRootIdentity(None)
        self.setDeviceIdentity(None)

    def updateTrustRules(self):
        """
        Should be called after either the device identity, trust root or network
        prefix is changed.

        Not called automatically in case they are all changing (typical for
        bootstrapping).

        Resets the validation rules if we don't have a trust root or enivronment

        """
        validatorTree = self._configTemplate["validator"][0].clone()

        if (self._environmentPrefix.size() > 0 and
            self._trustRootIdentity.size() > 0 and
            self._deviceIdentity.size() > 0):
            # don't sneak in a bad identity
            if not self._environmentPrefix.match(self._deviceIdentity):
                raise SecurityException("Device identity does not belong to configured network!")

            environmentUri = self._environmentPrefix.toUri()
            deviceUri = self._deviceIdentity.toUri()

            for rule in validatorTree["rule"]:
                ruleId = rule["id"][0].value
                if ruleId == 'Certificate Trust':
                    #modify the 'Certificate Trust' rule
                    rule["checker/key-locator/name"][0].value = environmentUri
                elif ruleId == 'Command Interests':
                    rule["filter/name"][0].value = deviceUri
                    rule["checker/key-locator/name"][0].value = environmentUri

        #remove old validation rules from config
        # replace with new validator rules
        self.config._root.subtrees["validator"] = [validatorTree]

    def inferSigningIdentity(self, fromName):
        """
        Used to map Data or Interest names to identitites.
        :param pyndn.Name fromName: The name of a Data or Interest packet
        """
        # works if you have an IotIdentityStorage
        return self._identityStorage.inferIdentityForName(fromName)

    def setTrustRootIdentity(self, identityName):
        """
        : param pyndn.Name identityName: The new identity to trust as the controller.
        """
        self._trustRootIdentity = Name(identityName)

    def getTrustRootIdentity(self):
        """
        : return pyndn.Name: The trusted controller's network name.
        """
        return Name(self._trustRootIdentity)

    def setEnvironmentPrefix(self, name):
        """
        : param pyndn.Name name: The new root of the network namespace (network prefix)
        """
        self._environmentPrefix = Name(name)

    def getEnvironmentPrefix(self):
        """
        :return: The root of the network namespace
        :rtype: pyndn.Name
        """
        return Name(self._environmentPrefix)

    def getDeviceIdentity(self):
        return self._deviceIdentity

    def setDeviceIdentity(self, identity):
        self._deviceIdentity = Name(identity)

    def hasRootCertificate(self):
        """
        :return: Whether we've downloaded the controller's network certificate
        :rtype: boolean
        """
        try:
            rootCertName = self._identityStorage.getDefaultCertificateNameForIdentity(
                    self._trustRootIdentity)
        except SecurityException:
            return False

        try:
            rootCert = self._identityStorage.getCertificate(rootCertName)
            if rootCert is not None:
                return True
        finally:
            return False

    def hasRootSignedCertificate(self):
        """
        :return: Whether we've received a network certificate from our controller
        :rtype: boolean
        """
        try:
            myCertName = self._identityStorage.getDefaultCertificateNameForIdentity(
                       self._deviceIdentity)
            myCert = self._identityStorage.getCertificate(myCertName)
            if self._trustRootIdentity.match(
                   myCert.getSignature().getKeyLocator().getKeyName()):
               return True
        except SecurityException:
           pass

        return False

    def removeTrustRules(self):
        """
        Resets the network prefix, device identity and trust root identity to
         empty values
        """
        self.setDeviceIdentity(None)
        self.setTrustRootIdentity(None)
        self.setEnvironmentPrefix(None)
        self.updateTrustRules()


#!/usr/bin/env python
#
# Sample Hello World! python script...
#

import subprocess
import sys
import os

animals = ["camel", "llama", "owl"]
fruit_color = { "apple":"red", "banana":"yellow"}
fruits = fruit_color.keys()
colors = fruit_color.items()
verinfo = subprocess.check_output(["git", "describe", "--abbrev=7",
		"--dirty", "--always", "--tags"])

# Obligatory Hello World with some version info
print("Hello Python World!  Version: {}".format(verinfo))

# Show list of command line arguments
print("Command line args: "),
for i in sys.argv[1:]:
	print(" {}".format(i))
print("")

# Determine number of physical CPU sockets
if os.name == 'nt':
	pass
else:
	socketcount = subprocess.check_output(["bash","-c",
            "(cat /proc/cpuinfo | grep 'physical id' | sort | uniq | wc -l)"])
	print("CPU socket count: {}".format(socketcount))

# print out our animals array in reverse-sorted order
print("\nValue of animals array:")
for i in sorted(animals, reverse=True):
	print("item = " + i)
	if (i == animals[0]):
		print("   Condition is TRUE, \"" +i+ "\" == " + animals[0])
	else:
		print("   Condition is FALSE, \"" +i+ "\" != " + animals[0])

# print our fruit_color dictionary (key/value pairs)
print("\nValue of fruit_color hash:")
for fruit,color in sorted(fruit_color.items(), reverse=True):
	print(fruit, " = ", color)

# Read a non-existent environment variable and supply default value...
some_env_var = os.getenv("PROBABLY_NOT_THERE", "Default Value")
print("\nPROBABLY_NOT_THERE is set to '"+some_env_var+"'\n")

# Let's read in the PATH variable from the OS and print it out...
path_env_var = os.environ['PATH']
print("OS PATH var is set to: '"+path_env_var+"'\n")

# Let's manipulate PATH variable and then restore it
print("Adding :/test to PATH variable")
os.environ['PATH'] = os.environ['PATH'] + ":/test"
print("OS PATH var is set to: '"+os.environ['PATH']+"'\n")
print("Restoring original PATH value.")
os.environ['PATH'] = path_env_var
print("OS PATH var is set to: '"+path_env_var+"'\n")

# Let's create the file test.out and write animals array to it...
print("\nWriting test.out file...")
file = open("test.out", "w")
for i in animals:
	file.write(i+"\n")
file.close()

# Now open the file we wrote and read it back...
print("\nContents of test.out file:")
file = open("test.out", "r")
linenum = 1
for line in file:
	print("line "+str(linenum)+": "+line)
	linenum += 1
file.close()

# All done, finish with normal exit code.
sys.exit(0)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2018/7/27 18:01
# @Author  : zhouie
# @File    : main.py
# @Software: PyCharm

import speech
import time
import csv

data = []  # 暂存 *.csv 文件每行数据
Interval_Time = 4  # 两次读词的间隔时间
LOOP_NUM = 8  # 循环基数
Think_Time = 15  # 回顾等待时间

csv_file = open('./res/test.csv', encoding='utf-8')
csv_reader_lines = csv.reader(csv_file)
# print(csv_reader_lines)

num = 0
for one_line in csv_reader_lines:
    data.append(one_line)
    num = num + 1

speech.say("计算机专业相关的英语单词 中英互译 测试小程序，demo版")
speech.say("This is a small routine (compiled by Python) for exercise about English phrases in the field of computer")

i = 0
while i < num:
    # print(i + 1, data[i][0])
    # speech.say(i + 1)
    # speech.say(data[i][0])
    # time.sleep(Interval_Time)
    # speech.say(data[i][0])
    if 0 == (i + 1) % LOOP_NUM:
        speech.say("来回顾一下 以上所学的几个词汇吧")
        speech.say("Just follow me , look back on the words you have learned...")
        print("!--#######--*--#######--!")
        print("第", int(i / LOOP_NUM) + 1, "组词汇：")
        for j in range(i - (LOOP_NUM - 1), i + 1):
            print(data[j][0], data[j][1])
            speech.say(data[j][0])
            speech.say(data[j][1])
        print("!--#######--*--#######--!")
        speech.say("你的正确率如何呢？")
        speech.say("So , What about your correct rate?")
        time.sleep(Think_Time)
    i = i + 1

speech.say("Congratulation!")


# coding:utf-8
from bs4 import BeautifulSoup
import urllib.request as req
import os
from requests_oauthlib import OAuth1Session
import json

os.environ["NLS_LANG"] = "JAPANESE_JAPAN.AL32UTF8"

### functions #############################################################################################
def init():
    path = os.path.abspath(os.path.dirname(__file__))
    return {
        'crawled_log' : path + '/press_released.log',
        'crawled_now' : path + '/press_release_now.log',
    }

def crawl_nikkei():
    url = "http://www.nikkei.com/pressrelease/?searchKeyword=%E4%B8%96%E7%95%8C%E5%88%9D%E3%80%80%E3%81%AB%E6%88%90%E5%8A%9F&au=0"
    res = req.urlopen(url)
    soup = BeautifulSoup(res, "html.parser")
    news_records = soup.select(".m-newsListDotBorder_link")

    f = open(settings['crawled_now'], 'w')
    for record in news_records:
        _str = record.find("span", {"class" : "m-newsListDotBorder_text"}).string
        if _str.count('に成功') or _str.count('を開発'):
            fbk = open(settings['crawled_log'], 'r')
            href = record.get("href")
            if fbk.read().count(href) < 1:
                print(href)
                f.write("http://www.nikkei.com" + href + "\n")
                fbk.close()
                fbk = open(settings['crawled_log'], 'a')
                fbk.write("http://www.nikkei.com" + href + "\n")
            fbk.close()
    f.close()

def tweet(tweet_list):
    twitter = OAuth1Session(
        's2pdn7qCF87cYcHaKPjvoNYl0',
        'PIWmcvN7Qlu76hnNI2ADRaYU3cclo139I0HRazl7uWeFm1XgsQ',
        '855057045992595456-TWXOYZcGabFanh9I8ATzkiVG7zFj4m2',
        'medD4a2rUHP0LlsAsuXqeZDZYjeowvHcp4bSpnJi0zkVL'
    )
    for line in tweet_list:
        txt = line.encode('utf-8')
        params = {"status": txt}
        req = twitter.post("https://api.twitter.com/1.1/statuses/update.json",params = params)

### main #############################################################################################
settings = init()

crawl_nikkei()
tweet(open(settings['crawled_now'], 'r'))


import logging
import numpy as np

import scipy.ndimage as ndim
from skimage.morphology import binary_dilation, convex_hull_image
from scipy.spatial.distance import dice

import nibabel as nib

from dipy.io.utils import (create_nifti_header, get_reference_info)
from dipy.tracking.streamline import select_random_set_of_streamlines
import dipy.tracking.utils as dtu

logger = logging.getLogger('AFQ')

def transform_inverse_roi(roi, mapping, bundle_name="ROI"):
    """
    After being non-linearly transformed, ROIs tend to have holes in them.
    We perform a couple of computational geometry operations on the ROI to
    fix that up.

    Parameters
    ----------
    roi : Nifti1Image, str, ndarray
        The ROI to transform. Can be a path or image, which will be
        converted to an ndarray.

    mapping : DiffeomorphicMap object
        A mapping between DWI space and a template.

    bundle_name : str, optional
        Name of bundle, which may be useful for error messages.
        Default: None

    Returns
    -------
    ROI after dilation and hole-filling
    """
    if isinstance(roi, str):
        roi = nib.load(roi)
    if isinstance(roi, nib.Nifti1Image):
        roi = roi.get_fdata()

    _roi = mapping.transform_inverse(roi, interpolation='linear')

    if np.sum(_roi) == 0:
        logger.warning(
            f'Lost ROI {bundle_name}, performing automatic binary dilation')
        _roi = binary_dilation(roi)
        _roi = mapping.transform_inverse(_roi, interpolation='linear')

    _roi = patch_up_roi(_roi > 0, bundle_name=bundle_name).astype(np.int32)

    return _roi

def patch_up_roi(roi, bundle_name="ROI"):
    """
    After being non-linearly transformed, ROIs tend to have holes in them.
    We perform a couple of computational geometry operations on the ROI to
    fix that up.

    Parameters
    ----------
    roi : 3D binary array
        The ROI after it has been transformed.

    bundle_name : str, optional
        Name of bundle, which may be useful for error messages.
        Default: None

    Returns
    -------
    ROI after dilation and hole-filling
    """
    hole_filled = ndim.binary_fill_holes(roi > 0)
    if not np.any(hole_filled):
        raise ValueError((
            f"{bundle_name} found to be empty after "
            "applying the mapping."))
    return hole_filled

def density_map(tractogram, n_sls=None, normalize=False):
    """
    Create a streamline density map.
    based on:
    https://dipy.org/documentation/1.1.1./examples_built/streamline_formats/

    Parameters
    ----------
    tractogram : StatefulTractogram
        Stateful tractogram whose streamlines are used to make
        the density map.
    n_sls : int or None, optional
        n_sls to randomly select to make the density map.
        If None, all streamlines are used.
        Default: None
    normalize : bool, optional
        Whether to normalize maximum values to 1.
        Default: False

    Returns
    -------
    Nifti1Image containing the density map.
    """
    tractogram.to_vox()

    sls = tractogram.streamlines
    if n_sls is not None:
        sls = select_random_set_of_streamlines(sls, n_sls)

    affine, vol_dims, voxel_sizes, voxel_order = get_reference_info(tractogram)
    tractogram_density = dtu.density_map(sls, np.eye(4), vol_dims)
    if normalize:
        tractogram_density = tractogram_density / tractogram_density.max()

    nifti_header = create_nifti_header(affine, vol_dims, voxel_sizes)
    density_map_img = nib.Nifti1Image(tractogram_density, affine, nifti_header)

    return density_map_img

def dice_coeff(arr1, arr2, weighted=True):
    """
    Compute Dice's coefficient between two images.

    Parameters
    ----------
    arr1 : Nifti1Image, str, ndarray
        One ndarray to compare. Can be a path or image, which will be
        converted to an ndarray.
    arr2 : Nifti1Image, str, ndarray
        The other ndarray to compare. Can be a path or image, which will be
        converted to an ndarray.
    weighted : bool, optional
        Whether or not to weight the DICE coefficient as in [Cousineau2017]_.
        The weighted Dice coefficient is calculated by adding the sum of all
        values in arr1 where arr2 is nonzero to the sum of all values in arr2
        where arr1 is nonzero, then dividing that by the sum of all values in
        arr1 and arr2.
        Default: True

    Returns
    -------
    The dice similarity between the images.

    Notes
    -----
    .. [1] Cousineau M, Jodoin PM, Morency FC, et al. A test-retest study on
           Parkinson's PPMI dataset yields statistically significant white
           matter fascicles. Neuroimage Clin. 2017;16:222-233. Published 2017
           Jul 25. doi:10.1016/j.nicl.2017.07.020
    """
    if isinstance(arr1, str):
        arr1 = nib.load(arr1)
    if isinstance(arr2, str):
        arr2 = nib.load(arr2)

    if isinstance(arr1, nib.Nifti1Image):
        arr1 = arr1.get_fdata()
    if isinstance(arr2, nib.Nifti1Image):
        arr2 = arr2.get_fdata()

    arr1 = arr1.flatten()
    arr2 = arr2.flatten()

    if weighted:
        return (
            np.sum(arr1 * arr2.astype(bool))
            + np.sum(arr2 * arr1.astype(bool)))\
            / (np.sum(arr1) + np.sum(arr2))
    else:
        # scipy's dice function returns the dice *dissimilarity*
        return 1 - dice(
            arr1.astype(bool),
            arr2.astype(bool))


import os

import mysql.connector
import boto3
import hashlib

BUCKET_NAME = 'gge-opiniones'
keyFile = open('./Resources/keys.txt', 'r') #Keyfile is in gitignore. Must be added manually
PUBLIC_KEY = keyFile.readline().rstrip()
PRIVATE_KEY = keyFile.readline().rstrip()

class DB_Driver:

    def __init__(self):
        self.session = boto3.Session(
            aws_access_key_id= PUBLIC_KEY,
            aws_secret_access_key= PRIVATE_KEY
        )
        self.s3 = self.session.resource('s3')
        self.bucket = self.s3.Bucket(BUCKET_NAME)

        self.connection = mysql.connector.connect(host="localhost",
                                                  port = "3306",
                                                  database="proyecto2",
                                                  user="root",
                                                  password="",
                                                  use_pure=True)

        if self.connection.is_connected():
            db_Info = self.connection.get_server_info()
            print("Connected to MySQL database... MySQL Server version on ", db_Info)
            cursor = self.connection.cursor()
            cursor.execute("select database();")
            record = cursor.fetchone()
            print("You are connected to - ", record)
        else:
            raise AssertionError('Connection to the DB could not be established')

        self.cursor = self.connection.cursor(prepared = True)

    def sanitizeInput(self,input):
        sanitized=""
        for character in input:
            if character == "'" or character ==")" or character == ";":
                break
            else:
                sanitized += character
        return sanitized

    def closeConnection(self):
        self.connection.close()

if __name__ == "__main__":
    #tests go here if needed
    db = DB_Driver()
    model = db.getModels()
    for i in model:
        print(i[0])


#!/usr/bin/env python3
import sys
import os

minimum_latency = [ 6610, 9870 ] # Minimum nanoseconds for read and write
#minimum_latency = [ 0, 0 ] # Minimum nanoseconds for read and write

if len(sys.argv) == 2 and sys.argv[1] == "max":
    delay_initial = [1, 1]
    per_op_latency = [1, 1]
    io_unit_shift = 15
else:
    if len(sys.argv) < 4:
        print("Usage: %s [Read latency] [Write latency] [Read target bandwidth] {Write target bandwidth}" % (sys.argv[0]))
        sys.exit(1)

    target_latency = [ float(sys.argv[1]), float(sys.argv[2]) ]
    target_bw = [ float(sys.argv[3]), float(sys.argv[3]) ]
    if len(sys.argv) == 5:
        target_bw[1] = float(sys.argv[4])

    io_unit_shift = 12
    io_unit_size = 1 << (io_unit_shift - 10)

    # The number of operations per second = bandwidth / per-operation size
    nr_ops = [ bw * 1024 / io_unit_size for bw in target_bw ]

    # Per-operation latency = 1 / # of ops
    per_op_latency = [ 1 / ops * 1000000000 for ops in nr_ops]

    delay_initial = [ max(target * 1000 - per_op - m, 1) for (target, per_op, m) in zip(target_latency, per_op_latency, minimum_latency) ]

os.system("sudo sh -c \"echo %d %d %d > /proc/nvmev/read_times\"" % (delay_initial[0], per_op_latency[0], 0))
os.system("sudo sh -c \"echo %d %d %d > /proc/nvmev/write_times\"" % (delay_initial[1], per_op_latency[1], 0))
os.system("sudo sh -c \"echo %d %d > /proc/nvmev/io_units\"" % (1, io_unit_shift))


import datetime
from io import BytesIO, StringIO
import json
import mock
import pytz
import os

from django.test import TestCase
from django.conf import settings
from django.utils import timezone

from identifiers import logic, models
from core.models import SettingGroup
from journal import logic as journal_logic
from submission import models as submission_models
from utils.testing import helpers
from utils.setting_handler import save_setting
from utils.shared import clear_cache
from lxml import etree
from bs4 import BeautifulSoup
import requests
class TestLogic(TestCase):

    @classmethod
    def setUpTestData(cls):

        # Create press and journals
        cls.press = helpers.create_press()
        cls.press.save()
        cls.journal_one, cls.journal_two = helpers.create_journals()

        # Configure settings
        for journal in [cls.journal_one, cls.journal_two]:
            save_setting('general', 'journal_issn', journal, '1234-5678')
            save_setting('general', 'print_issn', journal, '8765-4321')
            save_setting('Identifiers', 'use_crossref', journal, True)
            save_setting('Identifiers', 'crossref_prefix', journal, '10.0000')
            save_setting('Identifiers', 'crossref_email', journal, 'sample_email@example.com')
            save_setting('Identifiers', 'crossref_name', journal, 'Journal Name')
            save_setting('Identifiers', 'crossref_registrant', journal, 'registrant')

        # Make mock request
        cls.request = helpers.Request()
        cls.request.press = cls.press
        cls.request.journal = cls.journal_one

        # Issue 5.3 has one article
        cls.issue_five_three = helpers.create_issue(cls.journal_one, vol=5, number=3)

        cls.article_one = helpers.create_article(cls.journal_one, with_author=True)
        cls.article_published = helpers.create_article(cls.journal_one, with_author=True)
        cls.article_published.stage = submission_models.STAGE_PUBLISHED
        cls.article_published.date_published = timezone.now()
        cls.article_published.save()

        cls.doi_one = logic.generate_crossref_doi_with_pattern(cls.article_one)
        cls.issue_five_three.articles.add(cls.article_one)
        cls.article_one.primary_issue = cls.issue_five_three
        cls.article_one.abstract = 'Test abstract.'
        cls.article_one.snapshot_authors()
        cls.article_one.license = submission_models.Licence.objects.filter(
            journal=cls.journal_one,
        ).first()
        cls.article_one.page_numbers = '1-72'
        cls.article_one.save()

        cls.issue_five_three.save()

        # Issue 6.1 has two articles that should be registered together
        # in a single Crossref journal issue block
        cls.issue_six_one = helpers.create_issue(cls.journal_one, vol=6, number=1)

        cls.article_two = helpers.create_article(cls.journal_one, with_author=True)
        cls.doi_two = logic.generate_crossref_doi_with_pattern(cls.article_two)
        cls.issue_six_one.articles.add(cls.article_two)
        cls.article_two.primary_issue = cls.issue_six_one
        cls.article_two.save()

        cls.article_three = helpers.create_article(cls.journal_one, with_author=True)
        doi_options = {
            'id_type': 'doi',
            'identifier': '10.1234/custom',
            'article': cls.article_three,
        }
        cls.doi_three = models.Identifier.objects.create(**doi_options)
        cls.issue_six_one.articles.add(cls.article_three)
        cls.article_three.primary_issue = cls.issue_six_one
        cls.article_three.save()

        # But issue 6.1 also has another couple articles that should be registered individually
        # because they have special attributes
        cls.article_four = helpers.create_article(cls.journal_one, with_author=True)
        cls.article_four.ISSN_override = '5555-5555'
        cls.doi_four = logic.generate_crossref_doi_with_pattern(cls.article_four)
        cls.issue_six_one.articles.add(cls.article_four)
        cls.article_four.primary_issue = cls.issue_six_one
        cls.article_four.save()

        cls.article_five = helpers.create_article(cls.journal_one, with_author=True)
        cls.article_five.publication_title = 'A Very Special Old Publication'
        cls.doi_five = logic.generate_crossref_doi_with_pattern(cls.article_five)
        cls.issue_six_one.articles.add(cls.article_five)
        cls.article_five.primary_issue = cls.issue_six_one
        cls.article_five.save()

        cls.issue_six_one.save()

        # Journal 2 is a conference
        cls.journal_two.is_conference = True
        cls.issue_nine_nine = helpers.create_issue(cls.journal_two, vol=9, number=9)

        cls.article_six = helpers.create_article(cls.journal_two, with_author=True)
        cls.doi_six = logic.generate_crossref_doi_with_pattern(cls.article_six)
        cls.issue_nine_nine.articles.add(cls.article_six)
        cls.article_six.primary_issue = cls.issue_nine_nine
        cls.article_six.abstract = 'Test abstract.'
        cls.article_six.snapshot_authors()
        cls.article_six.license = submission_models.Licence.objects.filter(
            journal=cls.journal_two,
        ).first()
        cls.article_six.page_numbers = '58-62'
        cls.article_six.save()

        cls.article_seven = helpers.create_article(cls.journal_two, with_author=True)
        cls.doi_seven = logic.generate_crossref_doi_with_pattern(cls.article_seven)
        cls.issue_nine_nine.articles.add(cls.article_seven)
        cls.article_seven.primary_issue = cls.issue_nine_nine
        cls.article_seven.save()

        cls.issue_nine_nine.save()

        # Schema location for Crossref XML validation
        cls.schema_base_path = os.path.join(
            settings.BASE_DIR,
            'identifiers',
            'tests',
            'test_data',
            'schemas',
        )

    def test_create_crossref_doi_batch_context(self):
        self.maxDiff = None
        expected_data = {}

        expected_data['depositor_email'] = 'sample_email@example.com'
        expected_data['depositor_name'] = 'Journal Name'
        expected_data['registrant'] = 'registrant'
        expected_data['is_conference'] = self.journal_one.is_conference

        context = logic.create_crossref_doi_batch_context(
            self.journal_one,
            set([self.doi_one])
        )

        # A couple things need to be adjusted with context for test to work
        for key in ['batch_id', 'now', 'timestamp', 'timestamp_suffix']:
            context.pop(key)

        # Don't test lower levels of nested context in this test
        expected_data['crossref_issues'] = []
        if 'crossref_issues' in context:
            context['crossref_issues'] = []

        self.assertEqual(expected_data, context)

    def test_create_crossef_issues_context(self):
        # Just expect the right number of crossref_issues
        # each with the right keys
        expected_data = [
            {'journal':None,'issue':None,'articles':None} for x in range(4)
        ]

        identifiers = set()
        identifiers.add(self.doi_one) # Should be on its own in 5.3
        identifiers.add(self.doi_two) # Should go together with below in 6.1
        identifiers.add(self.doi_three) # Should go together with above in 6.1
        identifiers.add(self.doi_four) # Should be on its own due to Article.ISSN_override
        identifiers.add(self.doi_five) # Should be on its own due to Article.publication_title
        context = logic.create_crossref_issues_context(
            self.journal_one,
            identifiers,
        )

        # Knock out the lower levels of data
        for i, crossref_issue in enumerate(context):
            for key in context[i].keys():
                context[i][key] = None

        self.assertEqual(expected_data, context)

    def test_create_crossref_issue_context(self):
        expected_issue = self.issue_five_three
        expected_number_of_articles = 1
        context = logic.create_crossref_issue_context(
            self.journal_one,
            set([self.doi_one]),
            self.issue_five_three,
        )
        self.assertEqual(expected_issue, context['issue'])
        self.assertEqual(expected_number_of_articles, len(context['articles']))

        expected_issue = self.issue_six_one
        expected_number_of_articles = 2
        context = logic.create_crossref_issue_context(
            self.journal_one,
            set([self.doi_two, self.doi_three]),
            self.issue_six_one,
        )
        self.assertEqual(expected_issue, context['issue'])
        self.assertEqual(expected_number_of_articles, len(context['articles']))

    def test_create_crossref_journal_context(self):
        expected_data = {
            'title': 'Journal One',
            'journal_issn': '1234-5678',
            'print_issn': '8765-4321',
            'press': self.journal_one.press
        }
        context = logic.create_crossref_journal_context(self.journal_one)
        self.assertEqual(expected_data, context)

    def test_create_crossref_article_context_published(self):
        self.maxDiff = None
        expected_data = {
            'title': self.article_published.title,
            'abstract': '',
            'url': self.article_published.url,
            'authors': [
                author.email for author in self.article_published.frozenauthor_set.all()
            ],
            'citation_list': None,
            'date_accepted': None,
            'date_published': self.article_published.date_published,
            'doi': f'10.0000/TST.{self.article_published.id}',
            'id': self.article_published.id,
            'license': '',
            'pages': self.article_published.page_numbers,
            'scheduled': True,
        }

        context = logic.create_crossref_article_context(self.article_published)
        context['authors'] = [author.email for author in context['authors']]
        self.assertEqual(expected_data, context)

    def test_create_crossref_article_context_not_published(self):
        expected_data = {
            'title': self.article_one.title,
            'abstract': self.article_one.abstract,
            'url': self.article_one.url,
            'authors': [
                author.email for author in self.article_one.frozenauthor_set.all()
            ],
            'citation_list': None,
            'date_accepted': None,
            'date_published': None,
            'doi': self.doi_one.identifier,
            'id': self.article_one.id,
            'license': submission_models.Licence.objects.filter(
                journal=self.journal_one,
            ).first().url,
            'pages': self.article_one.page_numbers,
            'scheduled': False,
        }

        context = logic.create_crossref_article_context(self.article_one)
        context['authors'] = [author.email for author in context['authors']]

        self.assertEqual(expected_data, context)

    def test_preview_registration_information_when_use_crossref_on(self):
        clear_cache()
        save_setting('Identifiers', 'use_crossref', self.journal_one, True)
        self.assertTrue('Current metadata to send to Crossref' in self.article_one.registration_preview)

    def test_preview_registration_information_when_use_crossref_off(self):
        clear_cache()
        save_setting('Identifiers', 'use_crossref', self.journal_one, False)
        self.assertFalse(self.article_one.registration_preview)

    def test_preview_registration_information_when_custom_doi(self):
        clear_cache()
        self.assertTrue(self.doi_three.identifier in self.article_three.registration_preview)

    def test_deposit_xml_document_is_valid(self):
        self.maxDiff = None

        for journal in [self.journal_one, self.journal_two]:
            # Generate batch and validate against schema
            template = 'common/identifiers/crossref_doi_batch.xml'
            identifiers = set([identifier for identifier in models.Identifier.objects.filter(
                article__journal=journal
            )])
            template_context = logic.create_crossref_doi_batch_context(
                journal,
                identifiers,
            )
            deposit = logic.render_to_string(template, template_context)

            soup = BeautifulSoup(deposit, 'lxml')
            version = soup.find('doi_batch')['version']
            schema_filename = f'crossref{version}.xsd'
            with open(os.path.join(self.schema_base_path, schema_filename)) as fileref:
                xml_schema_doc = etree.parse(fileref)
            xml_schema = etree.XMLSchema(xml_schema_doc)

            deposit_bytes = BytesIO(str.encode(deposit))
            doc = etree.parse(deposit_bytes)
            xml_schema.assertValid(doc)
            self.assertTrue(xml_schema.validate(doc))

    def test_journal_deposit_xml_document_has_basically_correct_components(self):
        template = 'common/identifiers/crossref_doi_batch.xml'
        identifiers = set([identifier for identifier in models.Identifier.objects.filter(
            article__journal=self.journal_one
        )])
        template_context = logic.create_crossref_doi_batch_context(
            self.journal_one,
            identifiers,
        )
        deposit = logic.render_to_string(template, template_context)
        soup = BeautifulSoup(deposit, 'lxml')
        # There should be one doi_batch
        self.assertEqual(1, len(soup.find_all('doi_batch')))
        # There should be four crossref_issues
        self.assertEqual(4, len(soup.find_all('journal_issue')))
        # And so journal metadata should appear four times, once for each issue
        self.assertEqual(4, len(soup.find_all('journal_metadata')))
        # There should be five articles
        self.assertEqual(5, len(soup.find_all('journal_article')))

    def test_conference_deposit_xml_document_has_basically_correct_components(self):
        template = 'common/identifiers/crossref_doi_batch.xml'
        identifiers = set([identifier for identifier in models.Identifier.objects.filter(
            article__journal=self.journal_two
        )])
        template_context = logic.create_crossref_doi_batch_context(
            self.journal_two,
            identifiers,
        )
        deposit = logic.render_to_string(template, template_context)
        soup = BeautifulSoup(deposit, 'lxml')
        # There should be one doi_batch
        self.assertEqual(1, len(soup.find_all('doi_batch')))
        # There should be one conference (crossref_issue)
        self.assertEqual(1, len(soup.find_all('conference')))
        # And so conference metadata should appear one time, once for each issue
        self.assertEqual(1, len(soup.find_all('event_metadata')))
        # There should be two conference papers (articles)
        self.assertEqual(2, len(soup.find_all('conference_paper')))

    def test_issue_doi_deposited_correctly(self):
        template = 'common/identifiers/crossref_doi_batch.xml'
        issue = self.article_one.issue
        issue.doi = issue_doi = "10.0001/issue"
        issue.save()
        identifier = self.article_one.get_doi_object
        clear_cache()

        template_context = logic.create_crossref_doi_batch_context(
            self.article_one.journal,
            {identifier},
        )
        deposit = logic.render_to_string(template, template_context)

        soup = BeautifulSoup(deposit, 'lxml')
        # There should be one doi_batch
        issue_soup = soup.find('journal_issue')
        found = False
        if issue_soup:
            issue_doi_soup = issue_soup.find("doi_data")
            if issue_doi_soup:
                self.assertEqual(issue_doi_soup.find("doi").string, issue_doi)
                self.assertEqual(issue_doi_soup.find("resource").string, issue.url)
                found = True
        if not found:
            raise AssertionError("No Issue DOI found on article deposit")

    def test_journal_doi_deposited_correctly(self):
        template = 'common/identifiers/crossref_doi_batch.xml'
        issue = self.article_one.issue
        journal_doi = "10.0001/journal"
        save_setting('Identifiers', 'title_doi', issue.journal, journal_doi)
        identifier = self.article_one.get_doi_object
        clear_cache()

        template_context = logic.create_crossref_doi_batch_context(
            self.article_one.journal,
            {identifier},
        )
        deposit = logic.render_to_string(template, template_context)

        soup = BeautifulSoup(deposit, 'lxml')
        # There should be one doi_batch
        journal_soup = soup.find('journal_metadata')
        found = False
        if journal_soup:
            doi_soup = journal_soup.find("doi_data")
            if doi_soup:
                self.assertEqual(doi_soup.find("doi").string, journal_doi)
                self.assertEqual(
                    doi_soup.find("resource").string, issue.journal.site_url())
                found = True
        if not found:
            raise AssertionError("No Issue DOI found on article deposit")

    def test_issue_doi_auto_assign_enabled(self):
        issue = helpers.create_issue(self.journal_one, vol=99, number=99)
        self.request.POST = {"assign_issue": issue.pk}
        mock_messages = mock.patch('journal.logic.messages').start()
        mock_messages.messages = mock.MagicMock()
        save_setting('Identifiers', 'register_issue_dois', self.journal_one, 'on')
        from events import registration # Forces events to load into memory
        journal_logic.handle_assign_issue(self.request, self.article_one, issue)
        issue.refresh_from_db()
        self.assertTrue(issue.doi)

    def test_issue_doi_auto_assigned_disabled(self):
        issue = helpers.create_issue(self.journal_one, vol=99, number=99)
        self.request.POST = {"assign_issue": issue.pk}
        mock_messages = mock.patch('journal.logic.messages').start()
        mock_messages.messages = mock.MagicMock()
        save_setting('Identifiers', 'register_issue_dois', self.journal_one, '')
        from events import registration # Forces events to load into memory
        journal_logic.handle_assign_issue(self.request, self.article_one, issue)
        issue.refresh_from_db()
        self.assertEqual(issue.doi, None)

    def test_check_crossref_settings(self):

        # Missing settings
        save_setting('Identifiers', 'crossref_prefix', self.journal_one, '')
        save_setting('Identifiers', 'crossref_username', self.journal_one, '')
        save_setting('Identifiers', 'crossref_password', self.journal_one, '')

        use_crossref, test_mode, missing_settings = logic.check_crossref_settings(
            self.journal_one
        )

        # Put need missing setting back
        save_setting('Identifiers', 'crossref_prefix', self.journal_one, '10.0000')

        self.assertEqual(use_crossref, True)
        self.assertEqual(missing_settings, ['crossref_prefix', 'crossref_username', 'crossref_password'])


import torch
import numpy as np
import json
from model2 import KeyPointsTransformer
from collections import Counter

def load_data_from_json(json_path):
    with open(json_path, "r") as f:
        json_data = json.load(f)

    keypoints_list = []
    for annotation in json_data['annotations']:
        if 'keypoints' in annotation:  # Check if 'keypoints' key is present
            keypoints = annotation['keypoints']
            keypoints_list.append(keypoints)
        else:
            print(f"Skipping frame {annotation['image_id']} due to missing keypoints.")

    keypoints_array = np.array(keypoints_list)
    return keypoints_array.reshape(-1, 51)

def create_sequences(data, sequence_length):
    sequences = np.stack([data[i:i + sequence_length] for i in range(data.shape[0] - sequence_length + 1)], axis=0)
    return sequences

def predict(model, sequence):
    model.eval()

    # Convert the sequence to a tensor
    input_tensor = torch.tensor(sequence, dtype=torch.float32).unsqueeze(0)  # Add batch dimension

    with torch.no_grad():
        output = model(input_tensor)
    prediction = torch.argmax(output, dim=1).item()
    return prediction

def simplify_output(output):
    # Extract the class predictions from the output dictionary
    class_predictions = [frame['class'] for frame in output]

    # Count the occurrences of each class
    class_counts = Counter(class_predictions)

    # Find the most common class
    most_common_class, _ = class_counts.most_common(1)[0]

    # Return the simplified output
    return most_common_class

# Load the model
model_path = "D:/AIPROJECT2/models/trained_model2.pth"
num_classes = 6
model = KeyPointsTransformer(num_classes)
model.load_state_dict(torch.load(model_path))
model.eval()

# Load frames from JSON file
json_path = "D:/AIPROJECT2/upload/json/000000.json"
frames = load_data_from_json(json_path)
print("Frames shape:", frames.shape)

# Create sequences
sequence_length = 60
sequences = create_sequences(frames, sequence_length)

# Make predictions for each sequence
predictions = []
for i, sequence in enumerate(sequences):
    prediction = predict(model, sequence)
    predictions.append({"sequence": i + 1, "class": prediction})

simplified_output = simplify_output(predictions)

print(simplified_output)


from django.shortcuts import render,redirect
from home.models import Course
from django.db.models import Q
from django.views.generic import ListView
# Create your views here.

def search(request):
    query = request.GET.get('search')
    if query is None:
        return redirect('home')
    courses = Course.objects.filter(Q(name__contains=query)|Q(description__contains=query))
    params = {'courses':courses,'query':query}
    return render(request,'home/index.html',params)

class HomePageView(ListView):
    template_name = 'home/index.html'
    queryset = Course.objects.filter(active=True)
    context_object_name = "courses"


""""
Serializer for user.
"""
from rest_framework import serializers

from core.models import Address, User

class AddressSerializer(serializers.ModelSerializer):
    class Meta:
        model = Address
        exclude = ['user']
        read_only_fields = ['id', 'created_at', 'updated_at']

    def validate(self, data):
        user = self.context['request'].user
        address_line_1 = data.get('address_line_1')
        address_line_2 = data.get('address_line_2', '')
        city = data.get('city')
        country = data.get('country')
        address_type = data.get('type')
        is_primary = data.get('is_primary', False)

        existing_address = Address.objects.filter(
            user=user,
            address_line_1=address_line_1,
            address_line_2=address_line_2,
            city=city,
            country=country,
            type=address_type
        )

        if existing_address.exists():
            raise serializers.ValidationError("An address with the same details already exists.")

        if is_primary and Address.objects.filter(user=user, type=address_type, is_primary=True).exists():
            raise serializers.ValidationError("There is already a primary address with the same type.")

        return data

class UserSerializer(serializers.ModelSerializer):
    address = AddressSerializer(many=True, read_only=True)

    class Meta:
        model = User
        fields = ['id', 'first_name', 'last_name', 'email', 'phone', 'other_details', 'is_phone_verified',
                  'cart_total_price', 'address']
        read_only_fields = ['id', 'is_phone_verified', 'cart_total_price', 'address']


from numpy.lib.utils import source
import pandas as pd
from pandas import *
import glob
import os
import plotly.graph_objects as go
from supervenn import supervenn
import matplotlib.pyplot as plt
import re
import numpy as np
import seaborn as sns
import itertools
import sys, getopt

folder = "/home/elisa/LBDM-project/CTNNB1_csvs/*"

def retrieveName (f):
	base = f.split("/")[5].split(".")[0].split("@")[0]
	#base = "_".join(fname)
	return base

def readFile(fileName):
	fileObj = open(fileName, "r") #opens the file in read mode
	words = fileObj.read().splitlines() #puts the file into an array
	fileObj.close()
	return words

def findCommons(list1, list2):
	geness=[]
	for gene in list1:
		if (gene in list2):
			#print (gene)
			geness.append(gene)
	return geness

def retrieveARgs():
	inp=''
	emt=''
	try:
		opts, args = getopt.getopt(sys.argv[1:],"i:l:",["inputf=","list="]) #-i input -l list of emt genes
	except :
		print ('test.py -i <input folder> -l <EMTgenes.txt>')
		exit(2)
	for opt, arg in opts:
		if opt in ("-i", "--inputf"):
			inp = arg
		elif opt in ("-l", "--list"):
			emt = arg
	return inp, emt

###################################################################################################################
inputfolder, emtfile = retrieveARgs()
#print ('Input file is', inputfolder)
#print ('Output file is', emtfile)

files_dfs = []
for f in glob.glob(inputfolder+"/*"):

	if f.endswith('.csv') == False :
		continue

	#print (f)
	df = pd.read_csv(f,  sep=',', skiprows = 1, nrows=250)

	source=retrieveName(f)
	df['source_file']=source
	df.drop(["Fabs", "entrezgene_id", "hgnc_id", "uniprot_id","type", "ID"], axis=1, inplace=True)
	df.sort_values("rank", inplace=True)

	#This script merges on gene_name and not on every isoform to reduce complexity. The user must take this into consideration
	#and when proceeding when the analysis, better to take into account each isoform of the gene of interest.
	df.drop_duplicates(subset=['gene_name'], keep='first',  inplace=True)
	files_dfs.append(df)
	#print (df)

df_merged = reduce(lambda  left,right: pd.merge(left,right,on=['gene_name'],
													how='outer'), files_dfs)
print(df_merged)
df_merged.to_csv('significant.csv', index=False)

genes = df_merged["gene_name"].to_numpy()
#print (type(genes))

emt_genes = readFile(emtfile)
#print(emt_genes)

#emt_genes = np.array(emt)
print (type(emt_genes), len(emt_genes))

total = findCommons(genes, emt_genes)
print(total, len(total))


from api.auth import UserAuthHandler
from api.user import UserDeatilHandler, UserListHandler
from api.admin import LoginHandler, LogoutHandler

handlers = [
    (r'/admin/auth', UserAuthHandler),
    (r'/admin/login', LoginHandler),
    (r'/admin/logout', LogoutHandler),

    (r'/user/list', UserListHandler),
    (r'/user/detail', UserDeatilHandler),
]

# Need_Token_URLs = []


import cookielib
import logging
import re
import sys
import traceback
import urllib2
from Scraper import Scraper
import mechanize
import ImageDocument
import random
import GlobalConfiguration

log = logging.getLogger(__name__)

__author__ = 'Tryti'

gifre = re.compile('.*\\.gif.*')

WIKIPEDIA_EN_RANDOM = 'en.wikipedia.org/wiki/Special:Random'
MAXSCRAPES = 10
MAX_SCRAPES_PER_WALK = 15
USER_AGENT = 'Mozilla/21.0 (X11; U; Linux i686; en-US; rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/21'

class Crawler:
    def __init__(self, url_seeds=GlobalConfiguration.DEFAULT_URL_SEEDS):
        """
        Starts to crawl from the headurl and continues at random
        """
        self.visited = []
        self.url_seeds = url_seeds

    def get_random_url_seed(self):
        return self.url_seeds[random.randint(0, len(self.url_seeds)-1)]

    def get_browser(self):
        # shamelessly stolen from http://stockrt.github.io/p/emulating-a-browser-in-python-with-mechanize/

        # Browser
        br = mechanize.Browser()

        # Cookie Jar
        cj = cookielib.LWPCookieJar()
        br.set_cookiejar(cj)

        # Browser options
        br.set_handle_equiv(True)
        br.set_handle_redirect(True)
        br.set_handle_referer(True)
        br.set_handle_robots(True)

        # Follows refresh 0 but not hangs on refresh > 0
        br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)

        # Want debugging messages?
        # br.set_debug_http(True)
        # br.set_debug_redirects(True)
        # br.set_debug_responses(True)

        # User-Agent (this is cheating, ok?)
        br.addheaders = [('User-agent', USER_AGENT)]

        seed = self.get_random_url_seed()
        log.info('seeding browser with %s', seed)
        br.open(seed)
        self.br = br

        # dont scrape the seed page...
        self.next_link = self.get_next_link()

        return br

    def get_next_link(self):

        next_links = []
        #Get the links from the site and add them to the
        for i in self.br.links(url_regex='http'):
            for regex in GlobalConfiguration.page_block_regexs:
                if regex.search(i.url):
                    log.debug('link %s not feasible for scraping.', i.url)
                    continue
            if i.url != '#page':
                next_links.append(i.url)

        self.next_link = None

        #Unless we are at a sink we wants to continue
        if len(next_links) != 0:
            random.shuffle(next_links)
            for temp_link in next_links:
                if temp_link not in self.visited:
                    return temp_link
        else:
            log.info('couldn\'t find link to follow.')
            raise RuntimeError

    def crawl(self, maxnr=MAXSCRAPES):

        scraped = 0
        #Scrape until we scraped wanted amount or we fall into a sink
        while maxnr <= 0 or scraped < maxnr:
            try:

                walked = 0

                self.br = self.get_browser()

                while maxnr <= 0 or scraped < maxnr:

                    #Get page contents with Browser
                    success=False
                    while not success:
                        try:
                            self.next_link = self.get_next_link()
                            self.br.open(self.next_link)
                            walked += 1
                            if walked > MAX_SCRAPES_PER_WALK:
                                log.info('maximum link steps reached, restarting at a root.')
                                raise RuntimeError
                            success = True
                        except urllib2.HTTPError:
                            log.debug(traceback.format_exc())
                            sys.exc_clear()
                            walked += 1
                            if walked > MAX_SCRAPES_PER_WALK:
                                log.info('maximum link steps reached, restarting at a root.')
                                raise RuntimeError
                            self.br.back()

                    log.debug('scraping %s', self.next_link)

                    #And add it to the visited list
                    if self.next_link not in self.url_seeds:
                        self.visited.append(self.next_link)
                        if self.br.viewing_html():
                            #Get the imageDocuments on the site
                            s = Scraper(self.br.response().read(), self.next_link)
                            image_documents = s.get_image_documents()

                            for i in image_documents:
                                yield i
                            scraped += 1
                        else:
                            self.br.back()

            except KeyboardInterrupt:
                return
            except:
                log.debug(traceback.format_exc())
                sys.exc_clear()
                try:
                    self.br.close()
                except:
                    pass

        try:
            log.debug(traceback.format_exc())
            sys.exc_clear()
            self.br.close()

        except KeyboardInterrupt:
            return
        except:
            pass

    def random_link(self, links):
        return links.pop(random.randint(0, len(links)-1))

def main():
    test()

def test():
    url = 'http://www.xkcd.com'
    print('Starting test crawler on ' + url)

    c = Crawler(url)

    print('')
    print('Found these images')
    for i in c.crawl():
        print(i.url)

if __name__ == '__main__':
    main()