# -*- coding: utf-8 -*-
######################################################################
# 模型名称: 参数定义模块
# 模型编号: 无
# 程序文件名: hbase_data_load.py
# 创建时间: 2016年07月18日
# 作者: 张光洪
# 功能说明: 参数定义
# 更新记录: 暂无
#
######################################################################

# 定义SDR参数
p_sdr = {'A02': 'trainsite_valid',  # 车次有效
         'A03': 'speed_valid',  # 速度有效
         'A06': 'trainsite_id',  # 车次
         'A07': 'union_status',  # 联挂状态
         'A09': 'speed',  # 速度
         'A13': 'brake_pos',  # 牵引/制动档级
         'A14': 'const_speed',  # 恒速：挡位/制动信息
         'A16': 'mcr_coach',  # 主控车
         'C01': 'vcb_status',  # VCB状态        # 本单元主断状态 1=闭合状态；2=断开状态；3=切除状态；0=通信故障
         'C04': 'volt3_value',  # 3次电压
         'C99': 't16_k',  # K开通状态
         'B17': 'emergency_short',  # 紧急短路(空气制动切除M174)
         'B02': 'as_press',  # 空簧压力
         'B03': 'restop_graph_v',  # 再生制动力图形电压
         'B04': 'mr_press',  # 总风压
         'C07': 'restop_back',  # 再生制动力反馈
         'B00': 'stop_valid',  # 制动信息有效
         'B01': 'bc_press',  # BC压
         'C00': 'ci_valid',  # CI信息有效
         'C06': 'rotor_rate',  # 转子频率数
         'C05': 'dc_mm_back',  # 牵引电机电流
         'C08': 'spin_flag',  # 空转
         'B14': 'glide_flag',  # 滑行
         'B16': 'compressor_off',  # 压缩机切除
         'D00': 'battery_valid',  # 蓄电池电压有效
         'D01': 'battery_v',  # 蓄电池电压
         'C13': 'm_off',  # M车切除
         'C42': 'water_in_degree',  # 主变流器冷却水进口温度
         'C12': 'vcb_off',  # VCB切除
         'G00': 'aircond_valid',  # 空调信息有效
         'G02': 'aircond_work_status',  # 空调运转模式
         'G03': 'aircond_half_status',  # 空调负载减半
         'H25': 'out_sensor_degree',  # 外温传感器信息温度
         'G05': 'aircond_cool_degree',  # 空调控制温度制冷
         'A19': 'save_58_tatus',  # 救援58信息
         'B122': 'bpsave_device_status',  # BP救援装置
         'H28': 'bpsave_valid',  # BP救援有效信号
         'S23': 'save_conver_status',  # 救援转换装置电源空开
         'G07': 'coach_avg_degree',  # 室内平均温度
         'S19': 'mrc_egcs1',  # 主控端EGCS1（司机室保护接地开关）
         'S21': 'dirs_signal',  # DIRS电平信号
         'H71': 'tbr_signal',  # TBR电平信号
         'A45': 'control_order',  # 控制指令（CI）-10
         'B61': 'air_compressor_status',  # 空压机状态
         'B107': 'air_brake_status',  # 空气制动状态
         'A21': 'brake_lvl',  # 制动级位
         'B121': 'eb_cmd',  # 紧急制动EB命令/EB制动工况
         'A41': 'union_train_no',  # 联挂列号
         'B118': 'dc_ctv_off',  # 牵引逆变器切除（动车切除）
         'C67': 'dc_trans_degree',  # 牵引变压器温度
         'B119': 'axle_slide',  # 车轴滑行
         'K03': 'axle_slide_c1',  # 1轴累计滑行次数
         'K04': 'axle_slide_c2',  # 2轴累计滑行次数
         'K05': 'axle_slide_c3',  # 3轴累计滑行次数
         'K06': 'axle_slide_c4',  # 4轴累计滑行次数
         'C35': 'mid_dc_volt',  # 中间直流电压
         'C16': 'network_volt',  # 网压
         'B06': 'emergency_brake',  # 紧急制动
         'B132': 'emergency_brake_ebr',  # 紧急制动EBR
         'B131': 'emergency_brake_jtr',  # 紧急制动JTR
         'A47': 'ci_order',  # 空挡CI指令
         'C09': 'pantograph_status',  # 受电弓状态
         'A15': 'brake_info',  # 制动信息
         'B08': 'alert_brake',  # 警惕制动
         'C257': 'dc_low_voltage2',  # 直流低电压2
         'C258': 'cool_device_temp',  # 冷却设备过温度
         'C259': 'machine_temp_high',  # 机器室内过温度
         'C260': 'blower_stop',  # CI/MM鼓风机停止
         'C261': 'ovth_malfunction',  # OVTh误动作
         'C262': 'main_trans_abnormal',  # 主变压器异常
         'A82': 'traction_1_grade',  # 牵引1档  控制指令（CI）-11
         'A17': 'direction',  # 方向           00_01_10_11	零位_前进_后退_故障
         'C43': 'water_out_degree',  # 主变流器冷却水出口温度
         'C50': 'aux_trans_degree',  # 辅助变压器温度
         'B07': 'em_brake_switch',  # 紧急制动拉动开关
         'S48': 'park_brake_order',  # 停放制动指令
         'C11': 'pantograph_resection',  # 受电弓切除
         'B10': 'electric_brake_excision',  # 电制动切除
         'S33': 'usual_b1',  # 常用B1
         'S34': 'usual_b2',  # 常用B2
         'S35': 'usual_b3',  # 常用B3
         'S36': 'usual_b4',  # 常用B4
         'S37': 'usual_b5',  # 常用B5
         'S38': 'usual_b6',  # 常用B6
         'S39': 'usual_b7',  # 常用B7
         'B05': 'ep_current_value',  # EP阀电流
         'Y01': 'vigilance_test',  # 警惕试验
         'C345': 'traction_3gear',  # 牵引3档位
         'C338': 'traction_5gear',  # 牵引5档位
         'C333': 'traction_7gear',  # 牵引7档位
         'C327': 'traction_9gear',  # 牵引9档位
         'A91': 'traction_evengear',  # 牵引偶数档
         'H05': 'door_information1',  # 车门信息1
         'H06': 'door_information2',  # 车门信息2
         'H07': 'door_information3',  # 车门信息3
         'H08': 'door_information4',  # 车门信息4
         'K02': 'spin_num'  # 累计空转次数
         }

# 定义GLIDE和空转参数
p_glide = {
    'K01': 'glide_flag',  # 滑行标志
    'K03': 'glide1_num',  # 1轴累计滑行次数
    'K04': 'glide2_num',  # 2轴累计滑行次数
    'K05': 'glide3_num',  # 3轴累计滑行次数
    'K06': 'glide4_num',  # 4轴累计滑行次数
    'K00': 'spin_flag',  # 空转标志
    'K02': 'spin_num',  # 累计空转次数
}

# 故障数据参数
p_fault = {
    '001': 'fault_code_001',  # 牵引变流器-控制电源“关”
    '002': 'fault_code_002',  # 牵引变流器-传输不良
    '004': 'fault_code_004',  # 牵引变流器故障1
    '005': 'fault_code_005',  # 牵引变流器故障2
    '084': 'fault_code_084',  # 084抱死预告检测
    '113': 'fault_code_113',  # 配电柜NFB关
    '134': 'fault_code_134',  # 牵引变流器-通风机停止
    '135': 'fault_code_135',  # 辅助电源装置故障
    '139': 'fault_code_139',  # 牵引变流器-微机故障
    '141': 'fault_code_141',  # 牵引变流器-故障
    '143': 'fault_code_143',  # 辅助电源装置通风机停止
    '144': 'fault_code_144',  # 辅助电源装置ARfN2跳闸
    '146': 'fault_code_146',  # 辅助电源装置 ACVN1跳闸
    '147': 'fault_code_147',  # 辅助电源装置 ACVN2跳闸
    '148': 'fault_code_148',  # 辅助电源装置 ATN 跳闸
    '166': 'fault_code_166',  # 辅助电源装置VDTN跳闸
    '201': 'fault_code_201',  # 辅助电源装置控制电源“关”
    '204': 'fault_code_204',  # 辅助电源装置传输不良
    '132': 'fault_code_132',  # 牵引变压器-油流停止
    '008': 'fault_code_008',  # 门电源电压低下（GPLV）
    '014': 'fault_code_014',  # 电动机过电流 2(MMOC2)
    '015': 'fault_code_015',  # 电动机过电流 1(MMOC1)
    '018': 'fault_code_018',  # 电动机电流不平衡(PUD)
    '021': 'fault_code_021',  # PG 故障(PGD)
    '022': 'fault_code_022',  # 控制电源电压低下(CPLV)
    '024': 'fault_code_024',  # 0VTh 误动作(0VTGD)
    '030': 'fault_code_030',  # 牵引不动作(MFD)
    '050': 'fault_code_050',  # 2次侧接点 2(GD2)
    '054': 'fault_code_054',  # 直流 100V 异常(P100LV)
    '055': 'fault_code_055',  # 2次侧接点 1(GD1)
    '139': 'fault_code_139',  # 微机异常WDT
    '012': 'fault_code_012',  # 2次过电流 1(ISOC1)
    '006': 'fault_code_006',  # 主电路元件异常（IGTFD）
    '019': 'fault_code_019',  # 充电不良（CHF）
}

# 参数类：文件字段定义(从HBase导出数据时按此顺序映射)
cols_sdr = {
    'train_type_code': 0,  # 车型
    'train_no': 1,  # 列号
    'coach_id': 2,  # 车号
    'time_value': 3,  # 时间
    'trainsite_valid': 4,  # 车次有效
    'trainsite_id': 5,  # 车次
    'union_status': 6,  # 联挂有效
    'speed_valid': 7,  # 速度有效
    'speed': 8,  # 速度
    'brake_pos': 9,  # 牵引制动档位
    'mcr_coach': 10,  # 主控车
    'vcb_status': 11,  # VCB状态
    'volt3_value': 12,  # 3次电压
    't16_k': 13,  # K开通状态
    'emergency_short': 14,  # 紧急短路(空气制动切除M174)
    'restop_graph_v': 15,  # 再生制动力图形电压
    'mr_press': 16,  # 总风压
    'restop_back': 17,  # 再生制动力反馈
    'stop_valid': 18,  # 制动信息有效
    'bc_press': 19,  # BC压
    'ci_valid': 20,  # CI信息有效
    'rotor_rate': 21,  # 转子频率数
    'dc_mm_back': 22,  # 牵引电机电流
    'compressor_off': 23,  # 压缩机切除
    'battery_valid': 24,  # 蓄电池电压有效
    'battery_v': 25,  # 蓄电池电压
    'm_off': 26,  # M车切除
    'as_press': 27,  # 空簧压力
    'const_speed': 28,  # 恒速：挡位/制动信息
    'glide_flag': 29,  # 滑行
    'spin_flag': 30,  # 空转
    'water_in_degree': 31,  # 主变流器冷却水进口温度
    'vcb_off': 32,  # VCB切除
    'save_58_tatus': 33,  # 救援58信息
    'aircond_valid': 34,  # 空调信息有效
    'aircond_work_status': 35,  # 空调运转模式
    'aircond_half_status': 36,  # 空调负载减半
    'out_sensor_degree': 37,  # 外温传感器信息温度
    'aircond_cool_degree': 38,  # 空调控制温度制冷
    'bpsave_device_status': 39,  # BP救援装置
    'bpsave_valid': 40,  # BP救援有效信号
    'save_conver_status': 41,  # 救援转换装置电源空开
    'coach_avg_degree': 42,  # 室内平均温度
    'mrc_egcs1': 43,  # 主控端EGCS1（司机室保护接地开关）
    'dirs_signal': 44,  # DIRS电平信号
    'tbr_signal': 45,  # TBR电平信号
    'control_order': 46,  # 控制指令（CI）-10
    'air_compressor_status': 47,  # 空压机工作状态 0_1_2_3_4_5	信故障_故障_切除_工作_油预热_未工作
    'air_brake_status': 48,  # 空气制动状态
    'brake_lvl': 49,  # 制动级位
    'eb_cmd': 50,  # 紧急制动EB命令/EB制动工况
    'union_train_no': 51,  # 联挂列号
    'dc_ctv_off': 52,  # 牵引逆变器切除（动车切除）
    'dc_trans_degree': 53,  # 牵引变压器温度
    'axle_slide': 54,  # 车轴滑行
    'axle_slide_c1': 55,  # 1轴累计滑行次数
    'axle_slide_c2': 56,  # 2轴累计滑行次数
    'axle_slide_c3': 57,  # 3轴累计滑行次数
    'axle_slide_c4': 58,  # 4轴累计滑行次数
    'mid_dc_volt': 59,  # 中间直流电压
    'network_volt': 60,  # 网压
    'emergency_brake': 61,  # 紧急制动       0_1	正常_施加紧急制动
    'emergency_brake_ebr': 62,  # 紧急制动EBR   0_1	无效_有效
    'emergency_brake_jtr': 63,  # 紧急制动JTR   0_1	无效_有效
    'ci_order': 64,  # 空挡CI指令
    'pantograph_status': 65,  # 受电弓状态
    'brake_info': 66,  # 制动信息
    'alert_brake': 67,  # 警惕制动
    'dc_low_voltage2': 68,  # 直流低电压2
    'cool_device_temp': 69,  # 冷却设备过温度
    'machine_temp_high': 70,  # 机器室内过温度
    'blower_stop': 71,  # CI/MM鼓风机停止
    'ovth_malfunction': 72,  # OVTh误动作
    'main_trans_abnormal': 73,  # 主变压器异常
    'traction_1_grade': 74,  # 牵引1档  控制指令（CI）-11
    'direction': 75,  # 方向            00_01_10_11	无方向_前进_后退_故障
    'water_out_degree': 76,  # 主变流器冷却水出口温度
    'aux_trans_degree': 77,  # 辅助变压器温度
    'em_brake_switch': 78,  # 紧急制动拉动开关
    'park_brake_order': 79,  # 停放制动指令
    'pantograph_resection': 80,  # 受电弓切除
    'electric_brake_excision': 81,  # 电制动切除
    'usual_b1': 82,  # 常用B1
    'usual_b2': 83,  # 常用B2
    'usual_b3': 84,  # 常用B3
    'usual_b4': 85,  # 常用B4
    'usual_b5': 86,  # 常用B5
    'usual_b6': 87,  # 常用B6
    'usual_b7': 88,  # 常用B7
    'ep_current_value': 89,  # EP阀电流
    'vigilance_test': 90,  # 警惕试验
    'traction_3gear': 91,  # 牵引3档位
    'traction_5gear': 92,  # 牵引5档位
    'traction_7gear': 93,  # 牵引7档位
    'traction_9gear': 94,  # 牵引9档位
    'traction_evengear': 95,  # 牵引偶数档
    'door_information1': 96,  # 车门信息5
    'door_information2': 97,  # 车门信息6
    'door_information3': 98,  # 车门信息7
    'door_information4': 99,  # 车门信息8
    'spin_num':100            # 累计空转次数

}

# 故障类：文件字段定义(从HBase导出数据时按此顺序映射)
cols_fault = {
    'train_type_code': 0,  # 车型
    'train_no': 1,  # 列号
    'coach_id': 2,  # 车号
    'fault_code_001': 3,  # 牵引变流器-控制电源“关”
    'fault_code_002': 4,  # 牵引变流器-传输不良
    'fault_code_004': 5,  # 牵引变流器故障1
    'fault_code_005': 6,  # 牵引变流器故障2
    'fault_code_084': 7,  # 084抱死预告检测
    'fault_code_113': 8,  # 配电柜NFB关
    'fault_code_134': 9,  # 牵引变流器-通风机停止
    'fault_code_135': 10,  # 辅助电源装置故障
    'fault_code_139': 11,  # 牵引变流器-微机故障
    'fault_code_141': 12,  # 牵引变流器-故障
    'fault_code_143': 13,  # 辅助电源装置通风机停止
    'fault_code_144': 14,  # 辅助电源装置ARfN2跳闸
    'fault_code_146': 15,  # 辅助电源装置 ACVN1跳闸
    'fault_code_147': 16,  # 辅助电源装置 ACVN2跳闸
    'fault_code_148': 17,  # 辅助电源装置 ATN 跳闸
    'fault_code_166': 18,  # 辅助电源装置VDTN跳闸
    'fault_code_201': 19,  # 辅助电源装置控制电源“关”
    'fault_code_204': 20,  # 辅助电源装置传输不良
    'fault_code_132': 21,  # 牵引变压器-油流停止
    'fault_code_008': 22,  # 门电源电压低下（GPLV）
    'fault_code_014': 23,  # 电动机过电流 2(MMOC2)
    'fault_code_015': 24,  # 电动机过电流 1(MMOC1)
    'fault_code_018': 25,  # 电动机电流不平衡(PUD)
    'fault_code_021': 26,  # PG 故障(PGD)
    'fault_code_022': 27,  # 控制电源电压低下(CPLV)
    'fault_code_024': 28,  # 0VTh 误动作(0VTGD)
    'fault_code_030': 29,  # 牵引不动作(MFD)
    'fault_code_050': 30,  # 2次侧接点 2(GD2)
    'fault_code_054': 31,  # 直流 100V 异常(P100LV)
    'fault_code_055': 32,  # 2次侧接点 1(GD1)
    'fault_code_139': 33,  # 微机异常WDT
    'fault_code_012': 34,  # 2次过电流 1(ISOC1)
    'fault_code_006': 35,  # 主电路元件异常（IGTFD）
    'fault_code_019': 36,  # 充电不良（CHF）
    'time_value': 37,  # 时间

}

# 轴温传感器温度数据
p_sensor = {
    # 一路轴温
    'Z03': '1',  # 1位轴箱
    'Z04': '2',  # 2位轴箱
    'Z05': '3',  # 3位轴箱
    'Z06': '4',  # 4位轴箱
    'Z07': '5',  # 5位轴箱
    'Z08': '6',  # 6位轴箱
    'Z09': '7',  # 7位轴箱
    'Z10': '8',  # 8位轴箱
    'Z11': '9',  # 1位小齿轮电机侧
    'Z12': '10',  # 2位小齿轮电机侧
    'Z13': '11',  # 3位小齿轮电机侧
    'Z14': '12',  # 4位小齿轮电机侧
    'Z15': '13',  # 1位小齿轮车轮侧
    'Z16': '14',  # 2位小齿轮车轮侧
    'Z17': '15',  # 3位小齿轮车轮侧
    'Z18': '16',  # 4位小齿轮车轮侧
    'Z19': '17',  # 1位电机定子
    'Z20': '18',  # 1位电机传动端
    'Z21': '19',  # 1位电机非传动端
    'Z22': '20',  # 2位电机定子
    'Z23': '21',  # 2位电机传动端
    'Z24': '22',  # 2位电机非传动端
    'Z25': '23',  # 3位电机定子
    'Z26': '24',  # 3位电机传动端
    'Z27': '25',  # 3位电机非传动端
    'Z28': '26',  # 4位电机定子
    'Z29': '27',  # 4位电机传动端
    'Z30': '28',  # 4位电机非传动端
    'Z31': '29',  # 1位大齿轮电机侧
    'Z32': '30',  # 2位大齿轮电机侧
    'Z33': '31',  # 3位大齿轮电机侧
    'Z34': '32',  # 4位大齿轮电机侧
    'Z35': '33',  # 1位大齿轮车轮侧
    'Z36': '34',  # 2位大齿轮车轮侧
    'Z37': '35',  # 3位大齿轮车轮侧
    'Z38': '36',  # 4位大齿轮车轮侧
    # 2路轴温
    'Z39': '37',  # 1位轴箱
    'Z40': '38',  # 2位轴箱
    'Z41': '39',  # 3位轴箱
    'Z42': '40',  # 4位轴箱
    'Z43': '41',  # 5位轴箱
    'Z44': '42',  # 6位轴箱
    'Z45': '43',  # 7位轴箱
    'Z46': '44',  # 8位轴箱
    'Z47': '45',  # 1位小齿轮电机侧
    'Z48': '46',  # 2位小齿轮电机侧
    'Z49': '47',  # 3位小齿轮电机侧
    'Z50': '48',  # 4位小齿轮电机侧
    'Z51': '49',  # 1位小齿轮车轮侧
    'Z52': '50',  # 2位小齿轮车轮侧
    'Z53': '51',  # 3位小齿轮车轮侧
    'Z54': '52',  # 4位小齿轮车轮侧
    'Z55': '53',  # 1位电机定子
    'Z56': '54',  # 1位电机传动端
    'Z57': '55',  # 1位电机非传动端
    'Z58': '56',  # 2位电机定子
    'Z59': '57',  # 2位电机传动端
    'Z60': '58',  # 2位电机非传动端
    'Z61': '59',  # 3位电机定子
    'Z62': '60',  # 3位电机传动端
    'Z63': '61',  # 3位电机非传动端
    'Z64': '62',  # 4位电机定子
    'Z65': '63',  # 4位电机传动端
    'Z66': '64',  # 4位电机非传动端
    'Z67': '65',  # 1位大齿轮电机侧
    'Z68': '66',  # 2位大齿轮电机侧
    'Z69': '67',  # 3位大齿轮电机侧
    'Z70': '68',  # 4位大齿轮电机侧
    'Z71': '69',  # 1位大齿轮车轮侧
    'Z72': '70',  # 2位大齿轮车轮侧
    'Z73': '71',  # 3位大齿轮车轮侧
    'Z74': '72',  # 4位大齿轮车轮侧
    'A09': 'speed',
    'H25': 'out_degree',
    'A06': 'trainsite_id',
    'A13': 'brake_pos',
    'A02': 'trainsite_valid'
}

p_air = {'A02': 'trainsite_valid',  # 车次有效
         'A03': 'speed_valid',  # 速度有效
         'A06': 'trainsite_id',  # 车次
         'A09': 'speed',  # 速度
         'A13': 'brake_pos',  # 牵引/制动档级
         'G11': 'air_tartemp',  # 目标温度值
         'G12': 'air_dettemp',  # 客室内温度检测值
         'G15': 'air_run_modal',  # 空调运行模式
         'G01': 'air_con_modal',  # 空调控制模式
         'G02': 'air_rev_modal',  # 空调运转模式
         'A10': 'air_1',  # 运营里程
         'G03': 'air_2',  # 减载
         'G18': 'air_3',  # 制冷系统1高压压力值
         'G19': 'air_4',  # 制冷系统2高压压力值
         'G20': 'air_5',  # 制冷系统1低压压力值
         'G21': 'air_6',  # 制冷系统2低压压力值
         'G22': 'air_7',  # 紧急通风逆变器运转
         'G23': 'air_8',  # 1位端通过台电加热运转
         'G24': 'air_9',  # 2位端通过台电加热运转
         'G25': 'air_10',  # 风道电加热运转
         'G26': 'air_11',  # 电加热1故障
         'G27': 'air_12',  # 电加热1运转
         'G28': 'air_13',  # 电加热2故障
         'G29': 'air_14',  # 电加热2运转
         'G30': 'air_15',  # 通风机1故障
         'G31': 'air_16',  # 通风机1运转
         'G32': 'air_17',  # 通风机2故障
         'G33': 'air_18',  # 通风机2运转
         'G34': 'air_19',  # 冷凝风机1故障
         'G35': 'air_20',  # 冷凝风机1运转
         'G36': 'air_21',  # 冷凝风机2故障
         'G37': 'air_22',  # 冷凝风机2运转
         'G38': 'air_23',  # 新风A1阀关
         'G39': 'air_24',  # 新风A2阀关
         'G40': 'air_25',  # 新风B1阀关
         'G41': 'air_26',  # 新风B2阀关
         'G42': 'air_27',  # 废排风门关
         'G43': 'air_28',  # 废排压力波阀关
         'G44': 'air_29',  # 废排风机故障
         'G45': 'air_30',  # 废排风机运转
         'G46': 'air_31',  # 司机室压缩机运转
         'G47': 'air_32',  # 司机室冷凝风机运转
         'G48': 'air_33',  # 司机室通风机运转
         'B04': 'air_34',  # 总风压力
         'D04': 'air_35',  # 锂电池组1电量
         'D05': 'air_36',  # 锂电池组2电量
         'D06': 'air_37',  # 充电机1中间电压
         'D07': 'air_38',  # 充电机2中间电压
         'D08': 'air_39',  # 充电机1输出电压
         'D09': 'air_40',  # 充电机2输出电压
         'D14': 'air_41',  # 充电机1输出电流
         'D15': 'air_42',  # 充电机2输出电流
         'D16': 'air_43',  # 充电机1充电电流
         'D17': 'air_44',  # 充电机2充电电流
         'G17': 'new_wind_dettemp',  # 新风温度检测值
         'G16': 'tour_dettemp',  # 观光区温度检测值
         'G65': 'compressor1_run',  # 压缩机1运转
         'G64': 'compressor1_fault',  # 压缩机1故障
         'G63': 'compressor2_run',  # 压缩机2运转
         'G62': 'compressor2_fault',  # 压缩机2故障
         'B21': 'as_press1',  # 空簧压力1
         'B22': 'as_press2',  # 空簧压力2
         }

# 车次黑名单(需要排除的以下调试车次) 'G61'次交路已经开通，于20180918去掉
trainsiteBlackList = ['D60', 'D61', 'D62', 'D63', 'D64']

# HBase连接服务端主机名
hbase_thriftserver = '10.73.95.1'

# 定义SDR扩展参数
p_sdrext = {'A02': 'trainsite_valid',  # 车次有效
            'A03': 'speed_valid',  # 速度有效
            'A06': 'trainsite_id',  # 车次
            'A07': 'union_status',  # 联挂状态
            'A09': 'speed',  # 速度
            'A13': 'brake_pos',  # 牵引/制动档级
            'A14': 'const_speed',  # 恒速：挡位/制动信息
            'A16': 'mcr_coach',  # 主控车
            'C01': 'vcb_status',  # VCB状态
            'C04': 'volt3_value',  # 3次电压
            'C99': 't16_k',  # K开通状态
            'B17': 'emergency_short',  # 紧急短路(空气制动切除M174)
            'B02': 'as_press',  # 空簧压力
            'B03': 'restop_graph_v',  # 再生制动力图形电压
            'B04': 'mr_press',  # 总风压
            'C07': 'restop_back',  # 再生制动力反馈
            'B00': 'stop_valid',  # 制动信息有效
            'B01': 'bc_press',  # BC压
            'C00': 'ci_valid',  # CI信息有效
            'C06': 'rotor_rate',  # 转子频率数
            'C05': 'dc_mm_back',  # 牵引电机电流
            'C08': 'spin_flag',  # 空转
            # 'B119':'glide_flag',              # 滑行                # 同axle_slide 重复，去掉该字段并更改使用该字段的脚本 by lxj 20180622
            'B16': 'compressor_off',  # 压缩机切除
            'D00': 'battery_valid',  # 蓄电池电压有效
            'D01': 'battery_v',  # 蓄电池电压
            'C13': 'm_off',  # M车切除
            'C42': 'water_in_degree',  # 主变流器冷却水进口温度
            'C12': 'vcb_off',  # VCB切除
            'A17': 'direction',  # 方向
            'A41': 'union_train_no',  # 联挂列号
            'C14': 'vcb_down',  # VCB闭合
            'B116': 'main_compressor_off',  # 主空压机切除指令
            'C39': 'assist_volt',  # 辅变母线电压
            'A21': 'brake_lvl',  # 制动级位
            'B102': 'e_brake',  # 电制动能力/本车电制动可用
            'C376': 'dc_cvt_off',  # 牵引变流器切除
            'C377': 'dc_ctv_off',  # 牵引逆变器切除（动车切除）
            'C21': 'tcp_keep_model',  # TCU中压保持模式
            'B119': 'axle_slide',  # 车轴滑行
            'C58': 'tcu_restop_brake',  # TCU反馈再生制动力
            'C57': 'bcu_request_brake',  # BCU请求再生制动力
            'C24': 'ee1_pull_value',  # 1架电制力实际值/一架电机牵引
            'C25': 'ee2_pull_value',  # 2架电制力实际值/二架电机牵引
            'B107': 'air_brake_status',  # 空气制动状态
            'K03': 'axle_slide_c1',  # 1轴累计滑行次数
            'K04': 'axle_slide_c2',  # 2轴累计滑行次数
            'K05': 'axle_slide_c3',  # 3轴累计滑行次数
            'K06': 'axle_slide_c4',  # 4轴累计滑行次数
            'A39': 'train_weight',  # 列车实际质量
            'B120': 'coach_weight',  # 本车实际质量
            'B121': 'eb_cmd',  # 紧急制动EB命令/EB制动工况
            'C54': 'shelf1_speed',  # 一架电机综合速度
            'C55': 'shelf2_speed',  # 二架电机综合速度
            'C32': 'dc_trans_elec',  # 牵引变压器原边电流
            'C33': 'shelf1_quad4_elec',  # 一架四象限输入电流
            'C34': 'shelf2_quad4_elec',  # 一架四象限输入电流
            'C36': 'shelf1_engi_elec',  # 一架电机电流
            'C37': 'shelf2_engi_elec',  # 二架电机电流
            'C52': 'shelf1_engi_power',  # 一架电机实际力
            'C53': 'shelf2_engi_power',  # 二架电机实际力
            'C69': 'shelf_off',  # 转向架切除（牵引逆变器状态）
            'C61': 'power_percent',  # 牵引/电制动力百分比（实际牵引/电制动力百分比）
            'C77': 'urgent_pull_relay',  # 紧急牵引指令继电器
            'K02': 'pull_idle_count',  # 牵引空转次数
            'C16': 'network_volt',  # 网压
            'C35': 'mid_dc_volt',  # 中间直流电压
            'C67': 'dc_trans_degree',  # 牵引变压器温度
            'C42': 'dc_cvt_water_in1_degree',  # 牵引变流器冷却水进口1温度
            'C43': 'dc_cvt_water_out1_degree',  # 牵引变流器冷却水出口1温度
            'C44': 'dc_cvt_water_in2_degree',  # 牵引变流器冷却水进口2温度
            'C45': 'dc_cvt_water_out2_degree',  # 牵引变流器冷却水出口2温度
            'G18': 'refrigerator_1_high_press',  # 制冷系统1高压压力值
            'G19': 'refrigerator_2_high_press',  # 制冷系统2高压压力值
            'G20': 'refrigerator_1_low_press',  # 制冷系统1低压压力值
            'G21': 'refrigerator_2_low_press',  # 制冷系统2低压压力值
            'G17': 'new_wind_dettemp',  # E32B新风温度检测值
            'A28': 'new_wind_dettemp',  # E32 室外温度
            'G16': 'tour_dettemp',  # 观光区温度检测值
            'H67': 'car_door_1_closed',  # 车门1关闭
            'H68': 'car_door_2_closed',  # 车门2关闭
            'H69': 'car_door_3_closed',  # 车门3关闭
            'H70': 'car_door_4_closed',  # 车门4关闭
            'G12': 'air_dettemp',  # 客室内温度检测值
            'G11': 'air_tartemp',  # 目标温度值
            'D08': 'charger_out_v1',  # 充电机1输出电压
            'D09': 'charger_out_v2',  # 充电机2输出电压
            'D04': 'battery_group_qty1',  # 锂电池组1电量
            'D05': 'battery_group_qty2',  # 锂电池组2电量
            'D16': 'charger_up_v1',  # 充电机1充电电流
            'D17': 'charger_up_v2',  # 充电机2充电电流
            'D21': 'charger_off1',  # 充电机1切除
            'D22': 'charger_off2',  # 充电机2切除
            'B123': 'park_brake_order',  # 停放制动施加指令
            'B46': 'park_brake_status',  # 停放制动施加
            'B105': 'park_brake_press',  # 停放缸压力
            'B61': 'air_compressor_status',  # 空压机状态
            'C09': 'pantograph_status',  # 受电弓状态
            'C85': 'dc_trans_air_open',  # 牵引变压器油泵空开
            'C87': 'dc_trans_oil_stop_relay',  # 牵引变压器油流停止继电器
            'P19': 'high_off_switch_status',  # 高压隔离开关状态
            'C125': 'vcb_off_monitor',  # vcb闭合监视
            'C126': 'pantograph_up',  # 受电弓升起
            'P22': 'high_off_switch_down',  # 高压隔离开关闭合
            'C38': 'aux_out_voltage',  # 辅变输出电压
            'C40': 'aux_out_current',  # 辅变输出电流
            'C46': 'dc_cvt_water_in1_press',  # 牵引变流器冷却水进口1压力
            'C48': 'dc_cvt_water_in2_press',  # 牵引变流器冷却水进口2压力
            'G02': 'air_rev_modal',  # 空调运转模式
            'B106': 'stop_brake_status',  # 停放制动状态
            'C154': 'phase3_bus_relay1',  # 三相母线有电检测继电器1
            'C155': 'tmotor_fan_contactor1',  # 牵引电机风机1接触器1
            'C156': 'tmotor_fan_contactor2',  # 牵引电机风机1接触器2
            'Q32': 'ATP_power_switch',  # ATP电源控制开关            0_1 无效_有效
            'B157': 'ATP_7_brake',  # ATP七级制动
            'A62': 'direction_zero',  # 方向零位
            'A63': 'forward1',  # 前向1
            'A64': 'forward2',  # 前向2
            'Q16': 'rescue_switch_mid_pos',  # 救援开关中间位
            'A74': 'rescue_devic_contrl',  # 救援装置控制                0_1 无效_有效
            'Q27': 'rescue_switch_rescued',  # 救援开关被救援位          0_1 无效_有效
            'Q30': 'rescue_switch_rescue',  # 救援开关救援位          0_1 无效_有效
            'C70': 'aux_con_status',  # 辅助变流器状态        0_1_2_3_4_5_6无效_工作_非工作_故障_切除_隔离_通信故障"
            'C50': 'aux_trans_degree',  # 辅助变压器温度
            'C51': 'tconverter_air_degree',  # 牵引变流器柜体空气温度
            'B34': 'bp_press',  # BP压力 --列车管压力
            'A71': 'back1',  # 后向1
            'A72': 'back2',  # 后向2
            'C158': 'short1_contactor',  # 一架短接接触器
            'C159': 'short2_contactor',  # 二架短接接触器
            'C160': 'ac380_bus_contactor',  # 交流380V母线接触器 0_1 断开 闭合
            'C68': 'tract_conver_status',  # 牵引变流器状态 0_1_2_3_4_5_6  无效_工作_非工作_故障_切除_隔离_通信故障
            'C161': 'phase3_bus_relay2',  # 三相母线有电检测继电器2
            'A74': 'bpsave_devicontrl_air',  # BP救援装置控制空开
            'D23': 'charger1_115contactor',  # 充电机1的115线接触器
            'D25': 'charger2_115contactor',  # 充电机2的115线接触器
            'K08': 'tow_slide_count',  # 牵引滑行次数
            'C62': 'set_power_percent',  # 设定牵引/电制动力百分比
            'Y75': 'passing_neutral_section',  # 过分相
            'Q12': 'emergency_mode_switch1',  # 紧急模式开关1   0_1  非紧急牵引模式_紧急牵引模式
            'Q13': 'emergency_mode_switch2',  # 紧急模式开关2    0_1  非紧急牵引模式_紧急牵引模式
            'C256': 'Auxiliary_inverter_cut',  # 辅助逆变器切除状态   0_1 未切除_切除
            'B175': 'air_brake_isolation',  # 空气制动隔离
            'D24': 'charger1_103_contactor',  # 充电机1的103线接触器
            'D26': 'charger2_103_contactor',  # 充电机2的103线接触器
            'G65': 'compressor1_run',  # 压缩机1运转
            'G64': 'compressor1_fault',  # 压缩机1故障
            'G63': 'compressor2_run',  # 压缩机2运转
            'G62': 'compressor2_fault',  # 压缩机2故障
            'C75': 'Charger_1_status',  # 充电机1状态
            'C76': 'Charger_2_status',  # 充电机2状态
            'B314': 'UB_air_brake_effective',  # UB空气制动有效  0_1 对应 无效_有效
            'Q25': 'brake_isolation_switch',  # 保持制动隔离开关    0_1:无效_有效
            'B315': 'keep_brake_app_status',  # 保持制动施加状态    0_1:未施加_施加
            'Q38': 'relieve_keep_brake_switch',  # 缓解保持制动开关    0_1:无效_有效
            'G60': 'change_signal',  # 换端信号
            'D106': 'charge_battery_group1_tem1',  # 碱性蓄电池组1温度1|充电机1温度1
            'D105': 'charge_battery_group1_tem2',  # 碱性蓄电池组1温度2|充电机1温度2
            'D119': 'charge_battery_group2_tem1',  # 碱性蓄电池组2温度1|充电机2温度1
            'D118': 'charge_battery_group2_tem2',  # 碱性蓄电池组2温度2|充电机2温度2
            'Q03': 'director_control_traction_pos',  # 司控器牵引位
            'Y121': 'traction_energy1',  # 牵引使能1
            'B365': 'velocity_signal10',  # 10km/h速度信号
            'B366': 'velocity_signal05',  # 5km/h速度信号
            'B367': 'velocity_signal_less_than30',  # 小于30km/h速度信号
            'B368': 'velocity_signal_greater_than05',  # 大于5公里速度信号
            'B369': 'velocity_signal_less_than10',  # 小于10km/h速度信号
            'B363': 'Insufficient_wind_pressure',  # 总风压力不足
            'B134': 'wsp_speed1',  # WSP1轴速度
            'B135': 'wsp_speed2',  # WSP2轴速度
            'B136': 'wsp_speed3',  # WSP3轴速度
            'B137': 'wsp_speed4',  # WSP4轴速度
            'C56': 'cps_speed',  # 综合速度
            'B246':'auxiliary_aircompressor_status', #辅助空压机状态
            'B141':'auxiliary_aircompressor_contactor' # 辅助空压机接触器
            }

# 参数类：文件字段定义(从HBase导出数据时按此顺序映射)
cols_sdrext = {
    'train_type_code': 0,  # 车型
    'train_no': 1,  # 列号
    'coach_id': 2,  # 车号
    'time_value': 3,  # 时间
    'trainsite_valid': 4,  # 车次有效
    'trainsite_id': 5,  # 车次
    'union_status': 6,  # 联挂有效
    'speed_valid': 7,  # 速度有效
    'speed': 8,  # 速度
    'brake_pos': 9,  # 牵引制动档位
    'mcr_coach': 10,  # 主控车
    'vcb_status': 11,  # VCB状态
    'volt3_value': 12,  # 3次电压
    't16_k': 13,  # K开通状态
    'emergency_short': 14,  # 紧急短路(空气制动切除M174)
    'restop_graph_v': 15,  # 再生制动力图形电压
    'mr_press': 16,  # 总风压
    'restop_back': 17,  # 再生制动力反馈
    'stop_valid': 18,  # 制动信息有效
    'bc_press': 19,  # BC压
    'ci_valid': 20,  # CI信息有效
    'rotor_rate': 21,  # 转子频率数
    'dc_mm_back': 22,  # 牵引电机电流
    'compressor_off': 23,  # 压缩机切除
    'battery_valid': 24,  # 蓄电池电压有效
    'battery_v': 25,  # 蓄电池电压
    'm_off': 26,  # M车切除
    'as_press': 27,  # 空簧压力
    'const_speed': 28,  # 恒速：挡位/制动信息
    'glide_flag': 29,  # 滑行
    'spin_flag': 30,  # 空转
    'water_in_degree': 31,  # 主变流器冷却水进口温度
    'vcb_off': 32,  # VCB切除
    'direction': 33,  # 方向
    'union_train_no': 34,  # 联挂列号
    'vcb_down': 35,  # VCB闭合
    'main_compressor_off': 36,  # 主空压机切除指令
    'assist_volt': 37,  # 辅变母线电压
    'brake_lvl': 38,  # 制动级位
    'e_brake': 39,  # 电制动能力/本车电制动可用
    'dc_cvt_off': 40,  # 牵引变流器切除
    'dc_ctv_off': 41,  # 牵引逆变器切除
    'tcp_keep_model': 42,  # TCU中压保持模式
    'axle_slide': 43,  # 车轴滑行
    'tcu_restop_brake': 44,  # TCU反馈再生制动力
    'bcu_request_brake': 45,  # BCU请求再生制动力
    'ee1_pull_value': 46,  # 1架电制力实际值/一架电机牵引
    'ee2_pull_value': 47,  # 2架电制力实际值/二架电机牵引
    'air_brake_status': 48,  # 空气制动状态
    'axle_slide_c1': 49,  # 1轴累计滑行次数
    'axle_slide_c2': 50,  # 2轴累计滑行次数
    'axle_slide_c3': 51,  # 3轴累计滑行次数
    'axle_slide_c4': 52,  # 4轴累计滑行次数
    'train_weight': 53,  # 列车实际质量
    'coach_weight': 54,  # 本车实际质量
    'eb_cmd': 55,  # 紧急制动EB命令/EB制动工况
    'shelf1_speed': 56,  # 一架电机综合速度
    'shelf2_speed': 57,  # 二架电机综合速度
    'dc_trans_elec': 58,  # 牵引变压器原边电流
    'shelf1_quad4_elec': 59,  # 一架四象限输入电流
    'shelf2_quad4_elec': 60,  # 二架四象限输入电流
    'shelf1_engi_elec': 61,  # 一架电机电流
    'shelf2_engi_elec': 62,  # 二架电机电流
    'shelf1_engi_power': 63,  # 一架电机实际力
    'shelf2_engi_power': 64,  # 二架电机实际力
    'shelf_off': 65,  # 转向架切除（牵引逆变器状态）
    'power_percent': 66,  # 牵引/电制动力百分比（实际牵引/电制动力百分比）
    'urgent_pull_relay': 67,  # 紧急牵引指令继电器
    'pull_idle_count': 68,  # 牵引空转次数
    'network_volt': 69,  # 网压
    'mid_dc_volt': 70,  # 中间直流电压
    'dc_trans_degree': 71,  # 牵引变压器温度
    'dc_cvt_water_in1_degree': 72,  # 牵引变流器冷却水进口1温度
    'dc_cvt_water_out1_degree': 73,  # 牵引变流器冷却水出口1温度
    'dc_cvt_water_in2_degree': 74,  # 牵引变流器冷却水进口2温度
    'dc_cvt_water_out2_degree': 75,  # 牵引变流器冷却水出口2温度
    'refrigerator_1_high_press': 76,  # 制冷系统1高压压力值
    'refrigerator_2_high_press': 77,  # 制冷系统2高压压力值
    'refrigerator_1_low_press': 78,  # 制冷系统1低压压力值
    'refrigerator_2_low_press': 79,  # 制冷系统2低压压力值
    'new_wind_dettemp': 80,  # 新风温度检测值
    'tour_dettemp': 81,  # 观光区温度检测值
    'car_door_1_closed': 82,  # 车门1关闭
    'car_door_2_closed': 83,  # 车门2关闭
    'car_door_3_closed': 84,  # 车门3关闭
    'car_door_4_closed': 85,  # 车门4关闭
    'air_dettemp': 86,  # 客室内温度检测值
    'air_tartemp': 87,  # 目标温度值
    'charger_out_v1': 88,  # 充电机1输出电压
    'charger_out_v2': 89,  # 充电机2输出电压
    'battery_group_qty1': 90,  # 锂电池组1电量
    'battery_group_qty2': 91,  # 锂电池组2电量
    'charger_up_v1': 92,  # 充电机1充电电流
    'charger_up_v2': 93,  # 充电机2充电电流
    'charger_off1': 94,  # 充电机1切除
    'charger_off2': 95,  # 充电机2切除
    'park_brake_order': 96,  # 停放制动施加指令
    'park_brake_status': 97,  # 停放制动施加
    'park_brake_press': 98,  # 停放缸压力
    'air_compressor_status': 99,  # 空压机状态
    'pantograph_status': 100,  # 受电弓状态                      0_1_2_3	通信故障_升弓_降弓_切除
    'dc_trans_air_open': 101,  # 牵引变压器油泵空开              0_1
    'dc_trans_oil_stop_relay': 102,  # 牵引变压器油流停止继电器        0_1
    'high_off_switch_status': 103,  # 高压隔离开关状态  	             0_1_2_3	通信故障_闭合_断开_切除
    'vcb_off_monitor': 104,  # vcb闭合监视
    'pantograph_up': 105,  # 受电弓升起
    'high_off_switch_down': 106,  # 高压隔离开关闭合
    'aux_out_voltage': 107,  # 辅变输出电压
    'aux_out_current': 108,  # 辅变输出电流
    'dc_cvt_water_in1_press': 109,  # 牵引变流器冷却水进口1压力
    'dc_cvt_water_in2_press': 110,  # 牵引变流器冷却水进口2压力
    'air_rev_modal': 111,  # 空调运转模式
    'stop_brake_status': 112,  # 停放制动状态                 0_1_2_3  无效_施加_缓解_隔离
    'phase3_bus_relay1': 113,  # 三相母线有电检测继电器1
    'tmotor_fan_contactor1': 114,  # 牵引电机风机1接触器1
    'tmotor_fan_contactor2': 115,  # 牵引电机风机1接触器2
    'ATP_power_switch': 116,  # ATP电源控制开关
    'ATP_7_brake': 117,  # ATP七级制动
    'direction_zero': 118,  # 方向零位
    'forward1': 119,  # 前向1
    'forward2': 120,  # 前向2
    'rescue_switch_mid_pos': 121,  # 救援开关中间位
    'rescue_devic_contrl': 122,  # 救援装置控制
    'rescue_switch_rescued': 123,  # 救援开关被救援位
    'rescue_switch_rescue': 124,  # 救援开关救援位
    'aux_con_status': 125,  # 辅助变流器状态    0_1_2_3_4_5_6	无效_工作_非工作_故障_切除_隔离_通信故障
    'aux_trans_degree': 126,  # 辅助变压器温度
    'tconverter_air_degree': 127,  # 牵引变流器柜体空气温度
    'bp_press': 128,  # BP压力  --列车管压力
    'back1': 129,  # 后向1
    'back2': 130,  # 后向2
    'short1_contactor': 131,  # 一架短接接触器
    'short2_contactor': 132,  # 二架短接接触器
    'ac380_bus_contactor': 133,  # 交流380V母线接触器 0_1 断开 闭合
    'tract_conver_status': 134,  # 牵引变流器状态  0_1_2_3_4_5_6  无效_工作_非工作_故障_切除_隔离_通信故障
    'phase3_bus_relay2': 135,  # 三相母线有电检测继电器2
    'bpsave_devicontrl_air': 136,  # BP救援装置控制空开
    'charger1_115contactor': 137,  # 充电机1的115线接触器
    'charger2_115contactor': 138,  # 充电机2的115线接触器
    'tow_slide_count': 139,  # 牵引滑行次数
    'set_power_percent': 140,  # 设定牵引/电制动力百分比
    'passing_neutral_section': 141,  # 过分相
    'emergency_mode_switch1': 142,  # 紧急模式开关1     0_1  非紧急牵引模式_紧急牵引模式
    'emergency_mode_switch2': 143,  # 紧急模式开关2     0_1  非紧急牵引模式_紧急牵引模式
    'Auxiliary_inverter_cut': 144,  # 辅助逆变器切除状态  0_1 未切除_切除
    'air_brake_isolation': 145,  # 空气制动隔离
    'charger1_103_contactor': 146,  # 充电机1的103线接触器
    'charger2_103_contactor': 147,  # 充电机1的103线接触器
    'compressor1_run': 148,  # 压缩机1运转
    'compressor1_fault': 149,  # 压缩机1故障
    'compressor2_run': 150,  # 压缩机2运转
    'compressor2_fault': 151,  # 压缩机2故障
    'Charger_1_status': 152,  # 充电机1状态            0_1_2_3_4_5_6_7  通信故障_工作_故障_切除_未工作_无效_无效_无效
    'Charger_2_status': 153,  # 充电机2状态            0_1_2_3_4_5_6_7  通信故障_工作_故障_切除_未工作_无效_无效_无效
    'UB_air_brake_effective': 154,  # UB空气制动有效          0_1 无效_有效
    'brake_isolation_switch': 155,  # 保持制动隔离开关    0_1:无效_有效
    'keep_brake_app_status': 156,  # 保持制动施加状态    0_1:未施加_施加
    'relieve_keep_brake_switch': 157,  # 缓解保持制动开关    0_1:无效_有效
    'change_signal': 158,  # 换端信号
    'charge_battery_group1_tem1': 159,  # 碱性蓄电池组1温度1|充电机1温度1
    'charge_battery_group1_tem2': 160,  # 碱性蓄电池组1温度2|充电机1温度2
    'charge_battery_group2_tem1': 161,  # 碱性蓄电池组2温度1|充电机2温度1
    'charge_battery_group2_tem2': 162,  # 碱性蓄电池组2温度2|充电机2温度2
    'director_control_traction_pos': 163,  # 司控器牵引位
    'traction_energy1': 164,  # 牵引使能1
    'velocity_signal10': 165,  # 10km/h速度信号
    'velocity_signal05': 166,  # 5km/h速度信号
    'velocity_signal_less_than30': 167,  # 小于30km/h速度信号
    'velocity_signal_greater_than05': 168,  # 大于5公里速度信号
    'velocity_signal_less_than10': 169,  # 小于10km/h速度信号
    'Insufficient_wind_pressure': 170,  # 总风压力不足
    'wsp_speed1': 171,  # WSP1轴速度
    'wsp_speed2': 172,  # WSP2轴速度
    'wsp_speed3': 173,  # WSP3轴速度
    'wsp_speed4': 174,  # WSP4轴速度
    'cps_speed': 175,  # 综合速度
    'auxiliary_aircompressor_status':176, #辅助空压机状态
    'auxiliary_aircompressor_contactor':177 # 辅助空压机接触器
}

p_faultExt = {
    'N07': 'fault_code',
    'N01': 'fault_time',
    'N06': 'fault_status',
    'N08': 'coach_id'
}

cols_faultExt = {
    'train_type_code': 0,  # 车型
    'train_no': 1,  # 列号
    'coach_id': 2,  # 车号
    'fault_code': 3,  # 故障代码
    'fault_status': 4,  # 故障状态
    'time_value': 5  # 时间
}

p_sensor_mining = {
    # 1路轴温
    'Z03': '1',  # 1位轴箱
    'Z04': '2',  # 2位轴箱
    'Z05': '3',  # 3位轴箱
    'Z06': '4',  # 4位轴箱
    'Z07': '5',  # 5位轴箱
    'Z08': '6',  # 6位轴箱
    'Z09': '7',  # 7位轴箱
    'Z10': '8',  # 8位轴箱
    'Z11': '9',  # 1位小齿轮电机侧
    'Z12': '10',  # 2位小齿轮电机侧
    'Z13': '11',  # 3位小齿轮电机侧
    'Z14': '12',  # 4位小齿轮电机侧
    'Z15': '13',  # 1位小齿轮车轮侧
    'Z16': '14',  # 2位小齿轮车轮侧
    'Z17': '15',  # 3位小齿轮车轮侧
    'Z18': '16',  # 4位小齿轮车轮侧
    'Z19': '17',  # 1位电机定子
    'Z20': '18',  # 1位电机传动端
    'Z21': '19',  # 1位电机非传动端
    'Z22': '20',  # 2位电机定子
    'Z23': '21',  # 2位电机传动端
    'Z24': '22',  # 2位电机非传动端
    'Z25': '23',  # 3位电机定子
    'Z26': '24',  # 3位电机传动端
    'Z27': '25',  # 3位电机非传动端
    'Z28': '26',  # 4位电机定子
    'Z29': '27',  # 4位电机传动端
    'Z30': '28',  # 4位电机非传动端
    'Z31': '29',  # 1位大齿轮电机侧
    'Z32': '30',  # 2位大齿轮电机侧
    'Z33': '31',  # 3位大齿轮电机侧
    'Z34': '32',  # 4位大齿轮电机侧
    'Z35': '33',  # 1位大齿轮车轮侧
    'Z36': '34',  # 2位大齿轮车轮侧
    'Z37': '35',  # 3位大齿轮车轮侧
    'Z38': '36',  # 4位大齿轮车轮侧
    # 2路轴温
    'Z39': '37',  # 1位轴箱
    'Z40': '38',  # 2位轴箱
    'Z41': '39',  # 3位轴箱
    'Z42': '40',  # 4位轴箱
    'Z43': '41',  # 5位轴箱
    'Z44': '42',  # 6位轴箱
    'Z45': '43',  # 7位轴箱
    'Z46': '44',  # 8位轴箱
    'Z47': '45',  # 1位小齿轮电机侧
    'Z48': '46',  # 2位小齿轮电机侧
    'Z49': '47',  # 3位小齿轮电机侧
    'Z50': '48',  # 4位小齿轮电机侧
    'Z51': '49',  # 1位小齿轮车轮侧
    'Z52': '50',  # 2位小齿轮车轮侧
    'Z53': '51',  # 3位小齿轮车轮侧
    'Z54': '52',  # 4位小齿轮车轮侧
    'Z55': '53',  # 1位电机定子
    'Z56': '54',  # 1位电机传动端
    'Z57': '55',  # 1位电机非传动端
    'Z58': '56',  # 2位电机定子
    'Z59': '57',  # 2位电机传动端
    'Z60': '58',  # 2位电机非传动端
    'Z61': '59',  # 3位电机定子
    'Z62': '60',  # 3位电机传动端
    'Z63': '61',  # 3位电机非传动端
    'Z64': '62',  # 4位电机定子
    'Z65': '63',  # 4位电机传动端
    'Z66': '64',  # 4位电机非传动端
    'Z67': '65',  # 1位大齿轮电机侧
    'Z68': '66',  # 2位大齿轮电机侧
    'Z69': '67',  # 3位大齿轮电机侧
    'Z70': '68',  # 4位大齿轮电机侧
    'Z71': '69',  # 1位大齿轮车轮侧
    'Z72': '70',  # 2位大齿轮车轮侧
    'Z73': '71',  # 3位大齿轮车轮侧
    'Z74': '72',  # 4位大齿轮车轮侧
    'A09': 'speed',
    'H25': 'out_degree',
    'A06': 'trainsite_id',
    'A13': 'brake_pos',
    'A02': 'trainsite_valid',
    'B02': 'as_press',  # 空簧压力
    'A11': 'plus_minus_speed',  # 加减速度
}

p_E44 = {'U03': 'LRdoor_switch',  # 左右门开关触点断不开
         'U04': 'release_Ldoor_switch',  # 释放左门开关触点断不开
         'U05': 'apply_parking_switch',  # 施加停放开关触点断不开
         'U06': 'relief_parking_switch',  # 缓解停放开关触点断不开
         'U07': 'fault_reset_switch',  # 故障复位开关触点断不开
         'U08': 'em_reset_switch',  # 紧急复位开关触点断不开
         'U09': 'broken_VCB_switch',  # 断VCB开关触点断不开
         'U10': 'VCB_switch',  # 合VCB开关触点断不开
         'U11': 'bow_switch',  # 升弓开关触点断不开
         'U12': 'drop_bow_switch',  # 降弓开关触点断不开
         'U13': 'open_Rdoor_switch',  # 开右门开关触点断不开
         'U14': 'close_Rdoor_switch',  # 关右门开关触点断不开
         'U15': 'release_Rdoor_switch',  # 释放右门开关触点断不开
         'U16': 'open_Ldoor_switch',  # 开左门开关触点断不开
         'U17': 'asnormal_door11_close',  # 车门1-1关闭时间异常
         'U18': 'asnormal_door12_close',  # 车门1-2关闭时间异常
         'U19': 'asnormal_door21_close',  # 车门2-1关闭时间异常
         'U20': 'asnormal_door22_close',  # 车门2-2关闭时间异常
         'U21': 'relief_parking_brake',  # 缓解停放制动继电器动作不一致
         'U22': 'relief_hold_brake',  # 缓解保持制动继电器动作不一致
         'U23': 'em_brake_EB',  # 紧急制动EB安全环路继电器动作不一致
         'U24': 'em_brake_UB',  # 紧急制动UB安全环路继电器动作不一致
         'U25': 'TBM_input',  # TBM投入异常预警
         'U26': 'brake_force',  # 制动力异常预警
         'U27': 'air_brake_force',  # 空气制动力异常预警
         'U28': 'ins_dec_train_level',  # 列车级减速度不足预警
         'U29': 'cyc_ATP_brake7',  # 单车ATP七级制动继电器动作不一致
         'U30': 'cyc_brake_command1',  # 单车制动指令1继电器动作不一致
         'U31': 'cyc_brake_command2',  # 单车制动指令2继电器动作不一致
         'U32': 'cyc_brake_command3',  # 单车制动指令3继电器动作不一致
         'U33': 'cyc_control_brake_status',  # 单车司控器制动状态继电器动作不一致
         'U34': 'em_traction_command',  # 紧急牵引指令继电器动作不一致
         'U35': 'em_mode_relay',  # 紧急模式继电器动作不一致
         'U36': 'apply_parking_brake',  # 施加停放制动继电器动作不一致
         'U37': 'BJ_battery1_charged_floating_mode',  # 蓄电池1均充转浮充异常报警
         'U38': 'YJ_battery1_charged_floating_mode',  # 蓄电池1均充转浮充异常预警
         'U39': 'battery1_overcharged',  # 蓄电池1充电超时
         'U40': 'battery1_charging_vlotage',  # 蓄电池1充电电压异常
         'U41': 'BJ_battery2_charged_floating_mode',  # 蓄电池2均充转浮充异常报警
         'U42': 'YJ_battery1_charged_floating_mode',  # 蓄电池2均充转浮充异常预警
         'U43': 'battery2_overcharged',  # 蓄电池2充电超时
         'U44': 'battery2_charging_vlotage',  # 蓄电池2充电电压异常
         'U45': 'tcf_VCB_status',  # 牵引变流器反馈VCB状态信号不一致
         'U46': 'tcf_forward_signal',  # 牵引变流器反馈前向信号不一致
         'U47': 'tcf_backward_signal',  # 牵引变流器反馈后向信号不一致
         'U48': 'tcf_em_traction_mode',  # 牵引变流器反馈紧急牵引模式信号不一致
         'U49': 'tcf_em_traction_command',  # 牵引变流器反馈紧急牵引指令信号不一致
         'U50': 'tcf_em_braking_UB',  # 牵引变流器反馈紧急制动UB信号不一致
         'U51': 'tcf_disconnect_VCB_command',  # 牵引变流器反馈断VCB指令信号不一致
         'U52': 'tcf_broken_rescue_power_status',  # 牵引变流器反馈回送救援发电状态信号不一致
         'U53': 'tcm1_current_consist',  # 一架牵引变流器电机电流一致性异常
         'U54': 'tcm2_current_consist',  # 二架牵引变流器电机电流一致性异常
         'U55': 'tcm1_actual_force_consist',  # 一架牵引电机实际力一致性异常
         'U56': 'tcm2_actual_force_consist',  # 二架牵引电机实际力一致性异常
         'U57': 'current_trans_high_voltage',  # 电流互感器高压线缆接地
         'U58': 'CT2_high_voltage',  # 本单元CT2监测点后的连接线路高压线缆接地
         'U59': 'tt_primary_consist',  # 牵引变压器原边电流一致性异常预警
         'U60': 'tc1_q4_current_consist',  # 一架牵引变流器四象限电流一致性异常
         'U61': 'tc2_q4_current_consist',  # 二架牵引变流器四象限电流一致性异常
         'U62': 'motor1_force_consist',  # 一架电机设定力一致性异常预警
         'U63': 'motor2_force_consist',  # 二架电机设定力一致性异常预警
         'U64': 'inter_DC_voltage_consist',  # 中间直流电压一致性异常预警
         'U65': 'elec_brake_force_too_lage',  # 电制动力反馈偏差过大预警
         'U66': 'max_elec_brake_capacity_too_large',  # 最大电制动能力值偏差过大预警
         'U67': 'WSP1_motor_speed_too_large',  # 1轴WSP轴速度与电机速度偏差过大预警
         'U68': 'WSP2_motor_speed_too_large',  # 2轴WSP轴速度与电机速度偏差过大预警
         'U69': 'WSP3_motor_speed_too_large',  # 3轴WSP轴速度与电机速度偏差过大预警
         'U70': 'WSP4_motor_speed_too_large',  # 4轴WSP轴速度与电机速度偏差过大预警
         'U71': 'motor1_speed_too_large',  # 1轴电机速度与整车速度偏差过大预警
         'U72': 'motor2_speed_too_large',  # 2轴电机速度与整车速度偏差过大预警
         'U73': 'motor3_speed_too_large',  # 3轴电机速度与整车速度偏差过大预警
         'U74': 'motor4_speed_too_large',  # 4轴电机速度与整车速度偏差过大预警
         'U75': 'WPS1_vehicle_speed_too_large',  # 1轴WSP轴速度与整车速度偏差过大预警
         'U76': 'WPS2_vehicle_speed_too_large',  # 2轴WSP轴速度与整车速度偏差过大预警
         'U77': 'WPS3_vehicle_speed_too_large',  # 3轴WSP轴速度与整车速度偏差过大预警
         'U78': 'WPS4_vehicle_speed_too_large',  # 4轴WSP轴速度与整车速度偏差过大预警
         'A09': 'speed',  # 速度
         'U100': 'motor_fan2_contactor2_action',  # 牵引电机风机2接触器2动作次数
         'U101': 'converter_fan2_contactor1_action',  # 牵引变流器风机2接触器1动作次数
         'U102': 'converter_fan2_contactor2_action',  # 牵引变流器风机2接触器2动作次数
         'U103': 'transformer_fan1_contactor1_action',  # 牵引变压器风机1接触器1动作次数
         'U104': 'transformer_fan1_contactor2_action',  # 牵引变压器风机1接触器2动作次数
         'U105': 'transformer_fan2_contactor1_action',  # 牵引变压器风机2接触器1动作次数
         'U106': 'transformer_fan2_contactor2_action',  # 牵引变压器风机2接触器2动作次数
         'U107': 'breaker_action',  # 主断路器动作次数
         'U108': 'air_compressor_contactor_action',  # 主空压机接触器动作次数
         'U109': 'ground_protection_switch_action',  # 接地保护开关动作次数
         'U110': 'high_voltage_switch_action',  # 车高压隔离开关动作次数
         'U111': 'ATP_power_switch_action',  # ATP电源空开动作次数
         'U79': 'bow_switch_action',  # 升弓开关动作次数
         'U80': 'release_Rdoor_switch_action',  # 释放右门开关动作次数
         'U81': 'open_Rdoor_switch_action',  # 开右门开关动作次数
         'U82': 'close_Rdoor_switch_action',  # 关右门开关动作次数
         'U83': 'alert_pedal_switch_action',  # 警惕脚踏开关动作次数
         'U84': 'alert_manual_switch_action',  # 警惕手动开关动作次数
         'U85': 'fault_reset_switch_action',  # 故障复位开关动作次数
         'U86': 'emer_reset_switch_action',  # 紧急复位开关动作次数
         'U87': 'control_tow_action',  # 司控器牵引位继电器动作次数
         'U88': 'control_brake_action',  # 司控器制动位继电器动作次数
         'U89': 'control_constant_speed_action',  # 司控器恒速位继电器动作次数
         'U90': 'brake_command1_relay_action',  # 制动控制指令1继电器动作次数
         'U91': 'brake_command2_relay_action',  # 制动控制指令2继电器动作次数
         'U92': 'brake_command3_relay_action',  # 制动控制指令3继电器动作次数
         'U93': 'control_relay_action',  # 主控端继电器动作次数
         'U94': 'tow_relay_action',  # 牵引使能继电器动作次数
         'U95': '5km_speed_relay_action',  # 5km/h速度继电器动作次数
         'U96': 'em_brake_EB_relay_action',  # 紧急制动EB环路继电器动作次数
         'U97': 'tmotor1_contactor1_action',  # 牵引电机风机1接触器1动作次数
         'U98': 'tmotor1_contactor2_action',  # 牵引电机风机1接触器2动作次数
         'U99': 'tmotor2_contactor1_action'  # 牵引电机风机2接触器1动作次数
         }

cols_E44 = {
    'train_type_code': 0,  # 车型
    'train_no': 1,  # 列号
    'coach_id': 2,  # 车号
    'time_value': 3,  # 时间
    'LRdoor_switch': 4,  # 左右门开关触点断不开
    'release_Ldoor_switch': 5,  # 释放左门开关触点断不开
    'apply_parking_switch': 6,  # 施加停放开关触点断不开
    'relief_parking_switch': 7,  # 缓解停放开关触点断不开
    'fault_reset_switch': 8,  # 故障复位开关触点断不开
    'em_reset_switch': 9,  # 紧急复位开关触点断不开
    'broken_VCB_switch': 10,  # 断VCB开关触点断不开
    'VCB_switch': 11,  # 合VCB开关触点断不开
    'bow_switch': 12,  # 升弓开关触点断不开
    'drop_bow_switch': 13,  # 降弓开关触点断不开
    'open_Rdoor_switch': 14,  # 开右门开关触点断不开
    'close_Rdoor_switch': 15,  # 关右门开关触点断不开
    'release_Rdoor_switch': 16,  # 释放右门开关触点断不开
    'open_Ldoor_switch': 17,  # 开左门开关触点断不开
    'asnormal_door11_close': 18,  # 车门1-1关闭时间异常
    'asnormal_door12_close': 19,  # 车门1-2关闭时间异常
    'asnormal_door21_close': 20,  # 车门2-1关闭时间异常
    'asnormal_door22_close': 21,  # 车门2-2关闭时间异常
    'relief_parking_brake': 22,  # 缓解停放制动继电器动作不一致
    'relief_hold_brake': 23,  # 缓解保持制动继电器动作不一致
    'em_brake_EB': 24,  # 紧急制动EB安全环路继电器动作不一致
    'em_brake_UB': 25,  # 紧急制动UB安全环路继电器动作不一致
    'TBM_input': 26,  # TBM投入异常预警
    'brake_force': 27,  # 制动力异常预警
    'air_brake_force': 28,  # 空气制动力异常预警
    'ins_dec_train_level': 29,  # 列车级减速度不足预警
    'cyc_ATP_brake7': 30,  # 单车ATP七级制动继电器动作不一致
    'cyc_brake_command1': 31,  # 单车制动指令1继电器动作不一致
    'cyc_brake_command2': 32,  # 单车制动指令2继电器动作不一致
    'cyc_brake_command3': 33,  # 单车制动指令3继电器动作不一致
    'cyc_control_brake_status': 34,  # 单车司控器制动状态继电器动作不一致
    'em_traction_command': 35,  # 紧急牵引指令继电器动作不一致
    'em_mode_relay': 36,  # 紧急模式继电器动作不一致
    'apply_parking_brake': 37,  # 施加停放制动继电器动作不一致
    'BJ_battery1_charged_floating_mode': 38,  # 蓄电池1均充转浮充异常报警
    'YJ_battery1_charged_floating_mode': 39,  # 蓄电池1均充转浮充异常预警
    'battery1_overcharged': 40,  # 蓄电池1充电超时
    'battery1_charging_vlotage': 41,  # 蓄电池1充电电压异常
    'BJ_battery2_charged_floating_mode': 42,  # 蓄电池2均充转浮充异常报警
    'YJ_battery2_charged_floating_mode': 43,  # 蓄电池2均充转浮充异常预警
    'battery2_overcharged': 44,  # 蓄电池2充电超时
    'battery2_charging_vlotage': 45,  # 蓄电池2充电电压异常
    'tcf_VCB_status': 46,  # 牵引变流器反馈VCB状态信号不一致
    'tcf_forward_signal': 47,  # 牵引变流器反馈前向信号不一致
    'tcf_backward_signal': 48,  # 牵引变流器反馈后向信号不一致
    'tcf_em_traction_mode': 49,  # 牵引变流器反馈紧急牵引模式信号不一致
    'tcf_em_traction_command': 50,  # 牵引变流器反馈紧急牵引指令信号不一致
    'tcf_em_braking_UB': 51,  # 牵引变流器反馈紧急制动UB信号不一致
    'tcf_disconnect_VCB_command': 52,  # 牵引变流器反馈断VCB指令信号不一致
    'tcf_broken_rescue_power_status': 53,  # 牵引变流器反馈回送救援发电状态信号不一致
    'tcm1_current_consist': 54,  # 一架牵引变流器电机电流一致性异常
    'tcm2_current_consist': 55,  # 二架牵引变流器电机电流一致性异常
    'tcm1_actual_force_consist': 56,  # 一架牵引电机实际力一致性异常
    'tcm2_actual_force_consist': 57,  # 二架牵引电机实际力一致性异常
    'current_trans_high_voltage': 58,  # 电流互感器高压线缆接地
    'CT2_high_voltage': 59,  # 本单元CT2监测点后的连接线路高压线缆接地
    'tt_primary_consist': 60,  # 牵引变压器原边电流一致性异常预警
    'tc1_q4_current_consist': 61,  # 一架牵引变流器四象限电流一致性异常
    'tc2_q4_current_consist': 62,  # 二架牵引变流器四象限电流一致性异常
    'motor1_force_consist': 63,  # 一架电机设定力一致性异常预警
    'motor2_force_consist': 64,  # 二架电机设定力一致性异常预警
    'inter_DC_voltage_consist': 65,  # 中间直流电压一致性异常预警
    'elec_brake_force_too_lage': 66,  # 电制动力反馈偏差过大预警
    'max_elec_brake_capacity_too_large': 67,  # 最大电制动能力值偏差过大预警
    'WSP1_motor_speed_too_large': 68,  # 1轴WSP轴速度与电机速度偏差过大预警
    'WSP2_motor_speed_too_large': 69,  # 2轴WSP轴速度与电机速度偏差过大预警
    'WSP3_motor_speed_too_large': 70,  # 3轴WSP轴速度与电机速度偏差过大预警
    'WSP4_motor_speed_too_large': 71,  # 4轴WSP轴速度与电机速度偏差过大预警
    'motor1_speed_too_large': 72,  # 1轴电机速度与整车速度偏差过大预警
    'motor2_speed_too_large': 73,  # 2轴电机速度与整车速度偏差过大预警
    'motor3_speed_too_large': 74,  # 3轴电机速度与整车速度偏差过大预警
    'motor4_speed_too_large': 75,  # 4轴电机速度与整车速度偏差过大预警
    'WPS1_vehicle_speed_too_large': 76,  # 1轴WSP轴速度与整车速度偏差过大预警
    'WPS2_vehicle_speed_too_large': 77,  # 2轴WSP轴速度与整车速度偏差过大预警
    'WPS3_vehicle_speed_too_large': 78,  # 3轴WSP轴速度与整车速度偏差过大预警
    'WPS4_vehicle_speed_too_large': 79,  # 4轴WSP轴速度与整车速度偏差过大预警
    'speed': 80,  # 速度
    'motor_fan2_contactor2_action': 81,  # 牵引电机风机2接触器2动作次数
    'converter_fan2_contactor1_action': 82,  # 牵引变流器风机2接触器1动作次数
    'converter_fan2_contactor2_action': 83,  # 牵引变流器风机2接触器2动作次数
    'transformer_fan1_contactor1_action': 84,  # 牵引变压器风机1接触器1动作次数
    'transformer_fan1_contactor2_action': 85,  # 牵引变压器风机1接触器2动作次数
    'transformer_fan2_contactor1_action': 86,  # 牵引变压器风机2接触器1动作次数
    'transformer_fan2_contactor2_action': 87,  # 牵引变压器风机2接触器2动作次数
    'breaker_action': 88,  # 主断路器动作次数
    'air_compressor_contactor_action': 89,  # 主空压机接触器动作次数
    'ground_protection_switch_action': 90,  # 接地保护开关动作次数
    'high_voltage_switch_action': 91,  # 车高压隔离开关动作次数
    'ATP_power_switch_action': 92,  # ATP电源空开动作次数
    'bow_switch_action': 93,  # 升弓开关动作次数
    'release_Rdoor_switch_action': 94,  # 释放右门开关动作次数
    'open_Rdoor_switch_action': 95,  # 开右门开关动作次数
    'close_Rdoor_switch_action': 96,  # 关右门开关动作次数
    'alert_pedal_switch_action': 97,  # 警惕脚踏开关动作次数
    'alert_manual_switch_action': 98,  # 警惕手动开关动作次数
    'fault_reset_switch_action': 99,  # 故障复位开关动作次数
    'emer_reset_switch_action': 100,  # 紧急复位开关动作次数
    'control_tow_action': 101,  # 司控器牵引位继电器动作次数
    'control_brake_action': 102,  # 司控器制动位继电器动作次数
    'control_constant_speed_action': 103,  # 司控器恒速位继电器动作次数
    'brake_command1_relay_action': 104,  # 制动控制指令1继电器动作次数
    'brake_command2_relay_action': 105,  # 制动控制指令2继电器动作次数
    'brake_command3_relay_action': 106,  # 制动控制指令3继电器动作次数
    'control_relay_action': 107,  # 主控端继电器动作次数
    'tow_relay_action': 108,  # 牵引使能继电器动作次数
    '5km_speed_relay_action': 109,  # 5km/h速度继电器动作次数
    'em_brake_EB_relay_action': 110,  # 紧急制动EB环路继电器动作次数
    'tmotor1_contactor1_action': 111,  # 牵引电机风机1接触器1动作次数
    'tmotor1_contactor2_action': 112,  # 牵引电机风机1接触器2动作次数
    'tmotor2_contactor1_action': 113  # 牵引电机风机2接触器1动作次数
}

# 16车长编标准动车组车型列表
long_group_sdrext = ['E44']


import tensorflow as tf

tf.reset_default_graph()

import input_MNIST_data
from input_MNIST_data import shuffle_data
data = input_MNIST_data.read_data_sets("./data/", one_hot=True)

import numpy as np
import sys
import collections

import pickle
import pandas as pd

from sklearn.cluster import KMeans
from numpy import linalg as LA

import argparse
parser = argparse.ArgumentParser()

parser.add_argument(
        '--storage', '-m', default=4,
        help='The Memory Storage')
parser.add_argument(
        '--mini_batch', '-batch', default=1024,
        help='minibatch size')
parser.add_argument(
        '--whole_gradient', action='store_true',default=False,
        help='Compute the gradient using all data')
args = parser.parse_args()

print('----------------------------------------------')
print('architecture: LeNet-5 --- Data Set: MNIST')
print('----------------------------------------------')

# input and output shape
n_input   = data.train.images.shape[1]  # here MNIST data input (28,28)
n_classes = data.train.labels.shape[1]  # here MNIST (0-9 digits)

###############################################################################
########################## HYPER PARAMETER FOR LBFGS ##########################
###############################################################################
# memory limit
m = int(args.storage)
# minibatch size
minibatch = int(args.mini_batch)
# use entire data to compute gradient
use_whole_data_for_gradient = args.whole_gradient

# number of weights and bias in each layer
n_W = {}
dim_W = {}

# network architecture hyper parameters
input_shape = [-1,28,28,1]
W0 = 28
H0 = 28

# Layer 1 -- conv
D1 = 1
F1 = 5
K1 = 20
S1 = 1
W1 = (W0 - F1) // S1 + 1
H1 = (H0 - F1) // S1 + 1
conv1_dim = [F1, F1, D1, K1]
conv1_strides = [1,S1,S1,1]
n_W['w_conv1'] = F1 * F1 * D1 * K1
n_W['b_conv1'] = K1
dim_W['w_conv1'] = [F1, F1, D1, K1]
dim_W['b_conv1'] = [K1]
# Layer 2 -- max pool
D2 = K1
F2 = 2
K2 = D2
S2 = 2
W2 = (W1 - F2) // S2 + 1
H2 = (H1 - F2) // S2 + 1
layer2_ksize = [1,F2,F2,1]
layer2_strides = [1,S2,S2,1]

# Layer 3 -- conv
D3 = K2
F3 = 5
K3 = 50
S3 = 1
W3 = (W2 - F3) // S3 + 1
H3 = (H2 - F3) // S3 + 1
conv2_dim = [F3, F3, D3, K3]
conv2_strides = [1,S3,S3,1]
n_W['w_conv2'] = F3 * F3 * D3 * K3
n_W['b_conv2'] = K3
dim_W['w_conv2'] = [F3, F3, D3, K3]
dim_W['b_conv2'] = [K3]

# Layer 4 -- max pool
D4 = K3
F4 = 2
K4 = D4
S4 = 2
W4 = (W3 - F4) // S4 + 1
H4 = (H3 - F4) // S4 + 1
layer4_ksize = [1,F4,F4,1]
layer4_strides = [1,S4,S4,1]

# Layer 5 -- fully connected
n_in_fc = W4 * H4 * D4
n_hidden = 500
fc_dim = [n_in_fc,n_hidden]
n_W['w_fc'] = n_in_fc * n_hidden
n_W['b_fc'] = n_hidden
dim_W['w_fc'] = [n_in_fc,n_hidden]
dim_W['b_fc'] = [n_hidden]
# Layer 6 -- output
n_in_out = n_hidden
n_W['w_out'] = n_hidden * n_classes
n_W['b_out'] = n_classes
dim_W['w_out'] = [n_hidden,n_classes]
dim_W['b_out'] = [n_classes]

for key, value in n_W.items():
	n_W[key] = int(value)

# tf Graph input
x = tf.placeholder("float", [None, n_input])
y = tf.placeholder("float", [None, n_classes])

weights = {
    # 5 x 5 convolution, 1 input image, 20 outputs
    'w_conv1': tf.get_variable('w_conv1', shape=[F1, F1, D1, K1],
           			#initializer=tf.random_normal_initializer(stddev = 0.001)),
           			initializer=tf.contrib.layers.xavier_initializer()),
           			#initializer=tf.zeros_initializer()),
    # 'conv1': tf.Variable(tf.random_normal([F1, F1, D1, K1])),
    # 5x5 conv, 20 inputs, 50 outputs
    #'conv2': tf.Variable(tf.random_normal([F3, F3, D3, K3])),
    'w_conv2': tf.get_variable('w_conv2', shape=[F3, F3, D3, K3],
           			#initializer=tf.random_normal_initializer(stddev = 0.001)),
           			initializer=tf.contrib.layers.xavier_initializer()),
           			#initializer=tf.zeros_initializer()),
    # fully connected, 800 inputs, 500 outputs
    #'fc': tf.Variable(tf.random_normal([n_in_fc, n_hidden])),
    'w_fc': tf.get_variable('w_fc', shape=[n_in_fc, n_hidden],
           			#initializer=tf.random_normal_initializer(stddev = 0.001)),
           			initializer=tf.contrib.layers.xavier_initializer()),
           			#initializer=tf.zeros_initializer()),
    # 500 inputs, 10 outputs (class prediction)
    #'out': tf.Variable(tf.random_normal([n_hidden, n_classes]))
    'w_out': tf.get_variable('w_out', shape=[n_hidden, n_classes],
           			#initializer=tf.random_normal_initializer(stddev = 0.001)),
           			initializer=tf.contrib.layers.xavier_initializer()),
           			#initializer=tf.zeros_initializer()),
    'b_conv1': tf.get_variable('b_conv1', shape=[K1],
           			#initializer=tf.random_normal_initializer(stddev = 0.001)),
           			 #initializer=tf.zeros_initializer()),
           			initializer=tf.contrib.layers.xavier_initializer()),
    'b_conv2': tf.get_variable('b_conv2', shape=[K3],
           			#initializer=tf.random_normal_initializer(stddev = 0.001)),
           			#initializer=tf.zeros_initializer()),
           			initializer=tf.contrib.layers.xavier_initializer()),
    'b_fc': tf.get_variable('b_fc', shape=[n_hidden],
           			#initializer=tf.random_normal_initializer(stddev = 0.001)),
           			#initializer=tf.zeros_initializer()),
           			initializer=tf.contrib.layers.xavier_initializer()),
    'b_out': tf.get_variable('b_out', shape=[n_classes],
           			#initializer=tf.random_normal_initializer(stddev = 0.001))
           			#initializer=tf.zeros_initializer())
           			initializer=tf.contrib.layers.xavier_initializer())
}

S = {}
Y = {}

for k in range(m):
	i = str(k)
	S[str(k)] = {}
	Y[str(k)] = {}

for k in range(m):
	i = str(k)
	for layer, _ in weights.items():
		S[i][layer] = np.random.rand(n_W[layer]).reshape(dim_W[layer])
		Y[i][layer] = np.random.rand(n_W[layer]).reshape(dim_W[layer])

def enqueue(mp,new_s_val,new_y_val):
	for layer, _ in weights.items():
		i = str(mp)
		S[i][layer] = new_s_val[layer]
		Y[i][layer] = new_y_val[layer]

def dequeue():
	S_tmp = S
	Y_tmp = Y
	for k in range(m-1):
		i = str(k)
		j = str(k+1)
		for layer, _ in weights.items():
			S[i][layer] = S_tmp[j][layer]
			Y[i][layer] = Y_tmp[j][layer]

def model(x,_W):
	# Reshape input to a 4D tensor
    x = tf.reshape(x, shape = input_shape)
    # LAYER 1 -- Convolution Layer
    conv1 = tf.nn.relu(tf.nn.conv2d(input = x,
    								filter =_W['w_conv1'],
    								strides = [1,S1,S1,1],
    								padding = 'VALID') + _W['b_conv1'])
    # Layer 2 -- max pool
    conv1 = tf.nn.max_pool(	value = conv1,
    						ksize = [1, F2, F2, 1],
    						strides = [1, S2, S2, 1],
    						padding = 'VALID')

    # LAYER 3 -- Convolution Layer
    conv2 = tf.nn.relu(tf.nn.conv2d(input = conv1,
    								filter =_W['w_conv2'],
    								strides = [1,S3,S3,1],
    								padding = 'VALID') + _W['b_conv2'])
    # Layer 4 -- max pool
    conv2 = tf.nn.max_pool(	value = conv2 ,
    						ksize = [1, F4, F4, 1],
    						strides = [1, S4, S4, 1],
    						padding = 'VALID')
    # Fully connected layer
    # Reshape conv2 output to fit fully connected layer
    fc = tf.contrib.layers.flatten(conv2)
    fc = tf.nn.relu(tf.matmul(fc, _W['w_fc']) + _W['b_fc'])
    # fc = tf.nn.dropout(fc, dropout_rate)

    output = tf.matmul(fc, _W['w_out']) + _W['b_out']
    return output

# Construct model
output = model(x,weights)
# Softmax loss
loss = tf.reduce_mean(
	tf.nn.softmax_cross_entropy_with_logits(labels = y, logits = output))
correct_prediction = tf.equal(tf.argmax(output, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
###############################################################################
######################## TF GRADINETS #########################################
###############################################################################
grad_w = {}
for layer, _ in weights.items():
	grad_w[layer] = tf.gradients(xs=weights[layer], ys=loss)
###############################################################################
######################## TF Auxilary variables ################################
###############################################################################
aux_w = {}
# alpha_tf = tf.placeholder("float",shape=[])
# p_tf = {}

for layer, _ in weights.items():
	name = layer + 'aux_w_'
	aux_w[layer] = tf.get_variable(name=name, shape=weights[layer].get_shape(),
						initializer=tf.contrib.layers.xavier_initializer())

# for layer, _ in weights.items():
# 	p_tf[layer] = tf.placeholder("float", shape=weights[layer].get_shape())

aux_w_placeholder = {}
for layer, _ in weights.items():
	aux_w_placeholder[layer] = tf.placeholder(dtype="float",
										shape=weights[layer].get_shape())
aux_w_init = {}
for layer, _ in weights.items():
	aux_w_init[layer] = aux_w[layer].assign(aux_w_placeholder[layer])

aux_output = model(x,aux_w)
aux_loss = tf.reduce_mean(
	tf.nn.softmax_cross_entropy_with_logits(labels = y, logits = aux_output))
aux_grad_w = {}
for layer, _ in weights.items():
	aux_grad_w[layer] = tf.gradients(xs=aux_w[layer], ys=aux_loss)

update_w = {}
update_w_placeholder = {}
for layer, _ in weights.items():
	update_w_placeholder[layer] = tf.placeholder(dtype="float",
										shape=weights[layer].get_shape())
for layer, _ in weights.items():
	update_w[layer] = weights[layer].assign(update_w_placeholder[layer])

################################################################################
################################################################################
saver = tf.train.Saver()
init = tf.global_variables_initializer()
###############################################################################
######## training data and neural net architecture with weights w #############
###############################################################################
print('----------------------------------------------')
print('TRAINGING REFERENCE NET for LeNet-5')
print('----------------------------------------------')
################### TO SAVE TRAINING AND TEST LOSS AND ERROR ##################
################### FOR REFERENCE NET #########################################
# Total minibatches
total_steps = 100
# total_minibatches = 400
# number of minibatches in data
num_minibatches_data = data.train.images.shape[0] // minibatch

train_loss_steps = np.zeros(total_steps)
train_accuracy_steps = np.zeros(total_steps)
val_loss_steps = np.zeros(total_steps)
val_accuracy_steps = np.zeros(total_steps)
test_loss_steps = np.zeros(total_steps)
test_accuracy_steps = np.zeros(total_steps)
################### TO SAVE MODEL ##################
model_file_name = 'robust_LBFGS_model_lenet_5.ckpt'
model_file_path = './model/' + model_file_name
############################## L-BFGS #########################################
def compute_whole_gradient(sess,grad_tf,feed_dict,X_train,y_train):
	gw = {}
	for j in range(num_minibatches_data):
		index_minibatch = j % num_minibatches_data
		# mini batch
		start_index = index_minibatch     * minibatch
		end_index   = (index_minibatch+1) * minibatch
		X_batch = X_train[start_index:end_index]
		y_batch = y_train[start_index:end_index]
		feed_dict.update({	x: X_batch,
							y: y_batch})

		gw_list = sess.run(grad_tf, feed_dict=feed_dict)
		if j == 0:
			for layer, _ in weights.items():
				gw[layer] = gw_list[layer][0]
		else:
			for layer, _ in weights.items():
				gw[layer] = gw[layer] + gw_list[layer][0]

	for layer, _ in weights.items():
		gw[layer] = gw[layer] * 1 / num_minibatches_data
	return gw

def compute_multibatch_gradient(sess,grad_tf,feed_dict,X_train,y_train):
	gw = {}
	num_minibatches_here = X_train.shape[0] // minibatch
	for j in range(num_minibatches_here):
		index_minibatch = j % num_minibatches_here
		# mini batch
		start_index = index_minibatch     * minibatch
		end_index   = (index_minibatch+1) * minibatch
		X_batch = X_train[start_index:end_index]
		y_batch = y_train[start_index:end_index]
		feed_dict.update({	x: X_batch,
							y: y_batch})

		gw_list = sess.run(grad_tf, feed_dict=feed_dict)
		if j == 0:
			for layer, _ in weights.items():
				gw[layer] = gw_list[layer][0]
		else:
			for layer, _ in weights.items():
				gw[layer] = gw[layer] + gw_list[layer][0]

	for layer, _ in weights.items():
		gw[layer] = gw[layer] * 1 / num_minibatches_here
	return gw

def compute_whole_tensor(sess,tensor_tf,feed_dict,X_train,y_train):
	total = 0
	for j in range(num_minibatches_data):
		index_minibatch = j % num_minibatches_data
		# mini batch
		start_index = index_minibatch     * minibatch
		end_index   = (index_minibatch+1) * minibatch
		X_batch = X_train[start_index:end_index]
		y_batch = y_train[start_index:end_index]
		feed_dict.update({	x: X_batch,
							y: y_batch})

		value = sess.run(tensor_tf, feed_dict=feed_dict)
		total = total + value

	total = total * 1 / num_minibatches_data
	return total

def compute_multibatch_tensor(sess,tensor_tf,feed_dict,X_train,y_train):
	total = 0
	num_minibatches_here = X_train.shape[0] // minibatch
	for j in range(num_minibatches_here):
		index_minibatch = j % num_minibatches_here
		# mini batch
		start_index = index_minibatch     * minibatch
		end_index   = (index_minibatch+1) * minibatch
		X_batch = X_train[start_index:end_index]
		y_batch = y_train[start_index:end_index]
		feed_dict.update({	x: X_batch,
							y: y_batch})

		value = sess.run(tensor_tf, feed_dict=feed_dict)
		total = total + value

	total = total * 1 / num_minibatches_here
	return total

with tf.Session() as sess:
	sess.run(init)
	new_grad_w = {}
	old_grad_w = {}
	old_f = 0
	new_f = 0
	old_grad_w = {}
	new_grad_w = {}
	old_w = {}
	new_w = {}
	feed_dict = {}
	X_train, y_train = shuffle_data(data)
	for k in range(total_steps):
		if k % 2 == 0:
			start_index = 0
			end_index = 2 * X_train.shape[0] // 3
			X_train_multi = X_train[start_index:end_index]
			y_train_multi = y_train[start_index:end_index]
		else:
			start_index = X_train.shape[0] // 3
			end_index = X_train.shape[0]
			X_train_multi = X_train[start_index:]
			y_train_multi = y_train[start_index:]

		#old_grad_w = compute_whole_gradient(sess,grad_w,feed_dict,X_train, y_train)
		old_grad_w = compute_multibatch_gradient(sess,grad_w,feed_dict,
												X_train_multi, y_train_multi)
		if k < m:
			mp = k
		else:
			mp = m

		########################################################################
		####################### SEARCH DIRECTION ###############################
		########################################################################
		q = old_grad_w
		yTs = {}
		rho = {}
		sTq = {}
		alpha = {}
		yTr = {}
		eps = np.finfo(float).eps
		for t in range(mp):
			i = str(t)
			yTs[i] = 0
			rho[i] = 0
			sTq[i] = 0
			alpha[i] = 0
			yTr[i] = 0

		for t in range(mp-1,-1,-1):
			i = str(t)
			for layer, _ in weights.items():
				yTs[i] = yTs[i] + np.dot(S[i][layer].flatten(),
										 Y[i][layer].flatten())

			rho[i] = 1 / ( yTs[i] + eps)

			for layer, _ in weights.items():
				sTq[i] = sTq[i] + np.dot(S[i][layer].flatten(),q[layer].flatten())
			alpha[i] = rho[i] * sTq[i]

			for layer, _ in weights.items():
				q[layer] = q[layer] - alpha[i] * Y[i][layer]

		r = q

		if (mp >=1):
			sTy = 0
			yTy = 0
			for layer, _ in weights.items():
				sTy = sTy + np.dot( S[str(mp-1)][layer].flatten(),
									Y[str(mp-1)][layer].flatten())
				yTy = yTy + np.dot( Y[str(mp-1)][layer].flatten(),
									Y[str(mp-1)][layer].flatten())
			gamma = sTy / yTy
			gamma = np.max((gamma,0.1))
			for layer,_ in weights.items():
				r[layer] = gamma * q[layer]

		for t in range(mp):
			i = str(t)
			for layer, _ in weights.items():
				yTr[i] = yTr[i] + np.dot(Y[i][layer].flatten(), r[layer].flatten())

			beta = rho[i] * yTr[i]

			for layer, _ in weights.items():
				r[layer] = r[layer] + S[i][layer] * ( alpha[i] - beta )
		p = {}
		for layer, _ in weights.items():
			p[layer] = -1 * r[layer]
		p_val = p
		########################################################################
		############## FINDING ALPHA TO SATISFY ################################
		############## WOLFE CONDITIONS ########################################
		########################################################################
		alpha_step_vec = np.linspace(1.0,0.5,5,dtype='float')
		c1 = 1E-4
		c2 = 0.9
		old_w = sess.run(weights)

		feed_dict = {}

		# old_f = compute_whole_tensor(sess,loss,feed_dict,X_train, y_train)
		old_f = compute_multibatch_tensor(sess,loss,feed_dict,	X_train_multi,
																y_train_multi)
		for alpha_step in alpha_step_vec:
			new_w = {}
			feed_dict = {}
			for layer, _ in weights.items():
				new_w[layer] = old_w[layer] + alpha_step * p_val[layer]

			for layer, _ in weights.items():
				feed_dict.update({aux_w_placeholder[layer]: new_w[layer]})
			sess.run(aux_w_init, feed_dict=feed_dict)
			# new_f = sess.run(aux_loss,feed_dict=feed_dict)
			# new_f = compute_whole_tensor(sess,aux_loss,feed_dict,X_train,y_train)
			new_f = compute_multibatch_tensor(sess,aux_loss,feed_dict,X_train_multi,
																	y_train_multi)
			gradTp = 0

			for layer, _ in weights.items():
				gradTp = gradTp + np.dot(old_grad_w[layer].flatten(),
										 p_val[layer].flatten())
			rhs = c1 * alpha_step * gradTp

			if new_f <= (old_f + rhs):
				Wolfe_cond_1 = True
			else:
				Wolfe_cond_1 = False
			#new_grad_w = compute_whole_gradient(sess,aux_grad_w,feed_dict,X_train, y_train)
			new_grad_w = compute_multibatch_gradient(sess,aux_grad_w,feed_dict,
												X_train_multi, y_train_multi)
			new_grad_wTp = 0
			for layer, _ in weights.items():
				new_grad_wTp = new_grad_wTp + np.dot(new_grad_w[layer].flatten(),
													 p_val[layer].flatten())
			if new_grad_wTp >= c2 * gradTp:
				Wolfe_cond_2 = True
			else:
				Wolfe_cond_2 = False

			if Wolfe_cond_1 and Wolfe_cond_2:
				break
		########################################################################
		############## UPDATE Storage S and Y ##################################
		########################################################################
		new_s_val = {}
		new_y_val = {}
		for layer, _ in weights.items():
			new_s_val[layer] = new_w[layer] - old_w[layer]
			new_y_val[layer] = new_grad_w[layer] - old_grad_w[layer]

		if k < m:
			enqueue(k,new_s_val,new_y_val)
		else:
			dequeue()
			enqueue(m-1,new_s_val,new_y_val)

		norm_grad = 0
		for layer, _ in weights.items():
			norm_grad = norm_grad + LA.norm(old_grad_w[layer])
		if norm_grad < 1E-5:
			break
		########################################################################
		############## UPDATE Weights ##########################################
		########################################################################
		feed_dict_w = {}
		for layer, _ in weights.items():
			feed_dict_w.update({update_w_placeholder[layer]: new_w[layer]})
		sess.run(update_w,feed_dict=feed_dict_w)

		############### LOSS AND ACCURACY EVALUATION ##########################
		# X_batch = X_train[0:minibatch]
		# y_batch = y_train[0:minibatch]

		# train_loss, train_accuracy = \
		# 		sess.run([loss, accuracy], feed_dict = {x: X_batch,
		# 											    y: y_batch} )

		train_loss = new_f
		feed_dict = {};
		# train_accuracy = compute_whole_tensor(sess,accuracy,feed_dict,X_train, y_train)
		train_accuracy = compute_multibatch_tensor(sess,accuracy,feed_dict,
												X_train_multi, y_train_multi)

		train_loss_steps[k] = train_loss
		train_accuracy_steps[k] = train_accuracy

		val_loss, val_accuracy = \
		sess.run([loss, accuracy], feed_dict = {x: data.validation.images,
												y: data.validation.labels} )

		test_loss_steps[k] = val_loss
		test_accuracy_steps[k] = val_accuracy

		print('step: {}, train loss: {}, train acuracy: {}' \
			.format(k, train_loss, train_accuracy) )
		print('step: {}, val loss: {}, val acuracy: {}' \
			.format(k, val_loss, val_accuracy) )

# save the results
result_path = './results_robust_LBFGS/' + 'results_m_' + str(m)
np.savez(result_path,train_loss_steps=train_loss_steps,
					train_accuracy_steps=train_accuracy_steps,
					test_loss_steps = test_loss_steps,
					test_accuracy_steps=test_accuracy_steps)


# Methods for solving differential equations implemented in Python.
# Yes, Wolfram Alpha can already do all this... but do you know HOW it does it?

from math import *

def countChar(string,chars):
    '''Return the number of times a character in chars appears in string.'''
    num = 0
    for i in string:
        if i in chars: num+=1 #Increment count by 1
    return num

def zeros(ce):
    '''Returns the zeros of a polynomial equation defined by a list in the form [[c1,order1],[c2,order2],...]'''
    #Merge any same-order terms Eg. 2y^2 + y^2 ---> 3y^2
    newCe = [x[1] for x in ce] #Take only order
    newCe = [[0,x1] for x1 in newCe] #Set coefficient for each order to 0

    for term in ce: #Iterate across terms
        for i2 in range(len(newCe)): #Iterate across orders
            if term[1] == newCe[i2][1]:
                newCe[i2][0]+=term[0] #Add coefficient to matching order
                break
    ce = newCe #Done merging

    #Fill in any non-existent terms with 0
    maxOrder = max(ce,key=lambda x:x[1])[1]
    for i in range(maxOrder):
        if not(i in [x[1] for x in ce]):
            ce.append([0,i])
    ce.sort(key=lambda x: x[1]) #Sort by order
    ce = [x[0] for x in ce] #Strip the now useless order

    #Now that ce is in a manageable form, we can compute based off of order (now len(ce) - 1)
    if len(ce) == 1: #Constant 0 = c
        if ce[0] == 0:
            return "Infinite zeros."
        else:
            return "No zeros."

    if len(ce) == 2: #Line 0 = mx+c ---> x = -c/m
        return [-ce[0]/ce[1]]

    if len(ce) == 3: #Quadratic 0 = ax^2 + bx + c ---> x = (-b+-sqrt(b^2-4ac))/2a
        return quadraticRoots(ce[2],ce[1],ce[0])

    if len(ce) > 3: #How to find roots of larger orders???
        raise ValueError("Order of polynomial >2!")

def quadraticRoots(a,b,c):
    '''Returns all roots of a quadratic.'''
    det = b**2-4*a*c
    if det < 0:
        #Special handling required for complex roots...
        return ["Complex conjugate roots:",-b/2.*a,sqrt(-b**2+4*a*c)]
    return [(-b+sqrt(det))/(2.*a),(-b-sqrt(det))/(2.*a)]

def cubicRoot(a,b,c,d):
    '''Returns 1 root of a quartic if there is 1 real root.'''
    p = -b/(3*a)
    q = p**3 + (b*c - 3*a*d)/(6*a**2)
    r = c/(3*a)

    m = (q**2 + (r-p**2)**3)**0.5
    x = (q + m)**(1/3.)
    x+= (q - m)**(1/3.) + p
    return x

class MathError(Exception):pass #Custom exceptions

class Equation:
    '''Differential equation class.'''
    def __init__(self,string):
        #Example value for string is "y' = y + 2y"
        self.raw = string
        string = string.replace(" ","")
        LS,RS = string.split("=") #Split sides

        #Split terms
        LS = LS.replace("+",":")
        LS = LS.replace("-",":-") #Keep negative sign
        LS = LS.split(":")

        RS = RS.replace("+",":")
        RS = RS.replace("-",":-")
        RS = RS.split(":")

        self.RS,self.LS = RS,LS

        #Check for linearity
        linear = True
        for term in RS+LS:
            if countChar(term,["y"]) > 1: #Linear is y appears only once in each term
                linear = False
                break
        self.linear = linear

        #Determine order
        order = 0
        for term in RS+LS:
            order = max(order,countChar(term,["'","y"])) # ' means "prime", count these
        self.order = order

        #Check for homogenity
        #THIS NEEDS TO BE IMPROVED
        if LS == ["0"] or RS == ["0"]:
            self.homogenous = True
        else:
            self.homogenous = False

    def copy(self):
        '''Return copy of the equation.'''
        new = Equation(self.raw)
        #Preserve LS,RS orientation
        new.LS = list(self.LS)
        new.RS = list(self.RS)
        return new

E = Equation #Shorthand

def constant_coefficients(equation):
    '''Solve for form a+by+cy'+dy''+... = 0'''
    if not(equation.linear):
        raise MathError("Equation is not linear!")
    if not(equation.homogenous):
        raise MathError("Equation is not homogenous!")

    if len(equation.RS) > len(equation.LS):
        #Switch sides to be correct form
        equation.LS,equation.RS = equation.RS,equation.LS
    if len(equation.RS) != 1:
        raise MathError("Equation is not in correct form!")

    #Characteristic equation
    ce = []
    for i in equation.LS:
        order = countChar(i,["'","y"]) #Order of term
        a = i.replace("y","") #Get rid of y
        a = a.replace("'","") #Get rid of '

        if a == "" : #If nothing is left, coefficient must be 1
            a = "1"
        elif a == "-": #Same for negative
            a = "-1"

        c = int(a) #Convert coeffient to int
        if (order > 0): #Don't include constant term
            ce.append([c,order-1]) #Add to list (-1 because characteristic)
    r = zeros(ce) #Find the zeros for the characteristic equation
    solution = ""

    if r[0] == "Complex conjugate roots:": #Special case for complex conjugate roots
        solution = "c1*e^("+str(r[1])+"*x)*cos("+str(r[2])+"*x) + "
        solution+= "c2*e^("+str(r[1])+"*x)*sin("+str(r[2])+"*x)"
        return solution

    duplicates = [] #Count the number of duplicate roots
    for i in range(len(r)):
        new = True
        numDuplicates = 0
        for d in duplicates: #Iterate through logged duplicates
            if d[0] == r[i]: #If this is a duplicate, increment count
                d[1]+=1
                numDuplicates = d[1] #Save for later
                new = False #Don't log as a new duplicate
                break
        if new:
            duplicates.append([r[i],0]) #Log as a new duplicate

        #Convert solution to string output
        solution+="c"+str(i+1)+"*"+"x"*numDuplicates+"e^("+str(r[i])+"*x) + "
    return solution[:-3]

cc = constant_coefficients #Shorthand

def easy_input():
    string = input("Equation:")
    equation = E(string)

    try:
        print(cc(equation))
    except MathError: pass

    print("\n")

def demo():
    print("y' + 5y = 0")
    print("Constant coefficients...")
    print(cc(E("y' + 5y = 0")))
    print("\n")
    print("y'' = 0")
    print("Constant coefficients...")
    print(cc(E("y'' = 0")))
    print("\n")
    print("-y'' +y' +3y = 0")
    print("Constant coefficients...")
    print(cc(E("-y'' +y' +3y= 0")))
    print("\n")
    while True:
        easy_input()
demo()


import re
def polySum(poly1,poly2):
    sum_Poly = [0]
    len1 = 2
    len2 = 2
    count = 0
    while len1 <= len(poly1)-1 and len2 <= len(poly2)-1:
        if poly1[len1] < poly2[len2]:
            sum_Poly.append(poly2[len2-1])
            sum_Poly.append(poly2[len2])
            len2 += 2
            count += 1
        elif poly1[len1] > poly2[len2]:
            sum_Poly.append(poly1[len1-1])
            sum_Poly.append(poly1[len1])
            len1 += 2
            count += 1
        else:
            if 0 == poly1[len1 - 1] + poly2[len2 - 1]:
                len1 += 2
                len2 += 2
            else:
                sum_Poly.append(poly1[len1-1]+poly2[len2-1])
                sum_Poly.append(poly2[len2])
                len1 += 2
                len2 += 2
                count += 1
    while len1 <= len(poly1)-1:
        sum_Poly.append(poly1[len1-1])
        sum_Poly.append(poly1[len1])
        len1 += 2
        count += 1
    while len2 <= len(poly2) -1:
        sum_Poly.append(poly2[len2-1])
        sum_Poly.append(poly2[len2])
        len2 += 2
        count += 1
    sum_Poly[0] = count
    return sum_Poly

def polyMultiply(poly1,poly2):
    mult_Poly = [0]
    count = 0
    for i in range(2,len(poly2),2):
        mult_Poly.append(poly1[1]*poly2[i-1])
        mult_Poly.append(poly1[2]+poly2[i])
        count += 1
    for m in range(4, len(poly1),2):
        for n in range(2,len(poly2),2):
            index = poly1[m-1] * poly2[n-1]
            mi = poly1[m] + poly2[n]
            for j in range(2,len(mult_Poly),2):
                if mult_Poly[j] > mi :
                    if j < len(mult_Poly)-1:
                        continue
                    else:
                        mult_Poly.append(index)
                        mult_Poly.append(mi)
                        count += 1
                        break
                elif mult_Poly[j] == mi :
                    mult_Poly[j-1] = index + mult_Poly[j-1]
                    if mult_Poly[j-1] == 0:
                        del mult_Poly[j-1]
                        del mult_Poly[j-1]
                        count -= 1
                        break
                    else:
                        count += 1
                        break
                else:
                    mult_Poly.insert(j-1,mi)
                    mult_Poly.insert(j-1,index)
                    count += 1
                    break
    mult_Poly[0] = count
    return mult_Poly

def poly_sum_print(poly_sum):
    for i in range(1,len(poly_sum)):
        if i != len(poly_sum)-1:
            print(poly_sum[i], end=" ")
        else:
            print(poly_sum[i])

def poly_multi_print(poly_multi):
    for i in range(1,len(poly_multi)):
        if i != len(poly_multi)-1:
            print(poly_multi[i],end=' ')
        else:
            print(poly_multi[i])

def main():
    poly1 = re.split(r'\s+',input())
    poly2 = re.split(r'\s+',input())
    for i in range(len(poly1)):
        poly1[i] = eval(poly1[i])
    for i in range(len(poly2)):
         poly2[i] = eval(poly2[i])
    if poly1[0] != 0 and poly2[0] != 0:
        poly_sum = polySum(poly1,poly2)
        poly_multi = polyMultiply(poly1,poly2)
        if poly_multi[0] != 0:
            poly_multi_print(poly_multi)
        else:
            print(0,0)
        if poly_sum[0] != 0:
            poly_sum_print(poly_sum)
        else:
            print(0,0)
    elif poly1[0] == 0 and poly2[0] != 0:
        poly_multi = [0,0,0]
        poly_sum = poly2
        poly_multi_print(poly_multi)
        poly_sum_print(poly_sum)
    elif poly1[0] != 0 and poly2[0] == 0:
        poly_multi = [0,0,0]
        poly_sum = poly1
        poly_multi_print(poly_multi)
        poly_sum_print(poly_sum)
    else:
        print(0,0)
        print(0,0)
main()


# The MIT License (MIT)
#
# Copyright (c) 2014 Tom Shaked & Jonathan Dortheimer
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
from ga import chromosome, creature, evolution, fitness_equilateral, fitness_size, reproduction_divider
import random
#import rhinoscriptsyntax as rs
import math
import copy
import time

class Gene:
    def __init__(self, points=None, boundingPolygon=None):

        self.maxX = 0
        self.minX = 0
        self.minY = 0
        self.maxY = 0
        if boundingPolygon:
            self.setBoundingPolygon(boundingPolygon)
        if points is None:
            self.generateRandom()
        else:
            self.points = points

    # determine if a point is inside a given polygon or not
    # Polygon is a list of (x,y) pairs.
    def point_inside_polygon(self,x,y,poly):

        n = len(poly)
        inside =False
        p1x,p1y = poly[0]
        for i in range(n+1):
            p2x,p2y = poly[i % n]
            if y > min(p1y,p2y):
                if y <= max(p1y,p2y):
                    if x <= max(p1x,p2x):
                        if p1y != p2y:
                            xinters = (y-p1y)*(p2x-p1x)/(p2y-p1y)+p1x
                        if p1x == p2x or x <= xinters:
                            inside = not inside
            p1x,p1y = p2x,p2y

        return inside

    def setBoundingPolygon(self, boundingPolygon):

        self.boundingPolygon = boundingPolygon
        for pt in boundingPolygon:
            if pt[0]> self.maxX:
                # print(pt[0],self.maxX)
                self.maxX = pt[0]
            if pt[1]> self.maxY:
                self.maxY = pt[1]
            if pt[0]< self.minX:
                self.minX = pt[0]
            if pt[1]< self.minY:
                self.minY = pt[1]

    def generateRandom(self):
        self.points = []
        for x in range(10):
            point = [self.rand(self.minX,self.maxX),self.rand(self.minY, self.maxY),0]
            while True != self.point_inside_polygon(point[0],point[1],self.boundingPolygon):
                point = [self.rand(self.minX,self.maxX),self.rand(self.minY, self.maxY),0]
            self.points.append(point)

    def rand(self,min=None,max=None):
        if min == None: min = 0
        if max == None: max = 0

        return min + random.random()* (max-min)

    def size(self):
        return len(self.points)

    def grow(self):
        pts = [];
        for x in range(10):
            point = [self.rand(self.minX,self.maxX),self.rand(self.minY, self.maxY),self.points[0][2]+20]
            while True != self.point_inside_polygon(point[0],point[1],self.boundingPolygon):
                point = [self.rand(self.minX,self.maxX),self.rand(self.minY, self.maxY),self.points[0][2]+20]
            pts.append(point)
        return Gene(pts,self.boundingPolygon)

    def render(self, offsetX, offsetY, offsetZ):
        for i in range(len(self.points)) :
            tempP =  [self.points[i][0] + offsetX, self.points[i][1] + offsetY, self.points[i][2] + offsetZ]
            #rs.AddPoint(tempP)

def renderFile(f, creature, offsetX, offsetY, offsetZ):
    oldPoints = False
    for chromosome in creature.chromosomeList:
        for y in range(len(chromosome.genes)):

            #fileWrite(f, "rs.AddSrfPt([")
            i=0
            newPoints = []
            for point in chromosome.genes[y].points:
                p1 = str(float(point[0])+float(offsetX))
                p2 = str(float(point[1])+float(offsetY))
                p3 = str(float(point[2])+float(offsetZ))
                newPoints.append([p1,p2,p3])

            newPoints = reOrder(newPoints)

            fileWrite(f, "curve"+str(y)+" = rs.AddPolyline(["+doPolygon(newPoints)+"])")
            #extrude
            # fileWrite(f, "rs.ExtrudeCurveStraight( curve, ["+str(newPoints[0][0])+","+str(newPoints[0][1])+","+str(newPoints[0][2])+
            #              "],  ["+str(newPoints[0][0])+","+str(newPoints[0][1])+","+str(float(newPoints[0][2])+20)+"])")
            if oldPoints != False:
                fileWrite(f, "rs.AddLoftSrf( [curve"+str(y)+",curve"+str(y-1)+"])")

            oldPoints = newPoints

def reOrder(p):
    # center
    xs = 0;
    ys = 0;
    for p1 in p:
        ys = ys + float(p1[1])
        xs = xs + float(p1[0])
    center = [
        xs / len(p),
        ys / len(p)
    ]
    rad = []
    for p1 in p:
        val = [
            p1,
            math.atan2(float(p1[0])-center[0], float(p1[1])-center[1])
        ]
        rad.append(val)
    # print(rad)
    sort = sorted(rad, key=lambda x: x[1])

    rt = []
    for p1 in sort:
        rt.append(p1[0])

    return rt

def doPolygon(points):

    l = len(points)-1
    if points[0][0]!= points[l][0] or points[0][1]!=points[l][1]:
        points.append(points[0])
    return (', '.join([str(x) for x in points]))
    # return "rs.AddPolyline (["+p1+","+p2+","+p3+"])")

class fitnessAlgo:
    def __init__(self):
        return

    def score(self, creature):
        num = 0
        offset = 0
        oldPoint =[]
        for chromo in creature.chromosomeList:
            for line in range(len(chromo.genes[0].points)):
                for story in range(len(chromo.genes)):
                    num = num + 1
                    point = chromo.genes[story].points[line]
                    if len(oldPoint)!=0:
                        offset = offset + math.sqrt(abs(point[0]--oldPoint[0])**2 + abs(point[1]--oldPoint[1])**2)
                    oldPoint = point
        avg = offset/num/len(chromo.genes)
        score = 100 - abs(3-avg)
        # print(score)
        return score

def fileWrite(f,txt):
    f.seek(0,2)
    f.write(txt+"\n")

class reproduction_divider:
    def __init__(self):
        return

    def mitosis(self, creature1, creature2):
        chromo = []

        #cut this in half
        for i in range(len(creature1.chromosomeList)):
            genes = []
            for j in range(len(creature1.chromosomeList[i].genes)):
                if random.randint(0,1):
                    genes.append(creature1.chromosomeList[i].genes[j])
                else:
                    genes.append(creature2.chromosomeList[i].genes[j])
            chromo.append(chromosome.Chromosome( genes))
        return creature.Creature(chromo)

def main():
    #rs.EnableRes = draw(False)
    creatures = []
    boundingPolygon = [
        [0,0],
        [70,5],
        [75,60],
        [80,30],
        [10,50],
        [0,0]
    ]

    # generate creatures
    for i in range(1000):
        firstGene = Gene(None,boundingPolygon)
        crmsome = chromosome.Chromosome([firstGene])
        crmsome.grow(10)
        beast = creature.Creature([crmsome])
        creatures.append(beast)

    # initialize evolution engine
    fitnessAlgorithms = {
        '1': fitnessAlgo()    }
    #
    reproductionAlgorithm = reproduction_divider()
    #
    engine = evolution.Evolution(creatures)
    engine.population = engine.fitness(fitnessAlgorithms, 100)
    #
    for iteration in range(10):
        print ("Begin "+str(len( engine.population))+" creatures")
        engine.population = creatures + engine.reproduce(reproductionAlgorithm)
        print ("Reproduction "+str(len(engine.population))+" creatures")
        engine.population = engine.fitness(fitnessAlgorithms, 100)
        print ("Fitness "+str(len( engine.population))+" creatures")

    engine.population = engine.fitness(fitnessAlgorithms, 1)
    #render the best creatures
    i = 0
    j = 0
    f = open('generated/workfile'+str(time.time())+'.py', 'w')
    fileWrite(f,"import rhinoscriptsyntax as rs")
    fileWrite(f, "rs.EnableRedraw(False)")
    for beast in engine.population:
        # print(beast.fitnessScore)
        # myRender(beast,i * 500, j * 500)
        renderFile(f, beast, i * 500, j * 500, 0)
        i += 1
        if not (i % 5): j += 1; i = 0

        #rs.EnableRedraw(True)
    fileWrite(f, "rs.EnableRedraw(True)")
    f.close()
main()


# -*- coding: utf-8 -*-
"""
Created on Sat Dec  7 17:33:35 2019

@author: genev
"""

def heatmapdraw(jobs, search_job):
    '''
   This function is used to visualize the data of salary and job location in format of choropleth map
    Input arguments: cleaned data of job information collected from Glassdoor.
    This fuction plots 2 Heat Maps.
    THe first one is a heatmap for job number distribution.
    The second one is a heatmap for average salary distribution.

    Parameters
    ----------
    jobs : TYPE
        DESCRIPTION.
    search_job : TYPE
        DESCRIPTION.

    Returns
    -------
    list
        DESCRIPTION.

    '''
    import pandas as pd
    from collections import Counter
    z = jobs['State']
    a = Counter(z)
    df = pd.DataFrame.from_dict(a, orient='index').reset_index()
    df = df[df['index'].str.len() <= 2]
    import plotly.graph_objects as go

    # Load data frame and tidy it.

    figstate = go.Figure(data=go.Choropleth(
        locations=df['index'], # Spatial coordinates
        z = df[0].astype(float), # Data to be color-coded
        locationmode = 'USA-states', # set of locations match entries in `locations`
        colorscale = 'Blues',
        colorbar_title = "# of jobs",
    ))

    figstate.update_layout(
        title_text = '# of '+search_job+' jobs in USA by State',
        geo_scope='usa', # limite map scope to USA
    )

    figstate.show()

    salstate = jobs[['Average Sal','State']]

    sumavesal = pd.DataFrame()
    sumavesal['Sum'] = salstate.groupby(['State']).sum()['Average Sal']
    sumavesal['Count'] = salstate.groupby(['State']).count()['Average Sal']
    sumavesal['Result'] = sumavesal['Sum']/sumavesal['Count']

    figsal = go.Figure(data=go.Choropleth(
        locations=sumavesal.index, # Spatial coordinates
        z = sumavesal['Result'].astype(float), # Data to be color-coded
        locationmode = 'USA-states', # set of locations match entries in `locations`
        colorscale = 'Blues',
        colorbar_title = "K dollars",
    ))

    figsal.update_layout(
        title_text = 'Average salaries of '+search_job+' jobs in USA by State',
        geo_scope='usa', # limit map scope to USA
    )

    figsal.show()
    return [figstate, figsal]

'''
Function:This funtion generates 3 plots.
The first is a histogram of salary distribution in the nationwide.
The second is a barchart showing the top 5 job positions salary distribution
The third is a horizonal barchart presenting top 10 job
Input: cleaned job data from Glassdoor in pandas dataframe
Output: One histrogram and two bar charts
'''
def barchartdata(jobs):

    import matplotlib.pyplot as plt
    import pandas as pd
    import seaborn as sns
    from io import BytesIO
    import base64
    import numpy as np

    salDf = jobs
    df = salDf[pd.notnull(salDf['Average Sal'])]   # remove NaN

    # histogram salary distribution

    # summarize mean,median, 75% and 25% quantiles
    m = df['Average Sal'].mean()   # mean salary for all positions
    me = df['Average Sal'].median()
    q1 = df['Average Sal'].quantile(0.25)
    q2 = df['Average Sal'].quantile(0.75)
    sns.set_style("dark")
    salDist = sns.distplot(df['Average Sal'])
    salDist.axvline(me, lw=1.5, ls='dashed', color='black')
    salDist.axvline(m, lw=1.5, ls='dashed', color='red')
    salDist.axvline(q1, lw=1.5, ls='dashed', color='green')
    salDist.axvline(q2, lw=1.5, ls='dashed', color='green')
    plt.xlabel('Average Salary')
    plt.title('Salary($k) Distribution with Median (black), Mean (red) and Quantiles (green)')

    # save the plot
    buffer = BytesIO()
    plt.savefig(buffer)
    plot_data = buffer.getvalue()

    imb = base64.b64encode(plot_data)
    ims = imb.decode()
    imd = "data:image/png;base64,"+ims
    salbar_im = """""" % imd

    # plot the average salary for top 5 positions
    df['jobTitle'] = df['jobTitle'].str.replace('Sr.','Senior')
    df['jobTitle'] = df['jobTitle'].str.replace('Sr','Senior')
    jobCnt = df['jobTitle'].value_counts()
    top5Cnt = jobCnt[0:5].to_dict()

    # average the salaries by job title
    top5Sal = {}
    for job in list(top5Cnt.keys()):
        row = df[df['jobTitle'].str.match(job)]
        top5Sal[job] = row['Average Sal'].mean()

    job = list(top5Sal.keys())
    sal = list(top5Sal.values())

    plt.figure(figsize=(10, 6))
    plt.bar(job, sal)
    plt.xlabel('Job Title')
    plt.ylabel('Average Salary ($k)')
    plt.axhline(y = me, ls='dashed', color="black")
    plt.axhline(y = m, ls='dashed', color="red")

    plt.title('Average Salary for Top 5 Job Positions with Median (black) and Mean (red)')
    plt.tight_layout()  # show conplete labels in the saved image
    buffer = BytesIO()
    plt.savefig(buffer)
    plot_data = buffer.getvalue()

    # save the plot
    imb = base64.b64encode(plot_data)
    ims = imb.decode()
    imd = "data:image/png;base64,"+ims
    salpobar_im = """""" % imd

    # plot the counts of top 10 job titles

    data = jobs
    title_count=data['jobTitle'].value_counts()

    top=10
    title_count1=title_count[:top]
    titles=list(title_count1.index)
    counts=list(title_count1.values)

    fig1, ax1 = plt.subplots(dpi = 1200)
    y_pos = np.arange(len(titles))

    ax1.barh(y_pos,counts)
    ax1.set_yticks(y_pos)
    ax1.set_yticklabels(titles)
    ax1.invert_yaxis()  # labels read top-to-bottom
    title_String='Number ('+str(len(data['jobTitle']))+' Job Posts in Total)'
    ax1.set_xlabel(title_String)
    for i, v in enumerate(counts):
        ax1.text(v, i, str(v),fontsize=9)
    ax1.set_title('Top 10 Job Titles',fontsize=18)

    # save the plot
    buffer = BytesIO()
    plt.savefig(buffer)
    plot_data = buffer.getvalue()

    imb = base64.b64encode(plot_data)
    ims = imb.decode()
    imd = "data:image/png;base64,"+ims
    pobar_im = """""" % imd
    fig1.savefig('pobar_im.png', bbox_inches="tight")

    return [salbar_im,salpobar_im,pobar_im]

'''
Function: calculate the words frequency in job description and generate a wordCloud
Input: a list of job description
Output: the visualization of wordCloud
'''
def wordclouddata(indeed_jobs):
    import matplotlib.pyplot as plt
    from io import BytesIO
    import base64
    from wordcloud import WordCloud
    import nltk
    from nltk.corpus import stopwords
    from sklearn.feature_extraction.text import TfidfVectorizer
    plt.clf()
    data = indeed_jobs.drop_duplicates().reset_index(drop=True)

    # Change the dataframe to a string list
    jobDescription = []
    for i in range(len(data)):
        jobDescription.append(data['job summary'][i])

    # Use nltk to get the english stop words and add some new words to it
    nltk.download('stopwords')
    stop_words = stopwords.words('english')
    extra_stop_words = ["experience", "position", "work", "please", "click", "must", "may", "required", "preferred",
                        "type", "including", "strong", "ability", "needs", "apply", "skills", "requirements", "company",
                        "knowledge", "job", "responsibilities", "develop", "solution", "support", "build", "new",
                        "development", "years", "solutions", "etc", "using", "management", "environment", "time",
                        "related", "help", "working", "analysis", "tools", "technologies", "design", "scientist",
                        "research", "information", "techniques", "degree", "models", "status", "us", "people",
                        "opportunity", "role", "understanding", "advanced", "use", "well", "insights", "provide",
                        "products", "complex", "based", "across", "plus", "problems", "teams", "employees",
                        "employers", "one", "employer", "employee", "within", "improve", "industry", "reporting",
                        "field", "excellent", "best", "quality", "e", "impact", "g", "key", "building", "methods",
                        "looking", "deep", "developing", "create", "high", "multiple", "make", "benefits", "care",
                        'statistical', 'deliver', 'non', 'focus', 'maintenance', 'sets', 'manage', 'various']
    stop_words += extra_stop_words
    print("Generating Word Cloud...")

    # Build the parameter for calculating word frequency
    tfidf_para = {
        "stop_words": stop_words,
        "analyzer": 'word',   # analyzer in 'word' or 'character'
        "token_pattern": r'\w{1,}',    # match any word with 1 and unlimited length
        "sublinear_tf": False,  # False for smaller data size  #Apply sublinear tf scaling, to reduce the range of tf with 1 + log(tf)
        "dtype": int,   # return data type
        "norm": 'l2',     # apply l2 normalization
        "smooth_idf":False,   # no need to one to document frequencies to avoid zero divisions
        "ngram_range" : (1, 2),   # the min and max size of tokenized terms
        "max_features": 60    # the top 60 weighted features
    }
    tfidf_vect = TfidfVectorizer(**tfidf_para)
    transformed_job_desc = tfidf_vect.fit_transform(jobDescription)
    freqs_dict = dict([(word, transformed_job_desc.getcol(idx).sum()) for word, idx in tfidf_vect.vocabulary_.items()])

    # Use the word frequency to generate a wordCloud object with top 60 words
    wordcloud = WordCloud(
        background_color='white',
        width=1500,
        height=960,
        margin=10,
        mode='RGBA',
        max_words=60).fit_words(freqs_dict)

    # Use matplotlib to show and save the image
    plt.imshow(wordcloud)
    plt.axis("off")
    wordcloud.to_file('my_test1.png')

    buffer = BytesIO()
    plt.savefig(buffer)
    plot_data = buffer.getvalue()
    imb = base64.b64encode(plot_data)
    ims = imb.decode()
    imd = "data:image/png;base64,"+ims
    text_im = """""" % imd
    return text_im