import json
import os
import copy
import math

#将数字转化为文字，取消小数点和无用末尾0
def better_float_to_text(x):
    if isinstance(x,float) and int(x)==x:
        return str(int(x))
    else:
        return str(x)

# 查询数据范围
def findtb(source, start, end, tb, brace=0):
    i = j = 0
    tb[0] = start
    tb[1] = end
    for i in range(start, end):
        if source[i] == '{':
            brace += 1
            if brace > 0:
                tb[0] = i + 1
                for j in range(i + 1, tb[1]):
                    if source[j] == '{':
                        brace += 1
                    elif source[j] == '}':
                        brace -= 1
                        if brace == 0:
                            tb[1] = j - 1
                            return True
        elif source[i] == '}':
            return False
    return False

def findabilityname(source, tb):
    i = source.rfind('\"', 0, tb[0])
    j = source.rfind('\"', 0, i - 1)
    return source[j + 1:i]

# 寻找属性（查询源字符串，数据存储位置，上下限，属性序数，是否为存在属性）
def findabilitypro(source, data, tb, pro, bool=False):
    if bool:
        i = source.find(pro[1], tb[0], tb[1])
        if i != -1:
            data[pro[1]] = {"1": 1}
    else:
        i = source.find('\"' + pro[1] + '\"', tb[0], tb[1])
        if i != -1:
            j = source.find('\"', i + 1, tb[1])
            j = source.find('\"', j + 1, tb[1])
            k = source.find('\"', j + 1, tb[1])
            if k > j + 1:
                splitit = source[j + 1:k].split(' ')
                data[pro[1]] = {}
                bool = True
                for j in range(len(splitit)):
                    data[pro[1]][str(j + 1)] = float(splitit[j])
                    bool = bool and splitit[0] == splitit[j]
                if bool:
                    data[pro[1]] = {"1": data[pro[1]]["1"]}
    return

def findabilityspecial(source, data, tb):
    i = source.find("AbilitySpecial", tb[0], tb[1])
    if i == -1:
        return
    else:
        k = source.find("{", i, tb[1])
        while True:
            j = source.find("{", k + 1, tb[1])
            k = source.find("}", k + 1, tb[1])
            if j < k and j != -1:
                r = [0, 0, 0, 0]
                r[0] = source.find('\"', j, k) + 1
                r[0] = source.find('\"', r[0], k) + 1
                r[0] = source.find('\"', r[0], k) + 1
                r[0] = source.find('\"', r[0], k) + 1
                r[0] = source.find('\"', r[0], k) + 1
                r[1] = source.find('\"', r[0], k)
                r[2] = source.find('\"', r[1] + 1, k) + 1
                r[3] = source.find('\"', r[2], k)
                splitit = source[r[2]:r[3]].split(' ')
                if splitit[0] != '':
                    data[source[r[0]:r[1]]] = {}
                    bool = True
                    for l in range(len(splitit)):
                        data[source[r[0]:r[1]]][str(l + 1)] = float(splitit[l])
                        bool = bool and splitit[0] == splitit[l]
                    if bool:
                        data[source[r[0]:r[1]]] = {"1": data[source[r[0]:r[1]]]["1"]}
            else:
                return

def get_hero_data_from_txt(base_txt, address):
    this_file = open("E:/Steam/steamapps/common/dota 2 beta/game/dota/scripts/npc/npc_abilities.txt", mode="r")
    this_string = this_file.read()
    tb = [0, 0]
    findtb(this_string, 0, len(this_string), tb, -1)
    while (True):
        if (findtb(this_string, tb[1] + 2, len(this_string), tb, 0)):
            name = findabilityname(this_string, tb)
            base_txt[name] = {}
            for i in abilitypro_num:
                findabilitypro(this_string, base_txt[name], tb, i, False)
            for i in abilitypro_bool:
                findabilitypro(this_string, base_txt[name], tb, i, True)
            findabilityspecial(this_string, base_txt[name], tb)
        else:
            break

def get_source_to_data(all_json, upgrade_json, version):
    for ijk in all_json['技能']:
        unit_dic = copy.deepcopy(all_json['技能'][ijk])
        unit_dic["分类"] = "技能"
        unit_dic["版本"] = version
        unit_dic["应用"] = 1
        if unit_dic["技能排序"] == "":
            if unit_dic["默认按键"] == "Q":
                unit_dic["技能排序"] = "d"
            elif unit_dic["默认按键"] == "W":
                unit_dic["技能排序"] = "e"
            elif unit_dic["默认按键"] == "E":
                unit_dic["技能排序"] = "f"
            elif unit_dic["默认按键"] == "D":
                unit_dic["技能排序"] = "g"
            elif unit_dic["默认按键"] == "F":
                unit_dic["技能排序"] = "h"
            elif unit_dic["默认按键"] == "R":
                unit_dic["技能排序"] = "i"
            elif unit_dic["默认按键"] == "T":
                unit_dic["技能排序"] = "j"
            else:
                unit_dic["技能排序"] = "z"
        temp1 = copy.deepcopy(all_json['技能源'][unit_dic["数据来源"]])
        if unit_dic["次级分类"] == "天赋技能":
            unit_dic["图片"] = all_json["英雄"][unit_dic["技能归属"]]["图片"]
            unit_dic["迷你图片"] = all_json["英雄"][unit_dic["技能归属"]]["迷你图片"]
        elif unit_dic["次级分类"] == "物品技能":
            if unit_dic["技能归属"] in all_json["物品"]:
                unit_dic["图片"] = all_json["物品"][unit_dic["技能归属"]]["图片"]
                unit_dic["迷你图片"] = all_json["物品"][unit_dic["技能归属"]]["迷你图片"]
            else:
                unit_dic["图片"] = "items_" + temp1["代码"] + ".png"
                unit_dic["迷你图片"] = "items_" + temp1["代码"] + ".png"
        else:
            unit_dic["图片"] = "spellicons " + temp1["代码"] + ".png"
            unit_dic["迷你图片"] = "spellicons " + temp1["代码"] + ".png"
        for i in temp1:
            if i in ability_trait_level[0]:
                unit_dic[i] = temp1[i]
            elif i in ability_trait_level[1]:
                unit_dic[i] = {}
                j = 0
                while True:
                    j += 1
                    if str(j) in temp1[i]:
                        unit_dic[i][str(j)] = group_source(temp1[i][str(j)])
                    else:
                        break
            elif i in ability_trait_level[2]:
                unit_dic[i] = {}
                j = 0
                while True:
                    j += 1
                    if str(j) in temp1[i]:
                        unit_dic[i][str(j)] = {}
                        k = 0
                        while True:
                            k += 1
                            if str(k) in temp1[i][str(j)]:
                                unit_dic[i][str(j)][str(k)] = group_source(temp1[i][str(j)][str(k)])
                            else:
                                break
                        if '名称' in temp1[i][str(j)]:
                            unit_dic[i][str(j)]['名称'] = temp1[i][str(j)]['名称']
                    else:
                        break
            elif i == "升级":
                if "A杖" in temp1[i] and len(temp1[i]["A杖"]) > 0 or "技能" in temp1[i] and len(temp1[i]["技能"]) > 0 or "混合" in temp1[i] and len(temp1[i]["混合"]) > 0:
                    upgrade_json[unit_dic["页面名"]] = copy.deepcopy(temp1[i])
            elif i == "页面名":
                continue
            else:
                unit_dic[i] = group_source(temp1[i])
        unit_dic["技能升级信息"]={}
        all_json["技能"][unit_dic["页面名"]] = copy.deepcopy(unit_dic)

def group_source(a):
    temp = {"1": {}}
    for i in a:
        if i == "名称" or i == "后缀":
            temp[i] = a[i]
        else:
            temp["1"][i] = a[i]
    return temp

def input_upgrade(all_json, upgrade_json):
    for i in upgrade_json:
        for j in upgrade_json[i]["A杖"]:
            if upgrade_json[i]["A杖"][j]["目标"]["1"] == "":
                upgrade_json[i]["A杖"][j]["目标"]["1"] = "技能"
            if upgrade_json[i]["A杖"][j]["目标"]["2"] == "":
                upgrade_json[i]["A杖"][j]["目标"]["2"] = i
            if "代码" in upgrade_json[i]["A杖"][j]["值"] and isinstance(upgrade_json[i]["A杖"][j]["值"]["代码"], dict):
                if "1" in upgrade_json[i]["A杖"][j]["值"]["代码"] and upgrade_json[i]["A杖"][j]["值"]["代码"]["1"] == "":
                    upgrade_json[i]["A杖"][j]["值"]["代码"]["1"] = "技能"
                if "2" in upgrade_json[i]["A杖"][j]["值"]["代码"] and upgrade_json[i]["A杖"][j]["值"]["代码"]["2"] == "":
                    upgrade_json[i]["A杖"][j]["值"]["代码"]["2"] = all_json["技能"][i]["代码"]
            k = 1
            temp = all_json[upgrade_json[i]["A杖"][j]["目标"]["1"]]
            while True:
                k += 1
                if str(k) in upgrade_json[i]["A杖"][j]["目标"]:
                    if upgrade_json[i]["A杖"][j]["目标"][str(k)] in temp:
                        temp = temp[upgrade_json[i]["A杖"][j]["目标"][str(k)]]
                    else:
                        temp[upgrade_json[i]["A杖"][j]["目标"][str(k)]] = {}
                        temp = temp[upgrade_json[i]["A杖"][j]["目标"][str(k)]]
                else:
                    break
            if "0" in upgrade_json[i]["A杖"][j]["目标"] and upgrade_json[i]["A杖"][j]["目标"]["0"] == "替换":
                if "2" not in temp:
                    temp["2"] = copy.deepcopy(temp["1"])
                for k in upgrade_json[i]["A杖"][j]["值"]:
                    temp["2"][k] = copy.deepcopy(upgrade_json[i]["A杖"][j]["值"][k])
            else:
                temp["2"] = upgrade_json[i]["A杖"][j]["值"]
        for j in upgrade_json[i]["技能"]:
            if "代码" in upgrade_json[i]["技能"][j]["值"] and isinstance(upgrade_json[i]["技能"][j]["值"]["代码"], dict):
                if "1" in upgrade_json[i]["技能"][j]["值"]["代码"] and upgrade_json[i]["技能"][j]["值"]["代码"]["1"] == "":
                    upgrade_json[i]["技能"][j]["值"]["代码"]["1"] = "技能"
                if "2" in upgrade_json[i]["技能"][j]["值"]["代码"] and upgrade_json[i]["技能"][j]["值"]["代码"]["2"] == "":
                    upgrade_json[i]["技能"][j]["值"]["代码"]["2"] = all_json["技能"][i]["代码"]
            k = 1
            temp = all_json[upgrade_json[i]["技能"][j]["目标"]["1"]]
            while True:
                k += 1
                if str(k) in upgrade_json[i]["技能"][j]["目标"]:
                    if upgrade_json[i]["技能"][j]["目标"][str(k)] in temp:
                        temp = temp[upgrade_json[i]["技能"][j]["目标"][str(k)]]
                    else:
                        temp[upgrade_json[i]["技能"][j]["目标"][str(k)]] = {}
                        temp = temp[upgrade_json[i]["技能"][j]["目标"][str(k)]]
                else:
                    break
            if "0" in upgrade_json[i]["技能"][j]["目标"] and upgrade_json[i]["技能"][j]["目标"]["0"] == "替换":
                if "3" not in temp:
                    temp["3"] = copy.deepcopy(temp["1"])
                for k in upgrade_json[i]["技能"][j]["值"]:
                    temp["3"][k] = copy.deepcopy(upgrade_json[i]["技能"][j]["值"][k])
            else:
                temp["3"] = upgrade_json[i]["技能"][j]["值"]
            temp["3"]["升级来源"] = {"1": {"名称": i}}
            if all_json["技能"][i]["次级分类"] == "天赋技能":
                temp["3"]["升级来源"]["1"]["图片"] = "talent.png"
            else:
                temp["3"]["升级来源"]["1"]["图片"] = all_json["技能"][i]["迷你图片"]
            if upgrade_json[i]["技能"][j]["目标"]["1"]=='技能':
                upgrade_info = all_json['技能'][upgrade_json[i]["技能"][j]["目标"]["2"]]['技能升级信息']
                upbool = True
                for upi in upgrade_info:
                    if upgrade_info[upi]['技能名'] == i:
                        upbool = False
                if upbool:
                    upnum = len(upgrade_info)
                    upgrade_info[str(upnum + 1)] = {'技能名': i}
        for j in upgrade_json[i]["混合"]:
            if "代码" in upgrade_json[i]["混合"][j]["值"]:
                if "1" in upgrade_json[i]["混合"][j]["值"]["代码"] and upgrade_json[i]["混合"][j]["值"]["代码"]["1"] == "":
                    upgrade_json[i]["混合"][j]["值"]["代码"]["1"] = "技能"
                if "2" in upgrade_json[i]["混合"][j]["值"]["代码"] and upgrade_json[i]["混合"][j]["值"]["代码"]["2"] == "":
                    upgrade_json[i]["混合"][j]["值"]["代码"]["2"] = all_json["技能"][i]["代码"]
            k = 1
            temp = all_json[upgrade_json[i]["混合"][j]["目标"]["1"]]
            while True:
                k += 1
                if str(k) in upgrade_json[i]["混合"][j]["目标"]:
                    if upgrade_json[i]["混合"][j]["目标"][str(k)] in temp:
                        temp = temp[upgrade_json[i]["混合"][j]["目标"][str(k)]]
                    else:
                        temp[upgrade_json[i]["混合"][j]["目标"][str(k)]] = {}
                        temp = temp[upgrade_json[i]["混合"][j]["目标"][str(k)]]
                else:
                    break
            if "0" in upgrade_json[i]["混合"][j]["目标"] and upgrade_json[i]["混合"][j]["目标"]["0"] == "替换":
                if "3" not in temp:
                    temp["3"] = copy.deepcopy(temp["1"])
                for k in upgrade_json[i]["混合"][j]["值"]:
                    temp["3"][k] = copy.deepcopy(upgrade_json[i]["混合"][j]["值"][k])
            else:
                temp["3"] = upgrade_json[i]["混合"][j]["值"]
            temp["3"]["升级来源"] = {}
            for k in upgrade_json[i]["混合"][j]["来源技能"]:
                if upgrade_json[i]["混合"][j]["来源技能"][k] == "A杖":
                    temp["3"]["升级来源"][k] = {"名称": "阿哈利姆神杖", "图片": "agha.png"}
                elif upgrade_json[i]["混合"][j]["来源技能"][k] == "本技能":
                    temp["3"]["升级来源"][k] = {"名称": i}
                    if all_json["技能"][i]["次级分类"] == "天赋技能":
                        temp["3"]["升级来源"][k]["图片"] = "talent.png"
                    else:
                        temp["3"]["升级来源"][k]["图片"] = all_json["技能"][i]["迷你图片"]
                else:
                    temp["3"]["升级来源"][k] = {"名称": upgrade_json[i]["混合"][j]["来源技能"][k]}
                    if all_json["技能"][upgrade_json[i]["混合"][j]["来源技能"][k]]["次级分类"] == "天赋技能":
                        temp["3"]["升级来源"][k]["图片"] = "talent.png"
                    else:
                        temp["3"]["升级来源"][k]["图片"] = all_json["技能"][upgrade_json[i]["混合"][j]["来源技能"][k]]["迷你图片"]
            if upgrade_json[i]["技能"][j]["目标"]["1"]=='技能':
                upgrade_info = all_json['技能'][upgrade_json[i]["技能"][j]["目标"]["2"]]['技能升级信息']
                upbool = True
                for upi in upgrade_info:
                    if upgrade_info[upi]['技能名'] == i:
                        upbool = False
                if upbool:
                    upnum = len(upgrade_info)
                    upgrade_info[str(upnum + 1)] = {'技能名': i}
    for i in all_json["技能"]:
        for j in all_json["技能"][i]["效果"]:
            for k in ["2", "3"]:
                if k in all_json["技能"][i]["效果"][j]:
                    if '名称' in all_json["技能"][i]["效果"][j][k]:
                        all_json["技能"][i]["效果"][j]['名称'] = all_json["技能"][i]["效果"][j][k]['名称']
                        all_json["技能"][i]["效果"][j][k].pop('名称')
                    l = 0
                    while True:
                        l += 1
                        if str(l) in all_json["技能"][i]["效果"][j][k] and all_json["技能"][i]["效果"][j][k][str(l)] == '删除':
                            m = l
                            while True:
                                m = m + 1
                                if str(m) in all_json["技能"][i]["效果"][j][k]:
                                    all_json["技能"][i]["效果"][j][k][str(m - 1)] = copy.deepcopy(all_json["技能"][i]["效果"][j][k][str(m)])
                                else:
                                    all_json["技能"][i]["效果"][j][k].pop(str(m - 1))
                                    l -= 1
                                    break
                        else:
                            break
                    m = 0
                    l -= 1
                    while True:
                        m += 1
                        l += 1
                        if "+" + str(m) in all_json["技能"][i]["效果"][j][k]:
                            all_json["技能"][i]["效果"][j][k][str(l)] = copy.deepcopy(all_json["技能"][i]["效果"][j][k]["+" + str(m)])
                            all_json["技能"][i]["效果"][j][k].pop("+" + str(m))
                        else:
                            break

def complete_upgrade(all_json, base_txt):
    for i in all_json:
        for j in all_json[i]["属性"]:
            fulfil(all_json[i]["属性"][j]["1"]["代码"], all_json[i])
            one_upgrade(all_json[i]["属性"][j], base_txt)
        for j in all_json[i]["冷却时间"]:
            if all_json[i]["冷却时间"][j]["1"]["代码"]["1"] == '不存在':
                all_json[i]["冷却时间"][j]["1"]["代码"] = {"1": "不存在"}
            else:
                fulfil(all_json[i]["冷却时间"][j]["1"]["代码"], all_json[i])
                one_upgrade(all_json[i]["冷却时间"][j], base_txt)
        for j in all_json[i]["魔法消耗"]:
            for k in all_json[i]["魔法消耗"][j]:
                if k != '名称':
                    if all_json[i]["魔法消耗"][j][k]["1"]["代码"]["1"] == '不存在':
                        all_json[i]["魔法消耗"][j][k]["1"]["代码"] = {"1": "不存在"}
                    else:
                        fulfil(all_json[i]["魔法消耗"][j][k]["1"]["代码"], all_json[i])
                        one_upgrade(all_json[i]["魔法消耗"][j][k], base_txt)
        if all_json[i]["施法前摇"]["1"]["代码"]["1"] == '不存在':
            all_json[i]["施法前摇"]["1"]["代码"] = {"1": "不存在"}
        else:
            fulfil(all_json[i]["施法前摇"]["1"]["代码"], all_json[i])
            one_upgrade(all_json[i]["施法前摇"], base_txt)
        if all_json[i]["施法后摇"]["1"]["代码"]["1"] == '不存在':
            all_json[i]["施法后摇"]["1"]["代码"] = {"1": "不存在"}
        else:
            fulfil(all_json[i]["施法后摇"]["1"]["代码"], all_json[i])
            one_upgrade(all_json[i]["施法后摇"], base_txt)

def fulfil(arr, json):
    if '0' in arr and arr['0'] == '手填':
        return
    else:
        if "1" in arr and arr["1"] == "":
            if json['次级分类'] == '物品技能':
                arr["1"] = "物品"
            else:
                arr["1"] = "技能"
        if "2" in arr and arr["2"] == "":
            arr["2"] = json["代码"]

def one_upgrade(json, base_txt):
    inbool = ["2" in json, "3" in json]
    inbool.append(inbool[0] and inbool[1])
    getvalue = [[], [], [], []]
    calvalue = [[], [], [], []]
    caloprate = [[], [], []]
    if "0" in json["1"]["代码"] and json["1"]["代码"]['0'] != '':
        if json["1"]["代码"]["0"] == "手填":
            k = 0
            while True:
                k += 1
                if str(k) in json["1"]["代码"] and json["1"]["代码"][str(k)] != '':
                    try:
                        getvalue[0].append(float(json["1"]["代码"][str(k)]))
                    except ValueError:
                        getvalue[0].append(json["1"]["代码"][str(k)])
                else:
                    break
        elif json["1"]["代码"]["0"] == "不存在":
            return
    else:
        for k in base_txt[json["1"]["代码"]["1"]][json["1"]["代码"]["2"]][json["1"]["代码"]["3"]]:
            getvalue[0].append(base_txt[json["1"]["代码"]["1"]][json["1"]["代码"]["2"]][json["1"]["代码"]["3"]][k])
    if json["1"]["修正"]["1"] == "":
        inbool.append(False)
    else:
        inbool.append(True)
        caloprate[0].append(json["1"]["修正"]["1"])
        k = 1
        while True:
            k += 1
            if str(k) in json["1"]["修正"]:
                getvalue[1].append(json["1"]["修正"][str(k)])
            else:
                break
    if inbool[0]:
        if "0" in json["2"]["代码"] and json["2"]["代码"]["0"] == "手填":
            k = 0
            while True:
                k += 1
                if str(k) in json["2"]["代码"]:
                    try:
                        getvalue[2].append(float(json["2"]["代码"][str(k)]))
                    except ValueError:
                        getvalue[2].append(json["2"]["代码"][str(k)])
                else:
                    break
        else:
            for k in base_txt[json["2"]["代码"]["1"]][json["2"]["代码"]["2"]][json["2"]["代码"]["3"]]:
                getvalue[2].append(base_txt[json["2"]["代码"]["1"]][json["2"]["代码"]["2"]][json["2"]["代码"]["3"]][k])
        caloprate[1].append(json["2"]["修正"]["1"])
    if inbool[1]:
        if "0" in json["3"]["代码"] and json["3"]["代码"]["0"] == "手填":
            k = 0
            while True:
                k += 1
                if str(k) in json["3"]["代码"]:
                    try:
                        getvalue[3].append(float(json["3"]["代码"][str(k)]))
                    except ValueError:
                        getvalue[3].append(json["3"]["代码"][str(k)])
                else:
                    break
        else:
            for k in base_txt[json["3"]["代码"]["1"]][json["3"]["代码"]["2"]][json["3"]["代码"]["3"]]:
                getvalue[3].append(base_txt[json["3"]["代码"]["1"]][json["3"]["代码"]["2"]][json["3"]["代码"]["3"]][k])
        caloprate[2].append(json["3"]["修正"]["1"])
    maxlen=0
    for i in range(4):
        maxlen=max(maxlen,len(getvalue[i]))
    for i in range(4):
        for j in range(maxlen):
            if 0<len(getvalue[i]) and len(getvalue[i])<=j:
                getvalue[i].append(getvalue[i][j-1])
    calvalue[0] = copy.deepcopy(getvalue[0])
    calvalue[1] = copy.deepcopy(getvalue[0])
    calvalue[2] = copy.deepcopy(getvalue[0])
    calvalue[3] = copy.deepcopy(getvalue[0])
    if inbool[1]:
        array_cal(calvalue[2], getvalue[3], caloprate[2], -1)
        array_cal(calvalue[3], getvalue[3], caloprate[2], -1)
    if inbool[0]:
        array_cal(calvalue[1], getvalue[2], caloprate[1], 0)
        array_cal(calvalue[3], getvalue[2], caloprate[1], 0)
    if inbool[1]:
        array_cal(calvalue[2], getvalue[3], caloprate[2], 0)
        array_cal(calvalue[3], getvalue[3], caloprate[2], 0)
    if inbool[3]:
        array_cal(calvalue[0], getvalue[1], caloprate[0], 0)
        array_cal(calvalue[1], getvalue[1], caloprate[0], 0)
        array_cal(calvalue[2], getvalue[1], caloprate[0], 0)
        array_cal(calvalue[3], getvalue[1], caloprate[0], 0)
    if inbool[1]:
        array_cal(calvalue[2], getvalue[3], caloprate[2], 1)
        array_cal(calvalue[3], getvalue[3], caloprate[2], 1)
    cut_the_same_to_one(calvalue[0])
    for k in range(len(calvalue[0])):
        json["1"][str(k + 1)] = calvalue[0][k]
    if inbool[0]:
        cut_the_same_to_one(calvalue[1])
        for k in range(len(calvalue[1])):
            json["2"][str(k + 1)] = calvalue[1][k]
    if inbool[1]:
        cut_the_same_to_one(calvalue[2])
        for k in range(len(calvalue[2])):
            json["3"][str(k + 1)] = calvalue[2][k]
    if inbool[2]:
        cut_the_same_to_one(calvalue[3])
        json["4"] = {}
        for k in range(len(calvalue[3])):
            json["4"][str(k + 1)] = calvalue[3][k]
        json["4"]["升级来源"] = copy.deepcopy(json["3"]["升级来源"])

def cut_the_same_to_one(lists):
    bool = True
    i = 1
    delete_number = 0
    while i < len(lists):
        if lists[0] == lists[i]:
            delete_number += 1
            i = i + 1
        else:
            bool = False
            break
    if bool and i > 1:
        for i in range(delete_number):
            lists.pop(1)

def array_cal(arr1, arr2, op, num):
    for i in range(len(arr1)):
        if i < len(arr2):
            try:
                temp = float(arr2[i])
            except ValueError:
                temp = arr2[i]
        else:
            try:
                temp = float(arr2[0])
            except ValueError:
                temp = arr2[0]
        if num == 0:
            opp = '1' + op[0] + '1'
        elif num == -1:
            opp = op[0] + '1'
        elif num == 1:
            opp = '1' + op[0]
        if opp == '1=1':
            arr1[i] = temp
        elif opp == '1+1':
            arr1[i] = arr1[i] + temp
        elif opp == '1-1':
            arr1[i] = arr1[i] - temp
        elif opp == '1\\-1':
            arr1[i] = temp - arr1[i]
        elif opp == '1*1':
            arr1[i] = arr1[i] * temp
        elif opp == '1/1':
            arr1[i] = arr1[i] / temp
        elif opp == '1\\1':
            arr1[i] = temp / arr1[i]
        elif opp == '1%-1':
            arr1[i] = arr1[i] * (100 - temp) / 100
        elif opp == '1%+1':
            arr1[i] = arr1[i] * (100 + temp) / 100
        elif opp == '1-%1':
            arr1[i] = arr1[i] * (1 - temp)

def complete_mech(all_json, mech_json):
    for i in all_json:
        for j in all_json[i]["施法目标"]:
            mech_target(all_json[i]["施法目标"][j], mech_json["目标"])
        for j in all_json[i]["效果"]:
            for k in all_json[i]["效果"][j]:
                if "生效目标" in all_json[i]["效果"][j][k]:
                    mech_target(all_json[i]["效果"][j][k]["生效目标"], mech_json["目标"])
        mech_mech(all_json[i], mech_json["机制"])
        mech_junior(all_json[i], mech_json["次级"])
        mech_sign(all_json[i]["效果"], mech_json["标记"])
        mech_repeat(all_json[i]["效果"], mech_json["叠加"])
        mech_others(all_json[i], mech_json)

def mech_target(json, mech):
    for i in ["不分类", "英雄", "非英雄"]:
        if i in json:
            for j in json[i]:
                json[i][j]["值"] = mech["势力"][str(json[i][j]["代码"])]["名称"]
                json[i][j]["颜色"] = mech["势力"][str(json[i][j]["代码"])]["颜色"]
                k = 0
                while True:
                    k = k + 1
                    if str(k) in json[i][j]:
                        json[i][j][str(k)]["值"] = mech["分类"][str(json[i][j][str(k)]["代码"])]["名称"]
                        l = 0
                        while True:
                            l += 1
                            if str(l) in json[i][j][str(k)]:
                                json[i][j][str(k)][str(l)]["值"] = mech["分类"][str(json[i][j][str(k)]["代码"])][str(json[i][j][str(k)][str(l)]["代码"])]
                            else:
                                break
                    else:
                        break

def mech_mech(json, mech):
    for i in mech:
        if i == "施法类型":
            for j in json["施法类型"]:
                k = 0
                while True:
                    k += 1
                    if str(k) in json["施法类型"][j]:
                        json["施法类型"][j][str(k)]["值"] = mech["施法类型"][str(json["施法类型"][j][str(k)]["代码"])]
                    else:
                        break
        elif i == "效果":
            for j in json[i]:
                for k in json[i][j]:
                    if "代码" in json[i][j][k]:
                        json[i][j][k]["值"] = mech[i][str(json[i][j][k]["代码"])]["名称"]
                        json[i][j][k]["图片"] = mech[i][str(json[i][j][k]["代码"])]["图片"]
        else:
            for j in json[i]:
                if str(json[i][j]["代码"]) in mech[i]:
                    json[i][j]["值"] = mech[i][str(json[i][j]["代码"])]["名称"]
                    json[i][j]["图片"] = mech[i][str(json[i][j]["代码"])]["图片"]
                    if json[i][j]["简述"] == "":
                        json[i][j]["简述"] = mech[i][str(json[i][j]["代码"])]["简述"]

def mech_junior(json, mech):
    for i in mech:
        if i in ["即时攻击", "技能窃取"]:
            for j in json[i]:
                k = 0
                while True:
                    k += 1
                    if str(k) in json[i][j]:
                        if str(json[i][j][str(k)]["代码"]) in mech[i]:
                            json[i][j][str(k)]["值"] = mech[i][str(json[i][j][str(k)]["代码"])]["名称"]
                            if "简述" not in json[i][j][str(k)] or json[i][j][str(k)]["简述"] == "":
                                json[i][j][str(k)]["简述"] = mech[i][str(json[i][j][str(k)]["代码"])]["简述"]
                    else:
                        break
        else:
            for j in json["效果"]:
                for k in json["效果"][j]:
                    if isinstance(json["效果"][j][k], dict) and i in json["效果"][j][k]:
                        if str(json["效果"][j][k][i]["代码"]) in mech[i]:
                            json["效果"][j][k][i]["值"] = mech[i][str(json["效果"][j][k][i]["代码"])]["名称"]
                            if "图片" in mech[i][str(json["效果"][j][k][i]["代码"])]:
                                json["效果"][j][k][i]["图片"] = mech[i][str(json["效果"][j][k][i]["代码"])]["图片"]
                            if "简述" not in json["效果"][j][k][i] or json["效果"][j][k][i]["简述"] == "":
                                json["效果"][j][k][i]["简述"] = mech[i][str(json["效果"][j][k][i]["代码"])]["简述"]

def mech_sign(json, mech):
    for i in json:
        for j in json[i]:
            if "标记" in json[i][j]:
                for k in json[i][j]["标记"]:
                    if json[i][j]["标记"][k]["类型"] in mech and str(json[i][j]["标记"][k]["代码"]) in mech[json[i][j]["标记"][k]["类型"]]:
                        json[i][j]["标记"][k]["值"] = mech[json[i][j]["标记"][k]["类型"]][str(json[i][j]["标记"][k]["代码"])]

def mech_repeat(json, mech):
    for i in json:
        for j in json[i]:
            if isinstance(json[i][j], dict) and "叠加" in json[i][j]:
                for k in json[i][j]["叠加"]:
                    if str(json[i][j]["叠加"][k]["代码1"]) in mech["来源"] and str(json[i][j]["叠加"][k]["代码2"]) in mech["方式"]:
                        json[i][j]["叠加"][k]["来源"] = mech["来源"][str(json[i][j]["叠加"][k]["代码1"])]
                        json[i][j]["叠加"][k]["方式"] = mech["方式"][str(json[i][j]["叠加"][k]["代码2"])]

def mech_others(json, mech):
    for i in json["魔法消耗"]:
        for j in json["魔法消耗"][i]:
            if j != '名称':
                for k in json["魔法消耗"][i][j]:
                    if "类型" in json["魔法消耗"][i][j][k]:
                        json["魔法消耗"][i][j][k]["类型"]["值"] = mech["魔法消耗"][str(json["魔法消耗"][i][j][k]["类型"]["代码"])]['名称']
                        json["魔法消耗"][i][j][k]["类型"]["后缀"] = mech["魔法消耗"][str(json["魔法消耗"][i][j][k]["类型"]["代码"])]['后缀']
    for i in json["冷却时间"]:
        for j in json["冷却时间"][i]:
            if "类型" in json["冷却时间"][i][j]:
                json["冷却时间"][i][j]["类型"]["值"] = mech["冷却时间"][str(json["冷却时间"][i][j]["类型"]["代码"])]["名称"]
                json["冷却时间"][i][j]["类型"]["图片"] = mech["冷却时间"][str(json["冷却时间"][i][j]["类型"]["代码"])]["图片"]
    for i in json["施法前摇"]:
        if str(json["施法前摇"][i]["即时生效"]["代码"]) in mech["即时生效"]:
            json["施法前摇"][i]["即时生效"]["图片"] = mech["即时生效"][str(json["施法前摇"][i]["即时生效"]["代码"])]

def loop_check(json, data, all_json, name):
    for i in json:
        if isinstance(json[i], dict):
            if "混合文字" in json[i]:
                change_combine_txt(json, i, data, all_json, name)
            else:
                loop_check(json[i], data, all_json, name)

def change_combine_txt(json, ii, data, all_json, name):
    returntxt = ""
    i = 0
    while True:
        i += 1
        if str(i) in json[ii]["混合文字"]:
            if isinstance(json[ii]["混合文字"][str(i)], dict):
                j = 0
                while True:
                    j += 1
                    if str(j) in json[ii]["混合文字"][str(i)]:
                        if json[ii]["混合文字"][str(i)][str(j)]["0"] == "升级属性":
                            if json[ii]["混合文字"][str(i)][str(j)]["1"] == "":
                                json[ii]["混合文字"][str(i)][str(j)]["1"] = "技能"
                            if json[ii]["混合文字"][str(i)][str(j)]["2"] == "":
                                json[ii]["混合文字"][str(i)][str(j)]["2"] = name
                        elif json[ii]["混合文字"][str(i)][str(j)]["0"] == "数据库":
                            if json[ii]["混合文字"][str(i)][str(j)]["1"] == "":
                                json[ii]["混合文字"][str(i)][str(j)]["1"] = "技能"
                            if json[ii]["混合文字"][str(i)][str(j)]["2"] == "":
                                json[ii]["混合文字"][str(i)][str(j)]["2"] = all_json["技能"][name]["代码"]
                    else:
                        break
                temp = combine_txt_numbers(json[ii]["混合文字"][str(i)], [1], all_json, data)
                if "等级" in json[ii]["混合文字"][str(i)] and int(json[ii]["混合文字"][str(i)]["等级"]) > 0:
                    level = int(json[ii]["混合文字"][str(i)]["等级"])
                else:
                    level = 0
                if "后缀" in json[ii]["混合文字"][str(i)]:
                    post = json[ii]["混合文字"][str(i)]["后缀"]
                else:
                    post = ""
                for j in range(4):
                    bool = True
                    for k in range(len(temp[j + 1])):
                        bool = bool and temp[j + 1][0] == temp[j + 1][k]
                    if bool:
                        temp[j + 1] = [temp[j + 1][0]]
                returntxt += combine_numbers_post_level(temp[1], post, level)
                if temp[0][0] or temp[0][1]:
                    returntxt += "("
                    if temp[0][0]:
                        for j in temp[0][3]:
                            returntxt += "[[file:" + temp[0][3][j]["图片"].replace('talent.png', 'talentb.png') + "|x18px|link=" + temp[0][3][j]["名称"] + "]]"
                        returntxt += combine_numbers_post_level(temp[2], post, level)
                    if temp[0][1]:
                        for j in temp[0][4]:
                            returntxt += "[[file:" + temp[0][4][j]["图片"].replace('talent.png', 'talentb.png') + "|x18px|link=" + temp[0][4][j]["名称"] + "]]"
                        returntxt += combine_numbers_post_level(temp[3], post, level)
                    if temp[0][2]:
                        for j in temp[0][3]:
                            returntxt += "[[file:" + temp[0][3][j]["图片"].replace('talent.png', 'talentb.png') + "|x18px|link=" + temp[0][3][j]["名称"] + "]]"
                        for j in temp[0][4]:
                            returntxt += "[[file:" + temp[0][4][j]["图片"].replace('talent.png', 'talentb.png') + "|x18px|link=" + temp[0][4][j]["名称"] + "]]"
                        returntxt += combine_numbers_post_level(temp[4], post, level)
                    returntxt += ")"
            else:
                returntxt += json[ii]["混合文字"][str(i)]
        else:
            break
    json[ii] = returntxt

def combine_txt_numbers(json, index, all_json, base_txt):
    i = index[0]
    if json[str(i)]["1"] == "(" or json[str(i)]["1"] == "（":
        index[0] += 1
        re = combine_txt_numbers(json, index, all_json, base_txt)
        i = index[0]
    re = one_combine_txt_numbers(json[str(i)], all_json, base_txt)
    while True:
        i += 1
        if str(i) in json:
            if json[str(i)]["1"] == "(" or json[str(i)]["1"] == "（":
                index[0] += 1
                temp = combine_txt_numbers(json, index, all_json, base_txt)
                i = index[0]
            elif json[str(i)]["符号"] == ")" or json[str(i)]["符号"] == "）":
                return re
            else:
                temp = one_combine_txt_numbers(json[str(i)], all_json, base_txt)
            calculate_combine_txt_numbers(re, temp, json[str(i)]["符号"])
        else:
            return re

def one_combine_txt_numbers(json, all_json, base_txt):
    re = [[False, False, False, {"1": {"名称": "阿哈利姆神杖", "图片": "agha.png"}}, {}], [], [], [], []]
    if json["0"] == "属性":
        temp = all_json[json["1"]][json["2"]][json["3"]]
        i = 3
        while True:
            i += 1
            if str(i) in json and json[str(i)] != '':
                temp = temp[json[str(i)]]
            else:
                break
        j = 0
        while True:
            j += 1
            if str(j) in temp:
                re[1].append(temp[str(j)])
            else:
                break
    elif json["0"] == "数据库":
        temp = base_txt[json["1"]][json["2"]][json["3"]]
        i = 3
        while True:
            i += 1
            if str(i) in json and json[str(i)] != '':
                temp = temp[json[str(i)]]
            else:
                break
        j = 0
        while True:
            j += 1
            if str(j) in temp:
                re[1].append(temp[str(j)])
            else:
                break
    elif json['0'] == '普通属性':
        temp = all_json[json["1"]][json["2"]][json["3"]]
        i = 3
        while True:
            i += 1
            if str(i) in json and json[str(i)] != '':
                temp = temp[json[str(i)]]
            else:
                break
        re[1].append(temp)
    elif json['0'] == '图片链接':
        temp = all_json[json["1"]][json["2"]]
        re[1].append('[[file:' + temp['迷你图片'] + '|' + json['3'] + '|link=]][[' + temp['页面名'] + ']]')
    elif json["0"] == "手填":
        j = 0
        while True:
            j += 1
            if str(j) in json:
                try:
                    re[1].append(float(json[str(j)]))
                except ValueError:
                    re[1].append(json[str(j)])
            else:
                break
    elif json["0"] == "升级属性":
        temp = all_json[json["1"]][json["2"]][json["3"]]
        i = 3
        while True:
            i += 1
            if str(i) in json:
                if str(json[str(i)]) == '升级':
                    ii = i + 1
                    break
                else:
                    if str(json[str(i)]) in temp:
                        temp = temp[str(json[str(i)])]
                    elif str(json[str(i)]).isdigit():
                        temp = temp["1"]
            else:
                ii = 0
                break
        for i in range(4):
            if str(i + 1) in temp:
                if i > 0:
                    re[0][i - 1] = True
                    if i == 2:
                        re[0][4] = copy.deepcopy(temp["3"]["升级来源"])
                if ii == 0:
                    j = 0
                    while True:
                        j += 1
                        if str(j) in temp[str(i + 1)]:
                            re[i + 1].append(temp[str(i + 1)][str(j)])
                        else:
                            break
                else:
                    if str(json[str(ii)]) in temp[str(i + 1)]:
                        re[i + 1] = [temp[str(i + 1)][json[str(ii)]]]
                    elif str(json[str(ii)]).isdigit():
                        re[i + 1] = [temp[str(i + 1)]["1"]]
    else:
        temp = all_json[json["1"]][json["2"]][json["3"]]
        i = 3
        while True:
            i += 1
            if str(i) in json:
                if str(json[str(i)]) in temp:
                    temp = temp[str(json[str(i)])]
                elif str(json[str(i)]).isdigit():
                    temp = temp["1"]
            else:
                break
        for i in range(4):
            if str(i + 1) in temp:
                if i > 0:
                    re[0][i - 1] = True
                    if i == 2:
                        re[0][4] = copy.deepcopy(temp["3"]["升级来源"])
                if isinstance(temp[str(i + 1)], dict):
                    j = 0
                    while True:
                        j += 1
                        if str(j) in temp[str(i + 1)]:
                            re[i + 1].append(temp[str(i + 1)][str(j)])
                        else:
                            break
                else:
                    re[i + 1] = [temp[str(i + 1)]]
    if not re[0][0]:
        re[2] = copy.deepcopy(re[1])
    if not re[0][1]:
        re[3] = copy.deepcopy(re[1])
    if not re[0][2]:
        if re[0][0]:
            re[4] = copy.deepcopy(re[2])
        else:
            re[4] = copy.deepcopy(re[3])
    return re

def calculate_combine_txt_numbers(re, temp, op):
    re[0][0] = re[0][0] or temp[0][0]
    re[0][1] = re[0][1] or temp[0][1]
    re[0][2] = re[0][2] or temp[0][2] or re[0][0] and re[0][1]
    if temp[0][4] != {}:
        if re[0][4] == {}:
            re[0][4] = copy.deepcopy(temp[0][4])
        elif re[0][4]["1"]["名称"]!=temp[0][4]["1"]["名称"]:
            re[0][0]=True
            re[0][2]=True
            re[0][3]=copy.deepcopy(temp[0][4])
            temp[2]=copy.deepcopy(temp[3])
            temp[3]=copy.deepcopy(temp[1])
    for i in range(4):
        for j in range(max(len(re[i + 1]), len(temp[i + 1]))):
            if j >= len(re[i + 1]):
                re[i + 1].append(re[i + 1][0])
            if j >= len(temp[i + 1]):
                temp[i + 1].append(temp[i + 1][0])
        if op == 'max_stack':
            re[i + 1] = [re[i + 1][0]]
            j = 0
            while True:
                if j * temp[i + 1][0] < re[i + 1][j]:
                    re[i + 1].insert(j, j * temp[i + 1][0])
                    j += 1
                else:
                    break
        elif op == 'num_stack':
            jj = re[i + 1][0]
            re[i + 1] = []
            for j in range(int(jj)):
                re[i + 1].append(j * temp[i + 1][0])
        else:
            for j in range(len(re[i + 1])):
                if op == '+':
                    re[i + 1][j] += temp[i + 1][j]
                elif op == '-':
                    re[i + 1][j] -= temp[i + 1][j]
                elif op == '*':
                    re[i + 1][j] *= temp[i + 1][j]
                elif op == '/':
                    re[i + 1][j] /= temp[i + 1][j]
                elif op == '\\':
                    re[i + 1][j] = temp[i + 1][j] / re[i + 1][j]
                elif op == '^' or op == 'pow':
                    re[i + 1][j] = pow(re[i + 1][j], temp[i + 1][j])
                elif op == '%-':
                    re[i + 1][j] = re[i + 1][j] * (100 - temp[i + 1][j]) / 100
                elif op == '-%':
                    re[i + 1][j] = re[i + 1][j] * (1 - temp[i + 1][j])
                elif op == '%+':
                    re[i + 1][j] = re[i + 1][j] * (100 + temp[i + 1][j]) / 100
                elif op == '+%':
                    re[i + 1][j] = re[i + 1][j] * (1 + temp[i + 1][j])
                elif op == 'min' or op == "↓":
                    re[i + 1][j] = min(re[i + 1][j], temp[i + 1][j])
                elif op == 'max' or op == "↑":
                    re[i + 1][j] = max(re[i + 1][j], temp[i + 1][j])
                elif op == 'round':
                    re[i + 1][j] = round(re[i + 1][j], int(temp[i + 1][j]))
                elif op == 'ceil':
                    re[i + 1][j] = math.ceil(re[i + 1][j] * pow(10, temp[i + 1][j])) / pow(10, temp[i + 1][j])
                elif op == 'floor':
                    re[i + 1][j] = math.floor(re[i + 1][j] * pow(10, temp[i + 1][j])) / pow(10, temp[i + 1][j])
                elif op == '%*%':
                    re[i + 1][j] = (100 - (100 - re[i + 1][j]) * (100 - temp[i + 1][j]) / 100)
                elif op == 'a2%':
                    re[i + 1][j] = 13 * re[i + 1][j] / (225 + 12 * abs(re[i + 1][j])) * temp[i + 1][j]
                elif op == '%2a':
                    re[i + 1][j] = 225 * re[i + 1][j] / (13 * temp[i + 1][j] - 12 * abs(re[i + 1][j]))
                elif op == 'int':
                    re[i + 1][j] = int(re[i + 1][j])
                elif op == 'gp_s':#等比数列（GP）求和
                    re[i + 1][j] = (1-pow(re[i + 1][j],temp[i + 1][j]))/(1-re[i + 1][j])

def combine_numbers_post_level(arr, post, level):
    re = ""
    if level > 0:
        if level < len(arr):
            re += better_float_to_text(arr[level])
        else:
            re += better_float_to_text(arr[0])
        re += post
    else:
        for i in range(len(arr)):
            if i > 0:
                re += "/"
            re += better_float_to_text(arr[i])
            re += post
    return re

def confirm_upgrade_info(arr):
    for i in arr:
        if len(arr[i])>0:
            for j in arr[i]['技能升级信息']:
                arr[i]['技能升级信息'][j]['中文名']=arr[arr[i]['技能升级信息'][j]['技能名']]['中文名']
                arr[i]['技能升级信息'][j]['图片']=arr[arr[i]['技能升级信息'][j]['技能名']]['迷你图片']

def create_file(all_json):
    for i in all_json:
        file = open("E:/json/pythonupload/" + i + '.json', mode="w")
        file.write(json.dumps(all_json[i]))
        file.close()

abilitypro_num = [["a_cast_range", "AbilityCastRange"]
    , ["a_cast_range_buff", "AbilityCastRangeBuffer"]
    , ["a_cast", "AbilityCastPoint"]
    , ["a_channel", "AbilityChannelTime"]
    , ["a_cool", "AbilityCooldown"]
    , ["a_duration", "AbilityDuration"]
    , ["a_damage", "AbilityDamage"]
    , ["a_mana", "AbilityManaCost"]
    , ["a_charges", "AbilityCharges"]
    , ["a_charges_restore_time", "AbilityChargeRestoreTime"]]
abilitypro_bool = [["immediate", "DOTA_ABILITY_BEHAVIOR_IMMEDIATE"]
    , ["ignore_channel", "DOTA_ABILITY_BEHAVIOR_IGNORE_CHANNEL"]]
ability_trait_level = [["中文名", "英文名", "代码", "传说", "描述", "天赋代码", "A杖信息", "注释"], ["效果", "属性", "冷却时间"], ["魔法消耗"]]


#Jenn Debono
#dice roll game
#get user random selection from 1 to 6, like a 6-sided dice
#compare to computer random choice
#if user and computer choice match or is higher than computer choice, user wins
#turtle moves forward if user wins
#invalid inputs handled: integers less than 0 or greater than 7, negative integers,
# and string values
#play continues until user wins
#input for user recieved by keyboard entry and promps
# assistance with Counter, While and Try section by Jim Wei

#import random module
import random

#user defined function for game
def diceroll():

    # counter used for monitoring how many tries left
    counter = 0
    #counter for maximum of 3 tries
    while counter < 3:
        # start of error handling
        try:
            # create a die
            dice = [1,2,3,4,5,6]

            #get user selection from 1 to 6
            playerRoll = int(input("Choose a number between 1 and 6: "))
            print("Player Choice: ", playerRoll)

            # error handling for user entry of value other than the list from 1-6
            while playerRoll not in dice:
                print ("Incorrect numeric value. Please enter a number between 1 and 6")
                # send user back to enter input
                playerRoll = int(input("Choose a number between 1 and 6: "))
                print("Player Choice: ", playerRoll)

            #computer random roll
            dice_outcome = random.choice(dice)
            print("Computer Choice: ", dice_outcome)
            print()

            #final output to determine win and loss
            #win scenario 1: guess number higher than computer
            if dice_outcome < playerRoll:
                print ("You guessed a higher number than the computer! You Win!")
                break
            #win scenario 2: match same number as computer
            elif dice_outcome == playerRoll:
                print ("You matched the computer! Great guess! You Win!")
                break
            #loss scenario: computer select number higher than user
            else:
                print ("Computer Wins!")
                counter += 1
                print("Strike", counter, "!")

                # mechanism for ending game
                if counter == 3:
                    print("GAME OVER")
                    game = "gameover"
                    return game

        # error handling for string entries
        except ValueError:
            print ("Oops! Letters are not going to work here. Try again")
            # send user back to enter input
            diceroll()

# game function call
#diceroll()


# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import xml.etree.ElementTree as ET
from fwunit.ip import IP, IPSet
from fwunit.srx import parse
from nose.tools import eq_
from fwunit.test.util.srx_xml import route_xml_blackhole
from fwunit.test.util.srx_xml import route_xml_11_4R6
from fwunit.test.util.srx_xml import zones_empty_xml
from fwunit.test.util.srx_xml import FakeSRX

def parse_xml(xml, elt_path=None):
    elt = parse.strip_namespaces(ET.fromstring(xml))
    if elt_path:
        elt = elt.find(elt_path)
    return elt

def test_parse_zone():
    f = FakeSRX()
    z = f.add_zone('untrust')
    f.add_address(z, 'host1', '9.0.9.1/32')
    f.add_address(z, 'host2', '9.0.9.2/32')
    f.add_address(z, 'puppet', '9.0.9.3/32')
    f.add_address_set(z, 'hosts', 'host1', 'host2')
    f.add_interface(z, 'reth0')

    elt = parse_xml(
        f.fake_show('configuration security zones'), './/security-zone')
    z = parse.Zone._from_xml(elt)
    eq_(z.interfaces, ['reth0'])
    eq_(sorted(z.addresses.keys()),
        sorted(['any', 'any-ipv4', 'any-ipv6', 'host1', 'host2', 'hosts', 'puppet']))
    eq_(z.addresses['any'], IPSet([IP('0.0.0.0/0')]))
    eq_(z.addresses['any-ipv4'], IPSet([IP('0.0.0.0/0')]))
    eq_(z.addresses['any-ipv6'], IPSet([]))
    eq_(z.addresses['host1'], IPSet([IP('9.0.9.1')]))
    eq_(z.addresses['host2'], IPSet([IP('9.0.9.2')]))
    eq_(z.addresses['puppet'], IPSet([IP('9.0.9.3')]))
    eq_(z.addresses['hosts'], IPSet([IP('9.0.9.1'), IP('9.0.9.2')]))

def test_parse_zone_no_sets():
    f = FakeSRX()
    z = f.add_zone('untrust')
    f.add_address(z, 'trustedhost', '10.0.9.2/32')
    f.add_address(z, 'dmz', '10.1.0.0/16')
    f.add_address(z, 'shadow', '10.1.99.99/32')
    f.add_interface(z, 'reth1')

    elt = parse_xml(
        f.fake_show('configuration security zones'), './/security-zone')
    z = parse.Zone._from_xml(elt)
    eq_(z.interfaces, ['reth1'])
    eq_(sorted(z.addresses.keys()),
        sorted(['any', 'any-ipv4', 'any-ipv6', 'trustedhost', 'dmz', 'shadow']))
    eq_(z.addresses['any'], IPSet([IP('0.0.0.0/0')]))
    eq_(z.addresses['any-ipv4'], IPSet([IP('0.0.0.0/0')]))
    eq_(z.addresses['any-ipv6'], IPSet([]))
    eq_(z.addresses['trustedhost'], IPSet([IP('10.0.9.2')]))
    eq_(z.addresses['dmz'], IPSet([IP('10.1.0.0/16')]))
    eq_(z.addresses['shadow'], IPSet([IP('10.1.99.99')]))

def test_parse_global_policy():
    f = FakeSRX()
    f.add_policy('global',
                 dict(sequence=1, name='global-ping', src='any', dst='any',
                      app='ping', action='permit'))
    f.add_policy('global',
                 dict(sequence=10, name='global-deny', src='any', dst='any',
                      app='any', action='deny'))

    elt = parse_xml(
        f.fake_show('security policies global'), './/security-context')
    policies = [parse.Policy._from_xml(None, None, e) for e in elt.findall('./policies/policy-information')]
    eq_(policies[0].from_zone, None)
    eq_(policies[0].to_zone, None)
    eq_(policies[0].to_zone, None)
    eq_(policies[0].enabled, True)
    eq_(policies[0].sequence, 1)
    eq_(policies[0].source_addresses, ['any'])
    eq_(policies[0].destination_addresses, ['any'])
    eq_(policies[0].applications, ['ping'])
    eq_(policies[0].action, 'permit')
    eq_(policies[0].from_zone, None)
    eq_(policies[1].to_zone, None)
    eq_(policies[1].to_zone, None)
    eq_(policies[1].enabled, True)
    eq_(policies[1].sequence, 10)
    eq_(policies[1].source_addresses, ['any'])
    eq_(policies[1].destination_addresses, ['any'])
    eq_(policies[1].applications, ['any'])
    eq_(policies[1].action, 'deny')

def test_parse_addrbook_global():
    f = FakeSRX()
    ab = f.add_addrbook('global')
    f.add_address(ab, 'host1', '9.0.9.1/32')
    f.add_address(ab, 'host2', '9.0.9.2/32')
    f.add_address_set(ab, 'hosts', 'host1', 'host2')

    elt = parse_xml(
        f.fake_show('configuration security address-book'), './/address-book')
    z = parse.AddressBook._from_xml(elt)
    eq_(z.attaches, [])
    eq_(sorted(z.addresses.keys()),
        sorted(['any', 'any-ipv4', 'any-ipv6', 'host1', 'host2', 'hosts']))
    eq_(z.addresses['any'], IPSet([IP('0.0.0.0/0')]))
    eq_(z.addresses['any-ipv4'], IPSet([IP('0.0.0.0/0')]))
    eq_(z.addresses['any-ipv6'], IPSet([]))
    eq_(z.addresses['host1'], IPSet([IP('9.0.9.1')]))
    eq_(z.addresses['host2'], IPSet([IP('9.0.9.2')]))
    eq_(z.addresses['hosts'], IPSet([IP('9.0.9.1'), IP('9.0.9.2')]))

def test_parse_addrbook_attached():
    f = FakeSRX()
    ab = f.add_addrbook('general-stuff')
    f.add_address(ab, 'trustedhost', '10.0.9.2/32')
    f.add_attach(ab, 'untrust')
    f.add_attach(ab, 'trust')

    elt = parse_xml(
        f.fake_show('configuration security address-book'), './/address-book')
    z = parse.AddressBook._from_xml(elt)
    eq_(z.attaches, ['untrust', 'trust'])
    eq_(sorted(z.addresses.keys()),
        sorted(['any', 'any-ipv4', 'any-ipv6', 'trustedhost']))
    eq_(z.addresses['any'], IPSet([IP('0.0.0.0/0')]))
    eq_(z.addresses['any-ipv4'], IPSet([IP('0.0.0.0/0')]))
    eq_(z.addresses['any-ipv6'], IPSet([]))
    eq_(z.addresses['trustedhost'], IPSet([IP('10.0.9.2')]))

def test_parse_zones_empty():
    elt = parse_xml(zones_empty_xml, './/security-zone')
    z = parse.Zone._from_xml(elt)
    eq_(z.interfaces, [])
    eq_(sorted(z.addresses.keys()), sorted(['any', 'any-ipv6', 'any-ipv4']))

def test_parse_route_blackhole():
    elt = parse_xml(route_xml_blackhole, './/rt')
    r = parse.Route._from_xml(elt)
    eq_(r.destination, IP('20.0.0.0/8'))
    eq_(r.interface, None)
    eq_(r.is_local, False)

def test_parse_route_11_4R6():
    elt = parse_xml(route_xml_11_4R6, './/rt')
    r = parse.Route._from_xml(elt)
    eq_(r.destination, IP('0.0.0.0/0'))
    eq_(r.interface, 'reth0.10')
    eq_(r.is_local, False)


"""
Setup and launch a CloudMan instance.
"""
import datetime
import socket
from http.client import (
    BadStatusLine,
    HTTPConnection,
    HTTPException,
)
from urllib.parse import urlparse

import boto
import yaml
from boto.ec2.regioninfo import RegionInfo
from boto.exception import EC2ResponseError, S3ResponseError
from boto.s3.connection import OrdinaryCallingFormat, S3Connection, SubdomainCallingFormat

import bioblend
from bioblend.util import Bunch

# Uncomment the following line if no logging from boto is desired
# bioblend.logging.getLogger('boto').setLevel(bioblend.logging.CRITICAL)
# Uncomment the following line if logging at the prompt is desired
# bioblend.set_stream_logger(__name__)
def instance_types(cloud_name='generic'):
    """
    Return a list of dictionaries containing details about the available
    instance types for the given `cloud_name`.

    :type cloud_name: str
    :param cloud_name: A name of the cloud for which the list of instance
                       types will be returned. Valid values are: `aws`,
                       `nectar`, `generic`.

    :rtype: list
    :return: A list of dictionaries describing instance types. Each dict will
             contain the following keys: `name`, `model`, and `description`.
    """
    instance_list = []
    if cloud_name.lower() == 'aws':
        instance_list.append({"model": "c3.large",
                              "name": "Compute optimized Large",
                              "description": "2 vCPU/4GB RAM"})
        instance_list.append({"model": "c3.2xlarge",
                              "name": "Compute optimized 2xLarge",
                              "description": "8 vCPU/15GB RAM"})
        instance_list.append({"model": "c3.8xlarge",
                              "name": "Compute optimized 8xLarge",
                              "description": "32 vCPU/60GB RAM"})
    elif cloud_name.lower() in ['nectar', 'generic']:
        instance_list.append({"model": "m1.small",
                              "name": "Small",
                              "description": "1 vCPU / 4GB RAM"})
        instance_list.append({"model": "m1.medium",
                              "name": "Medium",
                              "description": "2 vCPU / 8GB RAM"})
        instance_list.append({"model": "m1.large",
                              "name": "Large",
                              "description": "4 vCPU / 16GB RAM"})
        instance_list.append({"model": "m1.xlarge",
                              "name": "Extra Large",
                              "description": "8 vCPU / 32GB RAM"})
        instance_list.append({"model": "m1.xxlarge",
                              "name": "Extra-extra Large",
                              "description": "16 vCPU / 64GB RAM"})
    return instance_list

class CloudManLauncher:

    def __init__(self, access_key, secret_key, cloud=None):
        """
        Define the environment in which this instance of CloudMan will be launched.

        Besides providing the credentials, optionally provide the ``cloud``
        object. This object must define the properties required to establish a
        `boto <https://github.com/boto/boto/>`_ connection to that cloud. See
        this method's implementation for an example of the required fields.
        Note that as long the as provided object defines the required fields,
        it can really by implemented as anything (e.g., a Bunch, a database
        object, a custom class). If no value for the ``cloud`` argument is
        provided, the default is to use the Amazon cloud.
        """
        self.access_key = access_key
        self.secret_key = secret_key
        if cloud is None:
            # Default to an EC2-compatible object
            self.cloud = Bunch(id='1',  # for compatibility w/ DB representation
                               name="Amazon",
                               cloud_type="ec2",
                               bucket_default="cloudman",
                               region_name="us-east-1",
                               region_endpoint="ec2.amazonaws.com",
                               ec2_port="",
                               ec2_conn_path="/",
                               cidr_range="",
                               is_secure=True,
                               s3_host="s3.amazonaws.com",
                               s3_port="",
                               s3_conn_path='/')
        else:
            self.cloud = cloud
        self.ec2_conn = self.connect_ec2(
            self.access_key,
            self.secret_key,
            self.cloud)
        self.vpc_conn = self.connect_vpc(
            self.access_key,
            self.secret_key,
            self.cloud)
        # Define exceptions that we want to catch and retry
        self.http_exceptions = (
            HTTPException,
            socket.error,
            socket.gaierror,
            BadStatusLine
        )

    def __repr__(self):
        return f"Cloud: {self.cloud.name}; acct ID: {self.access_key}"

    def launch(self, cluster_name, image_id, instance_type, password,
               kernel_id=None, ramdisk_id=None, key_name='cloudman_key_pair',
               security_groups=None, placement='', subnet_id=None,
               ebs_optimized=False, **kwargs):
        """
        Check all the prerequisites (key pair and security groups) for
        launching a CloudMan instance, compose the user data based on the
        parameters specified in the arguments and the cloud properties as
        defined in the object's ``cloud`` field.

        For the current list of user data fields that can be provided via
        ``kwargs``, see `<https://galaxyproject.org/cloudman/userdata/>`_

        Return a dict containing the properties and info with which an instance
        was launched, namely: ``sg_names`` containing the names of the security
        groups, ``kp_name`` containing the name of the key pair, ``kp_material``
        containing the private portion of the key pair (*note* that this portion
        of the key is available and can be retrieved *only* at the time the key
        is created, which will happen only if no key with the name provided in
        the ``key_name`` argument exists), ``rs`` containing the
        `boto <https://github.com/boto/boto/>`_ ``ResultSet`` object,
        ``instance_id`` containing the ID of a started instance, and
        ``error`` containing an error message if there was one.
        """
        if security_groups is None:
            security_groups = ['CloudMan']
        ret = {'sg_names': [],
               'sg_ids': [],
               'kp_name': '',
               'kp_material': '',
               'rs': None,
               'instance_id': '',
               'error': None}
        # First satisfy the prerequisites
        for sg in security_groups:
            # Get VPC ID in case we're launching into a VPC
            vpc_id = None
            if subnet_id:
                try:
                    sn = self.vpc_conn.get_all_subnets(subnet_id)[0]
                    vpc_id = sn.vpc_id
                except (EC2ResponseError, IndexError):
                    bioblend.log.exception("Trouble fetching subnet %s", subnet_id)
            cmsg = self.create_cm_security_group(sg, vpc_id=vpc_id)
            ret['error'] = cmsg['error']
            if ret['error']:
                return ret
            if cmsg['name']:
                ret['sg_names'].append(cmsg['name'])
                ret['sg_ids'].append(cmsg['sg_id'])
                if subnet_id:
                    # Must setup a network interface if launching into VPC
                    security_groups = None
                    interface = boto.ec2.networkinterface.NetworkInterfaceSpecification(
                        subnet_id=subnet_id, groups=[cmsg['sg_id']],
                        associate_public_ip_address=True)
                    network_interfaces = (boto.ec2.networkinterface.
                                          NetworkInterfaceCollection(interface))
                else:
                    network_interfaces = None
        kp_info = self.create_key_pair(key_name)
        ret['kp_name'] = kp_info['name']
        ret['kp_material'] = kp_info['material']
        ret['error'] = kp_info['error']
        if ret['error']:
            return ret
        # If not provided, try to find a placement
        # TODO: Should placement always be checked? To make sure it's correct
        # for existing clusters.
        if not placement:
            placement = self._find_placement(
                cluster_name).get('placement', None)
        # Compose user data for launching an instance, ensuring we have the
        # required fields
        kwargs['access_key'] = self.access_key
        kwargs['secret_key'] = self.secret_key
        kwargs['cluster_name'] = cluster_name
        kwargs['password'] = password
        kwargs['cloud_name'] = self.cloud.name
        ud = self._compose_user_data(kwargs)
        # Now launch an instance
        try:

            rs = None
            rs = self.ec2_conn.run_instances(image_id=image_id,
                                             instance_type=instance_type,
                                             key_name=key_name,
                                             security_groups=security_groups,
                                             # The following two arguments are
                                             # provided in the network_interface
                                             # instead of arguments:
                                             # security_group_ids=security_group_ids,
                                             # subnet_id=subnet_id,
                                             network_interfaces=network_interfaces,
                                             user_data=ud,
                                             kernel_id=kernel_id,
                                             ramdisk_id=ramdisk_id,
                                             placement=placement,
                                             ebs_optimized=ebs_optimized)
            ret['rs'] = rs
        except EC2ResponseError as e:
            err_msg = f"Problem launching an instance: {e} (code {e.error_code}; status {e.status})"
            bioblend.log.exception(err_msg)
            ret['error'] = err_msg
            return ret
        else:
            if rs:
                try:
                    bioblend.log.info("Launched an instance with ID %s", rs.instances[0].id)
                    ret['instance_id'] = rs.instances[0].id
                    ret['instance_ip'] = rs.instances[0].ip_address
                except EC2ResponseError as e:
                    err_msg = f"Problem with the launched instance object: {e} (code {e.error_code}; status {e.status})"
                    bioblend.log.exception(err_msg)
                    ret['error'] = err_msg
            else:
                ret['error'] = ("No response after launching an instance. Check "
                                "your account permissions and try again.")
        return ret

    def create_cm_security_group(self, sg_name='CloudMan', vpc_id=None):
        """
        Create a security group with all authorizations required to run CloudMan.

        If the group already exists, check its rules and add the missing ones.

        :type sg_name: str
        :param sg_name: A name for the security group to be created.

        :type vpc_id: str
        :param vpc_id: VPC ID under which to create the security group.

        :rtype: dict
        :return: A dictionary containing keys ``name`` (with the value being the
                 name of the security group that was created), ``error``
                 (with the value being the error message if there was an error
                 or ``None`` if no error was encountered), and ``ports``
                 (containing the list of tuples with port ranges that were
                 opened or attempted to be opened).

        .. versionchanged:: 0.6.1
            The return value changed from a string to a dict
        """
        ports = (('20', '21'),  # FTP
                 ('22', '22'),  # SSH
                 ('80', '80'),  # Web UI
                 ('443', '443'),  # SSL Web UI
                 ('8800', '8800'),  # NodeJS Proxy for Galaxy IPython IE
                 ('9600', '9700'),  # HTCondor
                 ('30000', '30100'))  # FTP transfer
        progress = {'name': None,
                    'sg_id': None,
                    'error': None,
                    'ports': ports}
        cmsg = None
        filters = None
        if vpc_id:
            filters = {'vpc-id': vpc_id}
        # Check if this security group already exists
        try:
            sgs = self.ec2_conn.get_all_security_groups(filters=filters)
        except EC2ResponseError as e:
            err_msg = f"Problem getting security groups. This could indicate a problem with your account credentials or permissions: {e} (code {e.error_code}; status {e.status})"
            bioblend.log.exception(err_msg)
            progress['error'] = err_msg
            return progress
        for sg in sgs:
            if sg.name == sg_name:
                cmsg = sg
                bioblend.log.debug("Security group '%s' already exists; will add authorizations next.", sg_name)
                break
        # If it does not exist, create security group
        if cmsg is None:
            bioblend.log.debug("Creating Security Group %s", sg_name)
            try:
                cmsg = self.ec2_conn.create_security_group(sg_name, 'A security '
                                                           'group for CloudMan',
                                                           vpc_id=vpc_id)
            except EC2ResponseError as e:
                err_msg = f"Problem creating security group '{sg_name}': {e} (code {e.error_code}; status {e.status})"
                bioblend.log.exception(err_msg)
                progress['error'] = err_msg
        if cmsg:
            progress['name'] = cmsg.name
            progress['sg_id'] = cmsg.id
            # Add appropriate authorization rules
            # If these rules already exist, nothing will be changed in the SG
            for port in ports:
                try:
                    if not self.rule_exists(
                            cmsg.rules, from_port=port[0], to_port=port[1]):
                        cmsg.authorize(
                            ip_protocol='tcp',
                            from_port=port[0],
                            to_port=port[1],
                            cidr_ip='0.0.0.0/0')
                    else:
                        bioblend.log.debug("Rule (%s:%s) already exists in the SG", port[0], port[1])
                except EC2ResponseError as e:
                    err_msg = f"A problem adding security group authorizations: {e} (code {e.error_code}; status {e.status})"
                    bioblend.log.exception(err_msg)
                    progress['error'] = err_msg
            # Add ICMP (i.e., ping) rule required by HTCondor
            try:
                if not self.rule_exists(
                        cmsg.rules, from_port='-1', to_port='-1', ip_protocol='icmp'):
                    cmsg.authorize(
                        ip_protocol='icmp',
                        from_port=-1,
                        to_port=-1,
                        cidr_ip='0.0.0.0/0')
                else:
                    bioblend.log.debug(
                        f"ICMP rule already exists in {sg_name} SG.")
            except EC2ResponseError as e:
                err_msg = f"A problem with security ICMP rule authorization: {e} (code {e.error_code}; status {e.status})"
                bioblend.log.exception(err_msg)
                progress['err_msg'] = err_msg
            # Add rule that allows communication between instances in the same
            # SG
            # A flag to indicate if group rule already exists
            g_rule_exists = False
            for rule in cmsg.rules:
                for grant in rule.grants:
                    if grant.name == cmsg.name:
                        g_rule_exists = True
                        bioblend.log.debug(
                            "Group rule already exists in the SG.")
                if g_rule_exists:
                    break
            if not g_rule_exists:
                try:
                    cmsg.authorize(
                        src_group=cmsg,
                        ip_protocol='tcp',
                        from_port=0,
                        to_port=65535)
                except EC2ResponseError as e:
                    err_msg = f"A problem with security group group authorization: {e} (code {e.error_code}; status {e.status})"
                    bioblend.log.exception(err_msg)
                    progress['err_msg'] = err_msg
            bioblend.log.info("Done configuring '%s' security group", cmsg.name)
        else:
            bioblend.log.warning(
                f"Did not create security group '{sg_name}'")
        return progress

    def rule_exists(
            self, rules, from_port, to_port, ip_protocol='tcp', cidr_ip='0.0.0.0/0'):
        """
        A convenience method to check if an authorization rule in a security group
        already exists.
        """
        for rule in rules:
            if rule.ip_protocol == ip_protocol and rule.from_port == from_port and \
               rule.to_port == to_port and cidr_ip in [ip.cidr_ip for ip in rule.grants]:
                return True
        return False

    def create_key_pair(self, key_name='cloudman_key_pair'):
        """
        If a key pair with the provided ``key_name`` does not exist, create it.

        :type sg_name: str
        :param sg_name: A name for the key pair to be created.

        :rtype: dict
        :return: A dictionary containing keys ``name`` (with the value being the
                 name of the key pair that was created), ``error``
                 (with the value being the error message if there was an error
                 or ``None`` if no error was encountered), and ``material``
                 (containing the unencrypted PEM encoded RSA private key if the
                 key was created or ``None`` if the key already eixsted).

        .. versionchanged:: 0.6.1
            The return value changed from a tuple to a dict
        """
        progress = {'name': None,
                    'material': None,
                    'error': None}
        kp = None
        # Check if a key pair under the given name already exists. If it does not,
        # create it, else return.
        try:
            kps = self.ec2_conn.get_all_key_pairs()
        except EC2ResponseError as e:
            err_msg = f"Problem getting key pairs: {e} (code {e.error_code}; status {e.status})"
            bioblend.log.exception(err_msg)
            progress['error'] = err_msg
            return progress
        for akp in kps:
            if akp.name == key_name:
                bioblend.log.info("Key pair '%s' already exists; reusing it.", key_name)
                progress['name'] = akp.name
                return progress
        try:
            kp = self.ec2_conn.create_key_pair(key_name)
        except EC2ResponseError as e:
            err_msg = f"Problem creating key pair '{key_name}': {e} (code {e.error_code}; status {e.status})"
            bioblend.log.exception(err_msg)
            progress['error'] = err_msg
            return progress
        bioblend.log.info("Created key pair '%s'", kp.name)
        progress['name'] = kp.name
        progress['material'] = kp.material
        return progress

    def assign_floating_ip(self, ec2_conn, instance):
        try:
            bioblend.log.debug("Allocating a new floating IP address.")
            address = ec2_conn.allocate_address()
        except EC2ResponseError:
            bioblend.log.exception("Exception allocating a new floating IP address")
        bioblend.log.info("Associating floating IP %s to instance %s", address.public_ip, instance.id)
        ec2_conn.associate_address(instance_id=instance.id,
                                   public_ip=address.public_ip)

    def get_status(self, instance_id):
        """
        Check on the status of an instance. ``instance_id`` needs to be a
        ``boto``-library copatible instance ID (e.g., ``i-8fehrdss``).If
        ``instance_id`` is not provided, the ID obtained when launching
        *the most recent* instance is used. Note that this assumes the instance
        being checked on was launched using this  class. Also note that the same
        class may be used to launch multiple instances but only the most recent
        ``instance_id`` is kept while any others will  to be explicitly specified.

        This method also allows the required ``ec2_conn`` connection object to be
        provided at invocation time. If the object is not provided, credentials
        defined for the class are used (ability to specify a custom ``ec2_conn``
        helps in case of stateless method invocations).

        Return a ``state`` dict containing the following keys: ``instance_state``,
        ``public_ip``, ``placement``, and ``error``, which capture CloudMan's
        current state. For ``instance_state``, expected values are: ``pending``,
        ``booting``, ``running``, or ``error`` and represent the state of the
        underlying instance. Other keys will return an empty  value until the
        ``instance_state`` enters ``running`` state.
        """
        ec2_conn = self.ec2_conn
        rs = None
        state = {'instance_state': "",
                 'public_ip': "",
                 'placement': "",
                 'error': ""}

        # Make sure we have an instance ID
        if instance_id is None:
            err = "Missing instance ID, cannot check the state."
            bioblend.log.error(err)
            state['error'] = err
            return state
        try:
            rs = ec2_conn.get_all_instances([instance_id])
            if rs is not None:
                inst_state = rs[0].instances[0].update()
                public_ip = rs[0].instances[0].ip_address
                state['public_ip'] = public_ip
                if inst_state == 'running':
                    # if there's a private ip, but no public ip
                    # attempt auto allocation of floating IP
                    if rs[0].instances[0].private_ip_address and not public_ip:
                        self.assign_floating_ip(ec2_conn, rs[0].instances[0])
                    # Wait until the CloudMan URL is accessible to return
                    # the data - this may only be correct initially at
                    # bootup for instances that configure themselves for
                    # https - they may ultimately block port 80. However,
                    # we don't have a good way to determine whether to
                    # check using http or https.
                    cm_url = f"http://{public_ip}/cloud"
                    if self._checkURL(cm_url) is True:
                        state['instance_state'] = inst_state
                        state['placement'] = rs[0].instances[0].placement
                    else:
                        state['instance_state'] = 'booting'
                else:
                    state['instance_state'] = inst_state
        except Exception as e:
            err = f"Problem updating instance '{instance_id}' state: {e}"
            bioblend.log.error(err)
            state['error'] = err
        return state

    def get_clusters_pd(self, include_placement=True):
        """
        Return *persistent data* of all existing clusters for this account.

        :type include_placement: bool
        :param include_placement: Whether or not to include region placement for
                                  the clusters. Setting this option will lead
                                  to a longer function runtime.

        :rtype: dict
        :return: A dictionary containing keys ``clusters`` and ``error``. The
                 value of ``clusters`` will be a dictionary with the following keys
                 ``cluster_name``, ``persistent_data``, ``bucket_name`` and optionally
                 ``placement`` or an empty list if no clusters were found or an
                 error was encountered. ``persistent_data`` key value is yet
                 another dictionary containing given cluster's persistent data.
                 The value for the ``error`` key will contain a string with the
                 error message.

        .. versionadded:: 0.3
        .. versionchanged:: 0.7.0
            The return value changed from a list to a dictionary.
        """
        clusters = []
        response = {'clusters': clusters, 'error': None}
        s3_conn = self.connect_s3(self.access_key, self.secret_key, self.cloud)
        try:
            buckets = s3_conn.get_all_buckets()
        except S3ResponseError as e:
            response['error'] = f"S3ResponseError getting buckets: {e}"
        except self.http_exceptions as ex:
            response['error'] = f"Exception getting buckets: {ex}"
        if response['error']:
            bioblend.log.exception(response['error'])
            return response
        for bucket in [b for b in buckets if b.name.startswith('cm-')]:
            try:
                # TODO: first lookup if persistent_data.yaml key exists
                pd = bucket.get_key('persistent_data.yaml')
            except S3ResponseError:
                # This can fail for a number of reasons for non-us and/or
                # CNAME'd buckets but it is not a terminal error
                bioblend.log.warning("Problem fetching persistent_data.yaml from bucket %s", bucket)
                continue
            if pd:
                # We are dealing with a CloudMan bucket
                pd_contents = pd.get_contents_as_string()
                pd = yaml.safe_load(pd_contents)
                if 'cluster_name' in pd:
                    cluster_name = pd['cluster_name']
                else:
                    for key in bucket.list():
                        if key.name.endswith('.clusterName'):
                            cluster_name = key.name.split('.clusterName')[0]
                cluster = {'cluster_name': cluster_name,
                           'persistent_data': pd,
                           'bucket_name': bucket.name}
                # Look for cluster's placement too
                if include_placement:
                    placement = self._find_placement(cluster_name, cluster)
                    cluster['placement'] = placement
                clusters.append(cluster)
        response['clusters'] = clusters
        return response

    def get_cluster_pd(self, cluster_name):
        """
        Return *persistent data* (as a dict) associated with a cluster with the
        given ``cluster_name``. If a cluster with the given name is not found,
        return an empty dict.

        .. versionadded:: 0.3
        """
        cluster = {}
        clusters = self.get_clusters_pd().get('clusters', [])
        for c in clusters:
            if c['cluster_name'] == cluster_name:
                cluster = c
                break
        return cluster

    def connect_ec2(self, a_key, s_key, cloud=None):
        """
        Create and return an EC2-compatible connection object for the given cloud.

        See ``_get_cloud_info`` method for more details on the requirements for
        the ``cloud`` parameter. If no value is provided, the class field is used.
        """
        if cloud is None:
            cloud = self.cloud
        ci = self._get_cloud_info(cloud)
        r = RegionInfo(name=ci['region_name'], endpoint=ci['region_endpoint'])
        ec2_conn = boto.connect_ec2(aws_access_key_id=a_key,
                                    aws_secret_access_key=s_key,
                                    is_secure=ci['is_secure'],
                                    region=r,
                                    port=ci['ec2_port'],
                                    path=ci['ec2_conn_path'],
                                    validate_certs=False)
        return ec2_conn

    def connect_s3(self, a_key, s_key, cloud=None):
        """
        Create and return an S3-compatible connection object for the given cloud.

        See ``_get_cloud_info`` method for more details on the requirements for
        the ``cloud`` parameter. If no value is provided, the class field is used.
        """
        if cloud is None:
            cloud = self.cloud
        ci = self._get_cloud_info(cloud)
        if ci['cloud_type'] == 'amazon':
            calling_format = SubdomainCallingFormat()
        else:
            calling_format = OrdinaryCallingFormat()
        s3_conn = S3Connection(
            aws_access_key_id=a_key, aws_secret_access_key=s_key,
            is_secure=ci['is_secure'], port=ci['s3_port'], host=ci['s3_host'],
            path=ci['s3_conn_path'], calling_format=calling_format)
        return s3_conn

    def connect_vpc(self, a_key, s_key, cloud=None):
        """
        Establish a connection to the VPC service.

        TODO: Make this work with non-default clouds as well.
        """
        if cloud is None:
            cloud = self.cloud
        ci = self._get_cloud_info(cloud)
        r = RegionInfo(name=ci['region_name'], endpoint=ci['region_endpoint'])
        vpc_conn = boto.connect_vpc(
            aws_access_key_id=a_key,
            aws_secret_access_key=s_key,
            is_secure=ci['is_secure'],
            region=r,
            port=ci['ec2_port'],
            path=ci['ec2_conn_path'],
            validate_certs=False)
        return vpc_conn

    def _compose_user_data(self, user_provided_data):
        """
        A convenience method used to compose and properly format the user data
        required when requesting an instance.

        ``user_provided_data`` is the data provided by a user required to identify
        a cluster and user other user requirements.
        """
        form_data = {}
        # Do not include the following fields in the user data but do include
        # any 'advanced startup fields' that might be added in the future
        excluded_fields = ['sg_name', 'image_id', 'instance_id', 'kp_name',
                           'cloud', 'cloud_type', 'public_dns', 'cidr_range',
                           'kp_material', 'placement', 'flavor_id']
        for key, value in user_provided_data.items():
            if key not in excluded_fields:
                form_data[key] = value
        # If the following user data keys are empty, do not include them in the
        # request user data
        udkeys = [
            'post_start_script_url',
            'worker_post_start_script_url',
            'bucket_default',
            'share_string']
        for udkey in udkeys:
            if udkey in form_data and form_data[udkey] == '':
                del form_data[udkey]
        # If bucket_default was not provided, add a default value to the user data
        # (missing value does not play nicely with CloudMan's ec2autorun.py)
        if not form_data.get(
                'bucket_default', None) and self.cloud.bucket_default:
            form_data['bucket_default'] = self.cloud.bucket_default
        # Reuse the ``password`` for the ``freenxpass`` user data option
        if 'freenxpass' not in form_data and 'password' in form_data:
            form_data['freenxpass'] = form_data['password']
        # Convert form_data into the YAML format
        ud = yaml.dump(form_data, default_flow_style=False, allow_unicode=False)
        # Also include connection info about the selected cloud
        ci = self._get_cloud_info(self.cloud, as_str=True)
        return ud + "\n" + ci

    def _get_cloud_info(self, cloud, as_str=False):
        """
        Get connection information about a given cloud
        """
        ci = {}
        ci['cloud_type'] = cloud.cloud_type
        ci['region_name'] = cloud.region_name
        ci['region_endpoint'] = cloud.region_endpoint
        ci['is_secure'] = cloud.is_secure
        ci['ec2_port'] = cloud.ec2_port if cloud.ec2_port != '' else None
        ci['ec2_conn_path'] = cloud.ec2_conn_path
        # Include cidr_range only if not empty
        if cloud.cidr_range != '':
            ci['cidr_range'] = cloud.cidr_range
        ci['s3_host'] = cloud.s3_host
        ci['s3_port'] = cloud.s3_port if cloud.s3_port != '' else None
        ci['s3_conn_path'] = cloud.s3_conn_path
        if as_str:
            ci = yaml.dump(ci, default_flow_style=False, allow_unicode=False)
        return ci

    def _get_volume_placement(self, vol_id):
        """
        Returns the placement of a volume (or None, if it cannot be determined)
        """
        try:
            vol = self.ec2_conn.get_all_volumes(volume_ids=[vol_id])
        except EC2ResponseError as ec2e:
            bioblend.log.error(f"EC2ResponseError querying for volume {vol_id}: {ec2e}")
            vol = None
        if vol:
            return vol[0].zone
        else:
            bioblend.log.error("Requested placement of a volume '%s' that does not exist.", vol_id)
            return None

    def _find_placement(self, cluster_name, cluster=None):
        """
        Find a placement zone for a cluster with the name ``cluster_name``.

        By default, this method will search for and fetch given cluster's
        *persistent data*; alternatively, *persistent data* can be provided via
        the ``cluster`` parameter. This dict needs to have ``persistent_data``
        key with the contents of cluster's *persistent data*.
        If the cluster or the volume associated with the cluster cannot be found,
        cluster placement is set to ``None``.

        :rtype: dict
        :return: A dictionary with ``placement`` and ``error`` keywords.

        .. versionchanged:: 0.7.0
            The return value changed from a list to a dictionary.
        """
        placement = None
        response = {'placement': placement, 'error': None}
        cluster = cluster or self.get_cluster_pd(cluster_name)
        if cluster and 'persistent_data' in cluster:
            pd = cluster['persistent_data']
            try:
                if 'placement' in pd:
                    response['placement'] = pd['placement']
                elif 'data_filesystems' in pd:
                    # We have v1 format persistent data so get the volume first and
                    # then the placement zone
                    vol_id = pd['data_filesystems']['galaxyData'][0]['vol_id']
                    response['placement'] = self._get_volume_placement(vol_id)
                elif 'filesystems' in pd:
                    # V2 format.
                    for fs in [fs for fs in pd['filesystems'] if fs.get(
                            'kind', None) == 'volume' and 'ids' in fs]:
                        # All volumes must be in the same zone
                        vol_id = fs['ids'][0]
                        response['placement'] = self._get_volume_placement(
                            vol_id)
                        # No need to continue to iterate through
                        # filesystems, if we found one with a volume.
                        break
            except Exception as exc:
                response['error'] = \
                    f"Exception while finding placement for cluster '{cluster_name}'. This can indicate malformed instance data. Or that this method is broken: {exc}"
                bioblend.log.error(response['error'])
                response['placement'] = None
        else:
            bioblend.log.debug(
                f"Insufficient info about cluster {cluster_name} to get placement."
            )
        return response

    def find_placements(
            self, ec2_conn, instance_type, cloud_type, cluster_name=None):
        """
        Find a list of placement zones that support the specified instance type.

        If ``cluster_name`` is given and a cluster with the given name exist,
        return a list with only one entry where the given cluster lives.

        Searching for available zones for a given instance type is done by
        checking the spot prices in the potential availability zones for
        support before deciding on a region:
        http://blog.piefox.com/2011/07/ec2-availability-zones-and-instance.html

        Note that, currently, instance-type based zone selection applies only to
        AWS. For other clouds, all the available zones are returned (unless a
        cluster is being recreated, in which case the cluster's placement zone is
        returned sa stored in its persistent data.

        :rtype: dict
        :return: A dictionary with ``zones`` and ``error`` keywords.

        .. versionchanged:: 0.3
            Changed method name from ``_find_placements`` to ``find_placements``.
            Also added ``cluster_name`` parameter.

        .. versionchanged:: 0.7.0
            The return value changed from a list to a dictionary.
        """
        # First look for a specific zone a given cluster is bound to
        zones = []
        response = {'zones': zones, 'error': None}
        if cluster_name:
            placement = self._find_placement(cluster_name)
            if placement.get('error'):
                response['error'] = placement['error']
                return response
            response['zones'] = placement.get('placement', [])
        # If placement is not found, look for a list of available zones
        if not response['zones']:
            in_the_past = datetime.datetime.now() - datetime.timedelta(hours=1)
            back_compatible_zone = "us-east-1e"
            for zone in [
                    z for z in ec2_conn.get_all_zones() if z.state == 'available']:
                # Non EC2 clouds may not support get_spot_price_history
                if instance_type is None or cloud_type != 'ec2':
                    zones.append(zone.name)
                elif ec2_conn.get_spot_price_history(instance_type=instance_type,
                                                     end_time=in_the_past.isoformat(),
                                                     availability_zone=zone.name):
                    zones.append(zone.name)
            # Higher-lettered zones seem to have more availability currently
            zones.sort(reverse=True)
            if back_compatible_zone in zones:
                zones = [back_compatible_zone] + \
                    [z for z in zones if z != back_compatible_zone]
            if len(zones) == 0:
                response['error'] = f"Did not find availabilty zone for {instance_type}"
                bioblend.log.error(response['error'])
                zones.append(back_compatible_zone)
        return response

    def _checkURL(self, url):
        """
        Check if the ``url`` is *alive* (i.e., remote server returns code 200(OK)
        or 401 (unauthorized)).
        """
        try:
            p = urlparse(url)
            h = HTTPConnection(p[1])
            h.putrequest('HEAD', p[2])
            h.endheaders()
            r = h.getresponse()
            # CloudMan UI is pwd protected so include 401
            return r.status in (200, 401)
        except Exception:
            # No response or no good response
            return False