import json
import time

import pymysql
from common.meta.common import create_tag_to_target, get_default_geog_tag
from django.conf import settings
from meta.public.common import field_allowed_roles
from meta.utils.basicapi import BasicApi
from meta.utils.common import paged_sql

MYSQL_BACKEND_TYPE = "mysql"
ATLAS_BACKEND_TYPE = "atlas"
DEFAULT_DATE_FORMAT = "%Y-%m-%d %H:%i:%s"

TAG_TYPE_RESULT_TABLE = "result_table"
TAG_TYPE_RAW_DATA = "raw_data"
TAG_TYPE_PROJECT = "project"
TAG_TYPE_DATA_PROCESSING = "data_processing"
TAG_TYPE_DATA_PROCESSING_RELATION = "data_processing_relation"
TAG_TYPE_DATA_TRANSFERRING = "data_transferring"
TAG_TYPE_CLUSTER_GROUP = "cluster_group"

VIRTUAL_BELONG_TO_FIELD = "belong_to"

def get_tag_params(params):
    tags = params.pop("tags", [])
    return tags

def create_tag_to_result_table(tags, bk_biz_id, project_id, *result_table_ids):
    tag_to_target(TAG_TYPE_RESULT_TABLE, tags, bk_biz_id, project_id, *result_table_ids)

def create_tag_to_raw_data(tags, bk_biz_id, *raw_data_ids):
    tag_to_target(TAG_TYPE_RAW_DATA, tags, bk_biz_id, None, *raw_data_ids)

def create_tag_to_data_processing(tags, *raw_data_ids):
    tag_to_target(TAG_TYPE_DATA_PROCESSING, tags, None, None, *raw_data_ids)

def create_tag_to_dpr(tags, *data_processing_ids):
    tag_to_target(TAG_TYPE_DATA_PROCESSING_RELATION, tags, None, None, *data_processing_ids)

def create_tag_to_data_transferring(tags, *raw_data_ids):
    tag_to_target(TAG_TYPE_DATA_TRANSFERRING, tags, None, None, *raw_data_ids)

def create_tag_to_project(tags, *project_id_ids):
    tag_to_target(TAG_TYPE_PROJECT, tags, None, None, *project_id_ids)

def create_tag_to_cluster_group(tags, *cluster_group_ids):
    tag_to_target(TAG_TYPE_CLUSTER_GROUP, tags, None, None, *cluster_group_ids)

def tag_to_target(target_type, tags, bk_biz_id, project_id, *target_ids):
    targets = []
    for target_id in target_ids:
        targets.append((target_type, target_id))
    create_tag_to_target(targets, tags, target_exists=False, bk_biz_id=bk_biz_id, project_id=project_id)

def add_manage_tag_to_project(result):
    return manage_tag_info(TAG_TYPE_PROJECT, result, "project_id")

def add_manage_tag_to_result_table(result):
    return manage_tag_info(TAG_TYPE_RESULT_TABLE, result, "result_table_id")

def add_manage_tag_to_data_processing(result):
    return manage_tag_info(TAG_TYPE_DATA_PROCESSING, result, "processing_id")

def add_manage_tag_to_data_transferring(result):
    return manage_tag_info(TAG_TYPE_DATA_TRANSFERRING, result, "transferring_id")

def add_manage_tag_to_cluster_group(result):
    return manage_tag_info(TAG_TYPE_CLUSTER_GROUP, result, "cluster_group_id")

def gen_managed_tag_dict(sql):
    tag_dict = {}
    tag_id_obj_dict = {}
    tag_list = BasicApi.get_complex_search(sql)
    # 转换标签,为每一个tag添加它所属的大类,并虚拟命名为:belong_to字段
    if tag_list:
        # 构建标签关系树
        for tag_obj in tag_list:
            tag_id = tag_obj["id"]
            parent_id = tag_obj["parent_id"]
            tag_code = tag_obj["code"]
            if parent_id == 0:  # 根节点
                tag_obj[VIRTUAL_BELONG_TO_FIELD] = tag_code
            tag_id_obj_dict[tag_id] = tag_obj
        # 从树中检索继承关系
        for tag_obj in tag_list:
            if VIRTUAL_BELONG_TO_FIELD not in tag_obj:
                parent_id = tag_obj["parent_id"]
                while True:
                    parent_dict = tag_id_obj_dict.get(parent_id)
                    if not parent_dict:
                        break
                    if VIRTUAL_BELONG_TO_FIELD in parent_dict:
                        tag_obj[VIRTUAL_BELONG_TO_FIELD] = parent_dict[VIRTUAL_BELONG_TO_FIELD]
                        break
                    parent_id = parent_dict["parent_id"]
        # 更新标签信息
        for tag_obj in tag_list:
            tag_dict[tag_obj["code"]] = {
                "code": tag_obj["code"],
                "alias": tag_obj["alias"],
                VIRTUAL_BELONG_TO_FIELD: tag_obj.get(VIRTUAL_BELONG_TO_FIELD),
            }
    return tag_dict

def gen_managed_tag_target_dict(sql, tag_dict):
    tag_target_dict = {}
    tag_target_list = BasicApi.get_complex_search(sql)
    if tag_target_list:
        for tag_target_obj in tag_target_list:
            target_id = tag_target_obj["target_id"]
            tag_code = tag_target_obj["tag_code"]
            ttt_dict = tag_dict.get(tag_code)
            if ttt_dict:
                belong_to = ttt_dict[VIRTUAL_BELONG_TO_FIELD]
                cp_ttt_dict = ttt_dict.copy()
                cp_ttt_dict.pop(VIRTUAL_BELONG_TO_FIELD)
                tg_dict = tag_target_dict.get(target_id)
                if not tg_dict:
                    tag_target_dict[target_id] = {belong_to: [cp_ttt_dict]}
                elif belong_to in tg_dict:
                    tg_dict[belong_to].append(cp_ttt_dict)
                else:
                    tg_dict[belong_to] = [cp_ttt_dict]
    return tag_target_dict

def manage_tag_info(target_type, result, data_set_name):  # 给返回结果添加上管理标签信息
    if not result:
        return

    if settings.ENABLED_TDW:
        from meta.extend.tencent.lol.utils import mixin_lol_manage_tag_info

    default_geog_area_dict = None
    if not settings.MULTI_GEOG_AREA:  # 非多地域,直接返回默认标签
        dgt_dict = get_default_geog_tag()
        if dgt_dict:
            default_geog_area_dict = {"code": dgt_dict["code"], "alias": dgt_dict["alias"]}
        if isinstance(result, list):
            for obj in result:
                obj["tags"] = {"manage": {"geog_area": [default_geog_area_dict]}}
                if (
                    settings.ENABLED_TDW
                    and target_type == TAG_TYPE_PROJECT
                    and obj.get("project_id", 0) == settings.LOL_BILLING_PRJ_ID
                ):
                    mixin_lol_manage_tag_info(obj)
        elif isinstance(result, dict):
            result["tags"] = {"manage": {"geog_area": [default_geog_area_dict]}}
            if (
                settings.ENABLED_TDW
                and target_type == TAG_TYPE_PROJECT
                and result.get("project_id", 0) == settings.LOL_BILLING_PRJ_ID
            ):
                mixin_lol_manage_tag_info(result)
        return

    target_id_list = []
    if isinstance(result, list):
        for obj in result:
            if data_set_name in obj:
                target_id_list.append(obj[data_set_name])
    elif isinstance(result, dict) and data_set_name in result:
        target_id_list.append(result[data_set_name])
    if not target_id_list:
        return

    # 生成tag字典
    tag_sql = "select id,code,alias,parent_id from tag where active=1 and tag_type='manage' order by parent_id asc"
    tag_dict = gen_managed_tag_dict(tag_sql)
    # 生成tag_target字典 {target_id:{"geog_area":[{"tag_code":"xxx","tag_name":"xxx"}]}}
    where_cond = ",".join("'" + escape_string(str(target_id)) + "'" for target_id in target_id_list)
    tag_target_sql = (
        "select id,target_id,target_type,tag_code from tag_target where active=1 "
        " and tag_type='manage' and tag_code=source_tag_code and target_type='{}' and target_id in({})".format(
            escape_string(target_type), where_cond
        )
    )
    tag_target_dict = gen_managed_tag_target_dict(tag_target_sql, tag_dict)

    # 添加标签信息到结果中
    if isinstance(result, list):
        for obj in result:
            data_set_id = str(obj[data_set_name])
            geog_area_dict = tag_target_dict.get(data_set_id)
            if geog_area_dict:
                if "geog_area" not in geog_area_dict:
                    geog_area_dict["geog_area"] = [default_geog_area_dict]
                obj["tags"] = {"manage": geog_area_dict}
            elif default_geog_area_dict:
                obj["tags"] = {"manage": {"geog_area": [default_geog_area_dict]}}
    elif isinstance(result, dict):
        data_set_id = str(result[data_set_name])
        geog_area_dict = tag_target_dict.get(data_set_id)
        if geog_area_dict:
            if "geog_area" not in geog_area_dict:
                geog_area_dict["geog_area"] = [default_geog_area_dict]
            result["tags"] = {"manage": geog_area_dict}
        elif default_geog_area_dict:
            result["tags"] = {"manage": {"geog_area": [default_geog_area_dict]}}

def escape_string(s):
    return pymysql.converters.escape_string(s)

def cover_storage_detail(cluster_dict, channel_dict, cluster_id_in_cond=None, channel_id_in_cond=None):
    cluster_sql, channel_sql, union_sql = None, None, ""
    if cluster_id_in_cond:
        cluster_id_in_cond = cluster_id_in_cond.lstrip(",").rstrip(",")
        cluster_sql = """select id,cluster_type,'1' as comm from storage_cluster_config where id in ({})""".format(
            cluster_id_in_cond
        )
    if channel_id_in_cond:
        channel_id_in_cond = channel_id_in_cond.lstrip(",").rstrip(",")
        channel_sql = (
            """select id,cluster_type,'2' as comm from databus_channel_cluster_config where id in({})""".format(
                channel_id_in_cond
            )
        )
    if cluster_sql:
        union_sql += cluster_sql
    if channel_sql:
        if union_sql:
            union_sql += " union all "
        union_sql += channel_sql

    if union_sql:
        union_result = BasicApi.get_complex_search(union_sql)
        if union_result:
            for union_obj in union_result:
                comm = union_obj["comm"]
                id_key = union_obj["id"]
                if comm == "1":
                    cluster_dict[id_key] = union_obj
                elif comm == "2":
                    channel_dict[id_key] = union_obj

def get_result_table_storages_v3(in_cond, cluster_type=None, need_storage_detail=True):
    if need_storage_detail and in_cond:
        storage_sql = """select id,physical_table_name,updated_by,generate_type,storage_config,active,priority,
        created_by,data_type,date_format(updated_at,'{0}') updated_at,expires,
        date_format(created_at,'{0}') created_at,description,storage_cluster_config_id,previous_cluster_name,
        storage_channel_id,result_table_id from storage_result_table where active=1
        and result_table_id in ({1})""".format(
            DEFAULT_DATE_FORMAT, in_cond
        )
    elif need_storage_detail and not in_cond:
        storage_sql = """select id,physical_table_name,updated_by,generate_type,storage_config,active,priority,
        created_by,data_type,date_format(updated_at,'{0}') updated_at,expires,
        date_format(created_at,'{0}') created_at,description,storage_cluster_config_id,previous_cluster_name,
        storage_channel_id,result_table_id from storage_result_table where active=1 """.format(
            DEFAULT_DATE_FORMAT
        )
    elif not need_storage_detail and in_cond:  # 不需要storage详情
        storage_sql = """select id,result_table_id,storage_cluster_config_id,storage_channel_id,
        active from storage_result_table where active=1 and result_table_id in ({})""".format(
            in_cond
        )
    else:
        storage_sql = """select id,result_table_id,storage_cluster_config_id,storage_channel_id,
        active from storage_result_table where active=1"""

    storage_rt_id_dict = {}
    cluster_id_in_cond, channel_id_in_cond = "", ""
    storage_result = BasicApi.get_complex_search(storage_sql)
    if not storage_result:
        return storage_rt_id_dict

    # 获取存储cluster id条件列表 & 总线channel id条件列表
    storage_result = parse_field_to_boolean(storage_result, "active")
    for storage_obj in storage_result:
        storage_cluster_config_id = storage_obj["storage_cluster_config_id"]
        storage_channel_id = storage_obj["storage_channel_id"]
        if storage_cluster_config_id is not None:
            if not cluster_id_in_cond:
                cluster_id_in_cond += ","
            if "," + str(storage_cluster_config_id) + "," not in cluster_id_in_cond:  # id去重
                cluster_id_in_cond += str(storage_cluster_config_id) + ","

        if storage_channel_id is not None:
            if not channel_id_in_cond:
                channel_id_in_cond += ","
            if "," + str(storage_channel_id) + "," not in channel_id_in_cond:
                channel_id_in_cond += str(storage_channel_id) + ","

    # 获取存储集群信息
    cluster_dict = {}
    if need_storage_detail and cluster_id_in_cond:
        cluster_id_in_cond = cluster_id_in_cond.lstrip(",").rstrip(",")
        cluster_sql = """select id,id as storage_cluster_config_id,cluster_name,cluster_type,priority,version,
        expires,connection_info,belongs_to,cluster_group from storage_cluster_config where id in ({})""".format(
            cluster_id_in_cond
        )
        if cluster_type:
            cluster_sql += " and cluster_type='" + escape_string(cluster_type) + "'"
        cluster_result = BasicApi.get_complex_search(cluster_sql)
        if cluster_result:
            for cluster_obj in cluster_result:
                storage_cluster_config_id = cluster_obj["storage_cluster_config_id"]
                cluster_dict[storage_cluster_config_id] = cluster_obj

    # 获取总线集群信息
    channel_dict = {}
    if need_storage_detail and channel_id_in_cond:
        channel_id_in_cond = channel_id_in_cond.lstrip(",").rstrip(",")
        channel_sql = """select id,id as channel_cluster_config_id,cluster_backup_ips,attribute,zk_port,
        description,cluster_name,cluster_type,cluster_role,priority,cluster_domain,cluster_port,active,
        zk_domain,zk_root_path from databus_channel_cluster_config where id in ({})""".format(
            channel_id_in_cond
        )
        if cluster_type:
            channel_sql += " and cluster_type='" + escape_string(cluster_type) + "'"
        channel_result = BasicApi.get_complex_search(channel_sql)
        channel_result = parse_field_to_boolean(channel_result, "active")
        if channel_result:
            for channel_obj in channel_result:
                channel_cluster_config_id = channel_obj["channel_cluster_config_id"]
                channel_dict[channel_cluster_config_id] = channel_obj

    if not need_storage_detail:  # 不需要storage详情, 用概要信息覆盖对应结果
        cover_storage_detail(
            cluster_dict, channel_dict, cluster_id_in_cond=cluster_id_in_cond, channel_id_in_cond=channel_id_in_cond
        )

    for storage_obj in storage_result:
        storage_rt_id = storage_obj["result_table_id"]
        storage_cluster_config_id = storage_obj["storage_cluster_config_id"]
        storage_channel_id = storage_obj["storage_channel_id"]
        tmp_storage_rt_id_dict = storage_rt_id_dict.get(storage_rt_id, {})
        tmp_cluster_obj = cluster_dict.get(storage_cluster_config_id, {})
        tmp_channel_obj = channel_dict.get(storage_channel_id, {})
        cluster_type = None

        if tmp_cluster_obj:
            cluster_type = tmp_cluster_obj.get("cluster_type")
        elif tmp_channel_obj:
            cluster_type = tmp_channel_obj.get("cluster_type")

        if cluster_type is None:  # 为了处理有cluster_type参数的情况
            continue

        storage_obj["storage_cluster"] = tmp_cluster_obj
        storage_obj["storage_channel"] = tmp_channel_obj
        if need_storage_detail:
            tmp_storage_rt_id_dict[cluster_type] = storage_obj
        else:  # 不需要storage详情
            tmp_storage_rt_id_dict[cluster_type] = {}
        storage_rt_id_dict[storage_rt_id] = tmp_storage_rt_id_dict
    return storage_rt_id_dict

def get_result_table_fields_v3(in_cond, need_result_table_id=False):
    field_sql = "select "
    if need_result_table_id:
        field_sql += "result_table_id,"
    if in_cond:
        field_sql += """ roles,field_type,description,date_format(created_at,'{0}') created_at,is_dimension,created_by,
        date_format(updated_at,'{0}') updated_at,origins,field_alias,field_name,id,field_index,updated_by
        from result_table_field where result_table_id in ({1}) order by result_table_id asc,field_index asc""".format(
            DEFAULT_DATE_FORMAT, in_cond
        )
    else:
        field_sql += """ roles,field_type,description,date_format(created_at,'{0}') created_at,is_dimension,created_by,
        date_format(updated_at,'{0}') updated_at,origins,field_alias,field_name,id,field_index,updated_by
        from result_table_field order by result_table_id asc,field_index asc""".format(
            DEFAULT_DATE_FORMAT,
        )
    field_result = BasicApi.get_complex_search(field_sql)
    field_result = parse_field_to_boolean(field_result, "is_dimension")
    for item in field_result:
        default_roles = {k: False for k in field_allowed_roles}
        roles = None
        try:
            roles = json.loads(item["roles"])
        except Exception:
            pass
        if roles and isinstance(roles, dict):
            default_roles.update(roles)
        item["roles"] = default_roles
    return field_result

def get_result_table_infos_v3(
    bk_biz_ids=None,
    project_id=None,
    result_table_ids=None,
    related=None,
    processing_type=None,
    page=None,
    page_size=None,
    need_storage_detail=True,
    tags=None,
    only_queryable=True,
):
    sql = """select a.result_table_name,a.bk_biz_id,date_format(a.created_at,'{0}') created_at,a.sensitivity,
    a.result_table_name_alias,a.updated_by,a.created_by,a.result_table_id,a.count_freq,a.description,
    date_format(a.updated_at,'{0}') updated_at,a.generate_type,a.result_table_type,a.processing_type,a.project_id,
    a.platform, a.is_managed,a.count_freq_unit,a.data_category,b.project_name from result_table a
    left join project_info b on a.project_id=b.project_id where 1=1""".format(
        DEFAULT_DATE_FORMAT
    )
    where_cond = ""
    if bk_biz_ids:
        where_cond += " and a.bk_biz_id in ({})".format(",".join(escape_string(bk_biz_id) for bk_biz_id in bk_biz_ids))
    if project_id:
        where_cond += " and a.project_id=" + escape_string(project_id)

    # 只查询可见的rt(queryset不可见)
    if only_queryable:
        if processing_type:
            if processing_type == "queryset":
                return []
            where_cond += " and a.processing_type='" + escape_string(processing_type) + "'"
        else:
            where_cond += " and a.processing_type!='queryset'"
    else:
        if processing_type:
            where_cond += " and a.processing_type='" + escape_string(processing_type) + "'"

    if settings.MULTI_GEOG_AREA and tags:  # 非多地域
        where_cond += (
            " and a.result_table_id in(select target_id from tag_target where tag_code=source_tag_code "
            "and active=1 and target_type='result_table' and tag_type='manage' and source_tag_code in ({}))".format(
                ",".join("'" + escape_string(tag) + "'" for tag in tags)
            )
        )

    if result_table_ids:
        page = None
        page_size = None
        where_cond += " and a.result_table_id in("
        where_cond += "".join("'" + escape_string(rt_id) + "'," for rt_id in result_table_ids)
        where_cond = where_cond.rstrip(",") + ") "
    if where_cond:
        sql += where_cond
    query_result = BasicApi.get_complex_search(paged_sql(sql, page=page, page_size=page_size))
    if only_queryable:
        query_result = [
            rt_obj
            for rt_obj in query_result
            if not (rt_obj["processing_type"] == "queryset" and rt_obj["generate_type"] == "system")
        ]
    if related and query_result:
        if not (result_table_ids or bk_biz_ids or project_id or processing_type):
            in_cond = None
        else:
            in_cond = ",".join("'" + escape_string(rt_obj["result_table_id"]) + "'" for rt_obj in query_result)
        field_rt_id_dict, dp_rt_id_dict, storage_rt_id_dict = {}, {}, None
        if "fields" in related:
            # 不允许拉取fields全表
            if not in_cond:
                pass
            else:
                field_result = get_result_table_fields_v3(in_cond, need_result_table_id=True)
                if field_result:
                    for field_obj in field_result:
                        field_rt_id = field_obj["result_table_id"]
                        del field_obj["result_table_id"]
                        field_list = field_rt_id_dict.get(field_rt_id, [])
                        field_list.append(field_obj)
                        field_rt_id_dict[field_rt_id] = field_list

        if "data_processing" in related:
            if not in_cond:
                dp_sql = """select tmp.*,c.project_name from
                (select a.processing_id,a.project_id,a.processing_alias,a.processing_type,a.description,
                a.generate_type,a.platform,a.created_by,date_format(a.created_at,'{0}') created_at,
                a.updated_by,date_format(a.updated_at,'{0}') updated_at,b.data_set_id
                from data_processing a,data_processing_relation b where a.processing_id=b.processing_id
                and b.data_set_type='result_table' and b.data_directing='output')tmp
                left join project_info c on tmp.project_id=c.project_id""".format(
                    DEFAULT_DATE_FORMAT
                )
            else:
                dp_sql = """select tmp.*,c.project_name from
                (select a.processing_id,a.project_id,a.processing_alias,a.processing_type,a.description,
                a.generate_type,a.platform,a.created_by,date_format(a.created_at,'{0}') created_at,
                a.updated_by,date_format(a.updated_at,'{0}') updated_at,b.data_set_id
                from data_processing a,data_processing_relation b where a.processing_id=b.processing_id
                and b.data_set_type='result_table' and b.data_directing='output' and b.data_set_id in({1}))tmp
                left join project_info c on tmp.project_id=c.project_id""".format(
                    DEFAULT_DATE_FORMAT, in_cond
                )
            dp_result = BasicApi.get_complex_search(dp_sql)
            if dp_result:
                for dp_obj in dp_result:
                    data_set_id = dp_obj["data_set_id"]
                    del dp_obj["data_set_id"]
                    dp_rt_id_dict[data_set_id] = dp_obj

        if "storages" in related:
            storage_rt_id_dict = get_result_table_storages_v3(in_cond, need_storage_detail=need_storage_detail)

        for rt_obj in query_result:
            rt_id = rt_obj["result_table_id"]
            if "fields" in related:
                rt_obj["fields"] = field_rt_id_dict.get(rt_id, [])
            if "data_processing" in related:
                rt_obj["data_processing"] = dp_rt_id_dict.get(rt_id, {})
            if "storages" in related:
                rt_obj["storages"] = storage_rt_id_dict.get(rt_id, {})
    return query_result

def get_result_table_geog_area(result_table_id=None):
    sql = None
    if result_table_id:
        sql = """select id, tag_code, tag_type from tag_target where target_id = {}
        and tag_code in ('inland', 'SEA', 'NA') and target_type = 'result_table'""".format(
            result_table_id
        )
    return BasicApi.get_complex_search(sql)

def parse_field_to_boolean(obj, field_name):
    if isinstance(obj, list):
        for single in obj:
            if single[field_name] == 1:
                single[field_name] = True
            elif single[field_name] == 0:
                single[field_name] = False
    elif isinstance(obj, dict):
        if obj[field_name] == 1:
            obj[field_name] = True
        elif obj[field_name] == 0:
            obj[field_name] = False
    return obj

def get_data_transfers_v3(where_cond, page=None, page_size=None):  # backend=mysql
    sql = """
    select a.project_id,b.project_name,a.transferring_id, a.transferring_alias, a.transferring_type, a.generate_type,
    a.created_by, date_format(a.created_at,'{0}') created_at, a.updated_by, date_format(a.updated_at,'{0}') updated_at,
    a.description from data_transferring a left join project_info b on a.project_id=b.project_id where 1=1 """.format(
        DEFAULT_DATE_FORMAT
    )
    if where_cond:
        sql += where_cond
    query_result = BasicApi.get_complex_search(paged_sql(sql, page=page, page_size=page_size))
    if query_result:
        dp_relation_sql = """
        select transferring_id,data_directing,data_set_id,data_set_type,storage_type,channel_cluster_config_id,
        storage_cluster_config_id from data_transferring_relation where transferring_id in(
        """
        for dp_obj in query_result:
            dp_obj["inputs"] = []
            dp_obj["outputs"] = []
        dp_relation_sql += "".join("'" + escape_string(dp_obj["transferring_id"]) + "'," for dp_obj in query_result)
        dp_relation_sql = dp_relation_sql[: len(dp_relation_sql) - 1] + ") order by transferring_id"
        dp_relation_result = BasicApi.get_complex_search(dp_relation_sql)
        dp_relation_dict = {}
        for dp_relation_obj in dp_relation_result:
            dp_id = dp_relation_obj["transferring_id"]
            data_directing = dp_relation_obj["data_directing"]
            del dp_relation_obj["transferring_id"]
            del dp_relation_obj["data_directing"]
            if dp_id in dp_relation_dict:
                if data_directing == "input":
                    dp_relation_dict[dp_id]["inputs"].append(dp_relation_obj)
                elif data_directing == "output":
                    dp_relation_dict[dp_id]["outputs"].append(dp_relation_obj)
            else:
                tmp_dict = {}
                inputs_list, outputs_list = [], []
                if data_directing == "input":
                    inputs_list.append(dp_relation_obj)
                elif data_directing == "output":
                    outputs_list.append(dp_relation_obj)
                tmp_dict["inputs"] = inputs_list
                tmp_dict["outputs"] = outputs_list
                dp_relation_dict[dp_id] = tmp_dict

        for dp_obj in query_result:
            dp_id = dp_obj["transferring_id"]
            dp_relation_obj = dp_relation_dict.get(dp_id)
            if dp_relation_obj:
                dp_obj["inputs"] = dp_relation_obj["inputs"]
                dp_obj["outputs"] = dp_relation_obj["outputs"]

    return query_result

def get_data_processings_v3(where_cond, page=None, page_size=None):  # backend=mysql
    sql = """
    select a.processing_id,a.project_id,b.project_name,a.processing_alias,a.processing_type,a.generate_type,
    a.created_by,date_format(a.created_at,'{0}') created_at,a.updated_by,date_format(a.updated_at,'{0}') updated_at,
    a.description,a.platform from data_processing a left join project_info b on a.project_id=b.project_id
    where 1=1 """.format(
        DEFAULT_DATE_FORMAT
    )
    if where_cond:
        sql += where_cond
    query_result = BasicApi.get_complex_search(paged_sql(sql, page=page, page_size=page_size))
    if query_result:
        dp_relation_sql = """
        select processing_id,data_directing,data_set_id,data_set_type,storage_type,channel_cluster_config_id,
        storage_cluster_config_id from data_processing_relation where processing_id in(
        """
        for dp_obj in query_result:
            dp_obj["inputs"] = []
            dp_obj["outputs"] = []
        dp_relation_sql += "".join("'" + escape_string(dp_obj["processing_id"]) + "'," for dp_obj in query_result)
        dp_relation_sql = dp_relation_sql[: len(dp_relation_sql) - 1] + ") order by processing_id"
        dp_relation_result = BasicApi.get_complex_search(dp_relation_sql)
        dp_relation_dict = {}
        for dp_relation_obj in dp_relation_result:
            dp_id = dp_relation_obj["processing_id"]
            data_directing = dp_relation_obj["data_directing"]
            del dp_relation_obj["processing_id"]
            del dp_relation_obj["data_directing"]
            if dp_id in dp_relation_dict:
                if data_directing == "input":
                    dp_relation_dict[dp_id]["inputs"].append(dp_relation_obj)
                elif data_directing == "output":
                    dp_relation_dict[dp_id]["outputs"].append(dp_relation_obj)
            else:
                tmp_dict = {}
                inputs_list, outputs_list = [], []
                if data_directing == "input":
                    inputs_list.append(dp_relation_obj)
                elif data_directing == "output":
                    outputs_list.append(dp_relation_obj)
                tmp_dict["inputs"] = inputs_list
                tmp_dict["outputs"] = outputs_list
                dp_relation_dict[dp_id] = tmp_dict

        for dp_obj in query_result:
            dp_id = dp_obj["processing_id"]
            dp_relation_obj = dp_relation_dict.get(dp_id)
            if dp_relation_obj:
                dp_obj["inputs"] = dp_relation_obj["inputs"]
                dp_obj["outputs"] = dp_relation_obj["outputs"]

    return query_result

def get_project_info_sql():
    sql = """select project_name,date_format(deleted_at,'{0}') deleted_at,description,
    date_format(created_at,'{0}') created_at,date_format(updated_at,'{0}') updated_at,created_by,deleted_by,bk_app_code,
    active,project_id,updated_by from project_info where 1=1""".format(
        DEFAULT_DATE_FORMAT
    )
    return sql

def parse_data_category_result(ret_result):
    parse_result = []
    tmp_result = []
    not_visible_dict = {}  # visible=0的id与parent_id
    if ret_result:
        ret_result.sort(key=lambda l: (l["parent_id"], l["seq_index"]), reverse=False)
        for ret_dict in ret_result:
            id_val = ret_dict["id"]
            active = ret_dict["active"]
            visible = ret_dict["visible"]
            parent_id = ret_dict["parent_id"]
            if not active:  # false
                continue
            if not visible:  # false
                not_visible_dict[id_val] = parent_id
                continue
            tmp_result.append(ret_dict)

    if tmp_result:
        for ret_dict in tmp_result:
            parent_id = ret_dict["parent_id"]
            if parent_id in not_visible_dict:
                p_parent_id = not_visible_dict[parent_id]
                while p_parent_id in not_visible_dict:
                    p_parent_id = not_visible_dict[p_parent_id]
                ret_dict["parent_id"] = p_parent_id

    if tmp_result:
        parent_id_dict = {}  # parent_id:value_list

        for ret_dict in tmp_result:
            parent_id = ret_dict["parent_id"]
            if parent_id in parent_id_dict:
                parent_id_dict[parent_id].append(ret_dict)
            else:
                parent_id_dict[parent_id] = [ret_dict]

        # for value in parent_id_dict.values():
        #     if value:
        #         value.sort(key=field_index_seq_index_sort, reverse=False)

        for ret_dict in tmp_result:
            id_val = ret_dict["id"]
            parent_id = ret_dict["parent_id"]
            ret_dict["sub_list"] = parent_id_dict[id_val] if id_val in parent_id_dict else []
            if parent_id == 0:
                parse_result.append(ret_dict)

    # if parse_result:
    #     parse_result.sort(key=field_index_seq_index_sort, reverse=False)
    return parse_result

def get_filter_attrs(related_filter, need_type_name):
    filter_attr_name = None
    filter_attr_value = None
    if related_filter:
        filter_obj = json.loads(related_filter)
        filter_type = filter_obj.get("type")
        if filter_type == need_type_name:
            filter_attr_name = filter_obj.get("attr_name", "")
            filter_attr_value = filter_obj.get("attr_value", "")
    return filter_attr_name, filter_attr_value

# Atlas公共字段
ATLAS_FIELD_ID = "id"
ATLAS_FIELD_NAME = "name"
ATLAS_FIELD_QUALIFIED_NAME = "qualifiedName"
ATLAS_FIELD_OWNER = "owner"
ATLAS_FIELD_DESCRIPTION = "description"
ATLAS_FIELD_INPUTS = "inputs"
ATLAS_FIELD_OUTPUTS = "outputs"

# Atlas状态
ATLAS_FIELD_ACTIVE = "ACTIVE"

def get_attrs_v2(obj, batch_guid_dict):
    if isinstance(obj, dict) and obj:
        guid = obj["guid"]
        obj_attrs = batch_guid_dict.get(guid)
        if obj_attrs:
            return obj_attrs
    return {}

def parse_storage_to_dict_v2(storage_list, batch_guid_dict, result_table_id):
    res_dict = {}
    for attrs in storage_list:
        # cluster_type = attrs['cluster_type'] # cluster_type 字段已去掉
        # storage_cluster 和 storage_channel 只会一个有值
        active = attrs.get("active")
        if active is not None and not active:  # 软删除的
            continue

        storage_cluster = attrs["storage_cluster"]
        storage_channel = attrs["storage_channel"]
        cluster_type = None

        if storage_channel:
            cluster_type = "kafka"

        storage_cluster_attrs = get_attrs_v2(storage_cluster, batch_guid_dict)
        if storage_cluster_attrs:
            storage_cluster_attrs["storage_cluster_config_id"] = storage_cluster_attrs.get("id")
        del_redundance_key(storage_cluster_attrs, "storage_configs", "storage_config", "dp_characters", "dt_characters")
        del_common_unuse_fields(storage_cluster_attrs)
        del_common_unuse_fields2(storage_cluster_attrs)
        del_common_unuse_fields3(storage_cluster_attrs)
        attrs["storage_cluster"] = storage_cluster_attrs
        if storage_cluster_attrs:
            cluster_type = storage_cluster_attrs["cluster_type"]

        storage_channel_obj = get_attrs_v2(storage_channel, batch_guid_dict)
        if storage_channel_obj:
            storage_channel_obj["channel_cluster_config_id"] = storage_channel_obj.get("id")
        del_redundance_key(
            storage_channel_obj,
            "storage_config",
            "storage_configs",
            ATLAS_FIELD_NAME,
            ATLAS_FIELD_OWNER,
            "dp_characters",
            "dt_characters",
        )
        del_common_unuse_fields(storage_channel_obj)
        del_common_unuse_fields2(storage_channel_obj)
        attrs["storage_channel"] = storage_channel_obj
        del_redundance_key(attrs, "result_tables", "result_table", ATLAS_FIELD_NAME, ATLAS_FIELD_OWNER)

        # attrs['result_table_id'] = result_table_id
        res_dict[cluster_type] = attrs
    return res_dict

def parse_lineage_attrs(entity_obj):
    type_name = entity_obj["typeName"]
    entity_attrs = entity_obj["attributes"]
    prefix_name = None
    field_name = None
    ret_type_name = None
    if type_name == "ResultTable":
        prefix_name = "result_table_"
        field_name = "result_table_id"
        ret_type_name = "result_table"
    elif type_name == "DataProcess":
        prefix_name = "data_processing_"
        field_name = "processing_id"
        ret_type_name = "data_processing"
    elif type_name == "RawData":
        prefix_name = "raw_data_"
        field_name = "id"
        ret_type_name = "raw_data"
    qualified_name = str(entity_attrs[field_name])
    name = prefix_name + qualified_name
    return name, ret_type_name, qualified_name

def parse_mysql_lineage_attrs(entity_obj):
    type_name = entity_obj["type"]
    qualified_name = entity_obj["qualified_name"]
    return type_name + "_" + qualified_name, type_name, qualified_name

def parse_mysql_lineage_node_attrs(entity_obj):
    type_name = entity_obj["type"]
    qualified_name = entity_obj["qualified_name"]
    extra = entity_obj.get("extra", {})
    return type_name + "_" + qualified_name, type_name, qualified_name, extra

def get_result_table_lineage_info(params):
    res_obj = {}
    res_data = BasicApi.get_lineage(params)
    backend_type = res_data.get("backend_type", None)
    if backend_type and backend_type in ("mysql", "dgraph"):
        lineage = res_data["lineage"]
        if lineage:
            mysql_criteria = lineage["criteria"]
            mysql_nodes = lineage["nodes"]
            mysql_relations = lineage["relations"]
            res_obj["depth"] = mysql_criteria["depth"]
            res_obj["direction"] = mysql_criteria["direction"]
            res_relations = []
            nodes = {}
            if mysql_relations:
                for relation in mysql_relations:
                    fm = relation["from"]
                    to = relation["to"]
                    from_name, _, _ = parse_mysql_lineage_attrs(fm)
                    to_name, _, _ = parse_mysql_lineage_attrs(to)
                    res_relations.append({"from": from_name, "to": to_name, "status": "ACTIVE"})
            if mysql_nodes:
                for node in mysql_nodes:
                    key_name, type_name, qualified_name, extra = parse_mysql_lineage_node_attrs(node)
                    if "extra_retrieve" in params:
                        tmp_node = {"type": type_name, "qualified_name": qualified_name, "extra": extra}
                    else:
                        tmp_node = {"type": type_name, "qualified_name": qualified_name}
                    if type_name == "result_table":
                        tmp_node["result_table_id"] = qualified_name
                    elif type_name == "data_processing":
                        tmp_node["processing_id"] = qualified_name
                    elif type_name == "raw_data":
                        tmp_node["id"] = int(qualified_name)
                    nodes[key_name] = tmp_node
            res_obj["nodes"] = nodes
            res_obj["relations"] = res_relations
    return res_obj

def parse_genealogy_info(rpc_ret):
    res_obj = {}
    genealogy = rpc_ret["genealogy"]
    if genealogy:
        mysql_criteria = genealogy["criteria"]
        mysql_nodes = genealogy["nodes"]
        mysql_relations = genealogy["relations"]
        res_obj["depth"] = mysql_criteria["depth"]
        res_relations = []
        nodes = {}
        if mysql_relations:
            for relation in mysql_relations:
                fm = relation["from"]
                to = relation["to"]
                from_name, _, _ = parse_mysql_lineage_attrs(fm)
                to_name, _, _ = parse_mysql_lineage_attrs(to)
                res_relations.append({"from": from_name, "to": to_name, "status": "ACTIVE"})
        if mysql_nodes:
            for node in mysql_nodes:
                key_name, type_name, qualified_name = parse_mysql_lineage_attrs(node)
                tmp_node = {"type": type_name, "qualified_name": qualified_name}
                if type_name == "result_table":
                    tmp_node["result_table_id"] = qualified_name
                elif type_name == "data_processing":
                    tmp_node["processing_id"] = qualified_name
                elif type_name == "raw_data":
                    tmp_node["id"] = int(qualified_name)
                nodes[key_name] = tmp_node
        res_obj["nodes"] = nodes
        res_obj["relations"] = res_relations
    return res_obj

def get_lineage_node(obj, type_name, qualified_name):
    obj_attrs = get_obj_attrs(obj, add_common=False)
    parse_lineage_nodes_obj(obj_attrs)
    obj_attrs["type"] = type_name
    obj_attrs["qualified_name"] = qualified_name
    return obj_attrs

def parse_lineage_nodes_obj(obj):
    del_redundance_key(obj, "createTime", ATLAS_FIELD_OWNER, ATLAS_FIELD_NAME, ATLAS_FIELD_DESCRIPTION)

def field_index_sort(elem):
    return elem["field_index"]

def get_result_guids(res_dict, field_name):
    result = res_dict["result"]
    ret_attrs = []
    if result:
        res_data = res_dict["data"]
        if res_data:
            if field_name == "entities":
                if "entities" in res_data:
                    entities = res_data[field_name]
                    for entity in entities:
                        status = entity["status"]
                        if status == ATLAS_FIELD_ACTIVE:
                            ret_attrs.append(entity["guid"])
            elif field_name == "entity":
                entity = res_data["entity"]
                status = entity["status"]
                if status == ATLAS_FIELD_ACTIVE:
                    ret_attrs.append(entity["guid"])
    return ret_attrs

def get_result_attrs(res_dict, available_attr=None, complex_attrs=None, need_guid=False):
    result = res_dict["result"]
    ret_attrs = []
    referred_entities = None
    if result:
        res_data = res_dict["data"]
        if res_data:
            if "entities" in res_data:
                entities = res_data["entities"]
                referred_entities = res_data.get("referredEntities")
                for entity in entities:
                    status = entity["status"]
                    if status == ATLAS_FIELD_ACTIVE:
                        attrs = entity["attributes"]
                        handler_common_fields(attrs)
                        ret_entity = {}
                        if need_guid:
                            ret_entity["guid"] = entity["guid"]
                        if complex_attrs:
                            for k, v in list(complex_attrs.items()):
                                field_arr = v.split("|")
                                for def_field_val in field_arr:
                                    show_field, get_field = def_field_val.split("=", 1)
                                    ret_entity[show_field] = None
                                    key_dict = attrs.get(k)
                                    if key_dict:
                                        k_guid = key_dict["guid"]
                                        if referred_entities:
                                            guid_dict = referred_entities.get(k_guid)
                                            if guid_dict:
                                                guid_attrs = guid_dict.get("attributes")
                                                if guid_attrs:
                                                    ret_entity[show_field] = guid_attrs.get(get_field)
                        if available_attr:
                            for field_name in available_attr:
                                if complex_attrs:
                                    if field_name in complex_attrs:
                                        continue
                                ret_entity[field_name] = attrs.get(field_name)
                            ret_attrs.append(ret_entity)
                        else:
                            if need_guid:
                                attrs["guid"] = entity["guid"]
                            ret_attrs.append(attrs)
    return ret_attrs, referred_entities

def get_result_obj_attrs(res_dict, need_guid=False):
    result = res_dict["result"]
    if result:
        res_data = res_dict["data"]
        if res_data:
            referred_entities = res_data["referredEntities"]
            entity = res_data["entity"]
            guid = entity["guid"]
            if guid not in list(referred_entities.keys()):
                # 加入referredEntities中,主要为了处理如:根据Project查询得到了ResultTable,然后在ResultTable中又需要取Project的信息的情况
                referred_entities[guid] = entity
            attrs = get_obj_attrs(entity, need_guid=need_guid)
            return attrs, referred_entities
    return {}, {}

def get_obj_attrs(entity_obj, add_common=True, need_guid=False):
    status = entity_obj["status"]
    if status == ATLAS_FIELD_ACTIVE:
        attrs = entity_obj["attributes"]
        if add_common:
            handler_common_fields(attrs)
        else:
            del_redundance_key(attrs, "created_by", "updated_by", "created_at", "updated_at")
        if need_guid:
            attrs["guid"] = entity_obj["guid"]
        del_common_unuse_fields(attrs)
        return attrs
    return {}

def handler_common_fields(attrs):
    if "deleted_at" in attrs:
        deleted_at = attrs["deleted_at"]
        if deleted_at is not None and deleted_at != 0:
            attrs["deleted_at"] = parse_time_format(deleted_at)
    if "created_by" not in attrs:
        attrs["created_by"] = "admin"
    if "updated_by" not in attrs:
        attrs["updated_by"] = "admin"
    if "created_at" not in attrs:
        attrs["created_at"] = None
    else:
        attrs["created_at"] = parse_time_format(attrs["created_at"])
    if "updated_at" not in attrs:
        attrs["updated_at"] = None
    else:
        attrs["updated_at"] = parse_time_format(attrs["updated_at"])

def del_common_unuse_fields(attrs_dict):
    del_redundance_key(attrs_dict, "qualifiedName", "qualified_name_ref", "name_ref", "qualified_name")
    return attrs_dict

def del_common_unuse_fields2(attrs_dict):
    del_redundance_key(attrs_dict, "created_by", "updated_by", "created_at", "updated_at")
    return attrs_dict

def del_common_unuse_fields3(attrs_dict):
    del_redundance_key(attrs_dict, ATLAS_FIELD_ID, ATLAS_FIELD_NAME, ATLAS_FIELD_OWNER, ATLAS_FIELD_DESCRIPTION)
    return attrs_dict

def del_redundance_key(obj, *keys):
    if isinstance(obj, dict):
        for key in keys:
            if key in list(obj.keys()):
                del obj[key]
    elif isinstance(obj, (list, tuple)):
        for elem in obj:
            if isinstance(elem, dict):
                for key in keys:
                    if key in list(elem.keys()):
                        del elem[key]

def parse_time_format(time_stamp):
    if isinstance(time_stamp, str):
        return time_stamp
    if time_stamp:
        # time_stamp /= 1000.0
        time_arr = time.localtime(time_stamp)
        return time.strftime("%Y-%m-%d %H:%M:%S", time_arr)
    return None


import pandas as pd
import ipywidgets as widgets
import numpy as np
import matplotlib.pyplot as plt
import xlsxwriter as xl
import pygaps as pg
from pygaps.parsing.csv import isotherm_to_csv
from pygaps.graphing.calc_graphs import bet_plot, roq_plot
from pygaps.graphing.isotherm_graphs import plot_iso
from pygaps.characterisation.area_bet import bet_transform, roq_transform, area_BET_raw
from pygaps.core.adsorbate import Adsorbate

import os
from scipy.constants import R

class BET:

    def __init__(self, file, limits=None):
        self.file = file
        self.adsorption_data, self.desorption_data, self.all_data = self.read_DAT()
        self.isotherm = self.create_isotherm()
        self.adsorbate = Adsorbate.find(self.isotherm.adsorbate)
        self.limits = limits
        self.BET_results = self.BET_analysis()

    def read_DAT(self):

        def clean_df(df):

            df.drop(columns=['No.', 'Unnamed: 5'], inplace=True)
            df['Pe/P0'] = df['Pe/kPa']/df['P0/kPa']

            columns = df.columns.to_list()
            new_column_order = [columns[-1]] + columns[:-1]
            df = df[new_column_order].copy()
            return df

        # Here we look for the line no. definint adsorption and desorption data
        with open(self.file, 'r') as f:
            id_ads = []
            id_des = []
            i = 0
            for line in f:
                if line.strip() == 'Adsorption data':
                    id_ads.append(i+2)  # We skip two lines
                elif line.strip() == 'Desorption data':
                    id_ads.append(i-2) # We subtract two lines
                    id_des.append(i+2) # We skip to lines

                i += 1

        # --- loading measurment metadata ---
        instrument_metadata = pd.read_csv(self.file, sep='\t', skiprows=3, nrows=28, names=['instrument', 'value'])
        instrument_metadata = instrument_metadata.loc[
                                instrument_metadata['instrument'].isin([
                                    'Adsorptive:',
                                    'Meas. Temp./K:',
                                    'Vs/ml:',
                                    'Sample weight/g:',
                                    'Date of measurement:',
                                    'Time of measurement:'
                                    ])
                                ]
        meta_keys = instrument_metadata.iloc[:, 0].to_list()
        meta_keys = [key[:-1] for key in meta_keys]
        meta_values = instrument_metadata.iloc[:, 1].to_list()

        self.metadata = {key: val for key, val in zip(meta_keys, meta_values)}  # metadata dictionary
        self.metadata['Meas. Time.'] = self.metadata['Date of measurement'] + ' ' + self.metadata['Time of measurement']
        for key in ['Date of measurement', 'Time of measurement']:
            del self.metadata[key]

        # --- loading measurment data ---
        no_ads_rows = id_ads[1] - id_ads[0] #  Number of lines spannind the adsorption data
        adsorption_data = pd.read_csv(self.file, sep='\t', skiprows=id_ads[0], nrows=no_ads_rows-1, engine='python')
        adsorption_data = clean_df(adsorption_data)

        desorption_data = pd.read_csv(self.file, sep='\t', skiprows=id_des[0], skipfooter=1, engine='python')
        desorption_data = clean_df(desorption_data)

        all_data = adsorption_data.append(desorption_data, ignore_index=True)

        return adsorption_data, desorption_data, all_data

    def create_isotherm(self):

        isotherm = pg.PointIsotherm(
                        pressure = self.all_data['Pe/P0'].to_numpy(),
                        loading = self.all_data['V/ml(STP) g-1'].to_numpy(),

                        pressure_mode = 'relative',
                        pressure_unit = 'kPa',
                        loading_basis = 'molar',
                        loading_unit = 'cm3(STP)',

                        material = 'zeolite',
                        material_basis = 'mass',
                        material_unit = 'g',
                        adsorbate = self.metadata['Adsorptive'],
                        temperature = str(int(float(self.metadata['Meas. Temp./K']))),
                        temperature_unit = 'K'
                    )

        return isotherm

    def plot_isotherm(self):
        file_name, _, _ = self._get_result_path()

        fig_iso, ax_iso = plt.subplots(figsize=[5.5, 4.5])
        ax_iso.set_title('Isotherm: ' + file_name, fontsize=14)

        plot_iso(self.isotherm, ax=ax_iso)
        fig_iso.tight_layout()

        return ax_iso

    def BET_analysis(self):

        bet_area, c_const, n_monolayer, p_monolayer, slope, intercept, minimum, maximum, corr_coef = area_BET_raw(
                            self.isotherm.pressure(branch='ads'),
                            self.isotherm.loading(
                                branch='ads',
                                loading_unit='mol',
                                loading_basis='molar'
                                ),
                            cross_section=self.adsorbate.get_prop("cross_sectional_area"),
                            limits=self.limits)

        BET_results = {
                            'bet_area': np.round(bet_area, 2),
                            'c_const': np.round(c_const, 2),
                            'n_monolayer': n_monolayer,
                            'p_monolayer': np.round(p_monolayer, 4),
                            'bet_slope': np.round(slope, 2),
                            'bet_intercept': np.round(intercept, 2),
                            'corr_coef': np.round(corr_coef, 5),
                            'p_select_min' : np.round(self.isotherm.pressure(branch='ads')[minimum], 4),
                            'p_select_max' : np.round(self.isotherm.pressure(branch='ads')[maximum], 4) ,
                            'minimum' : minimum,
                            'maximum' : maximum,
                            'limits': [minimum, maximum]
                        }
        self.BET_results = BET_results
        return BET_results

    def plot_bet(self):
        file_name, _, _ = self._get_result_path()

        fig_bet, ax_bet = plt.subplots(figsize=[5.5, 4.5])
        _, _, n_monolayer, p_monolayer, slope, intercept, _, _, _, minimum, maximum, _= self.BET_results.values()

        ax_bet = bet_plot(
            self.isotherm.pressure(branch='ads'),
            bet_transform(
                self.isotherm.pressure(branch='ads'),
                self.isotherm.loading(
                         branch='ads',
                         loading_unit='mol',
                         loading_basis='molar')
                         ),
            minimum, maximum, slope, intercept, p_monolayer,
            bet_transform(p_monolayer, n_monolayer),
            ax=ax_bet
        )

        ax_bet.set_title('BET plot: ' + file_name, fontsize=14)
        fig_bet.tight_layout()

        return ax_bet

    def plot_roq(self):

        file_name, _, _ = self._get_result_path()

        fig_roq, ax_roq = plt.subplots(figsize=[5.5, 4.5])

        _, _, n_monolayer, p_monolayer, slope, intercept, _, _, _, minimum, maximum, _ = self.BET_results.values()

        ax_roq = roq_plot(
            self.isotherm.pressure(branch='ads'),
            roq_transform(
                self.isotherm.pressure(branch='ads'),
                self.isotherm.loading(
                         branch='ads',
                         loading_unit='mol',
                         loading_basis='molar')
                         ),
            minimum, maximum, p_monolayer,
            roq_transform(p_monolayer, n_monolayer),
            ax=ax_roq
            )

        ax_roq.set_title('Rouquerol plot: ' + file_name, fontsize=14)
        fig_roq.tight_layout()

        return ax_roq

    ##### EXPORT METHODS

    def to_json(self, filepath=None):

        _, result_file_name, default_path = self._get_result_path()

        if not filepath:
            res_path = default_path + '.xlsx'
        else:
            res_path = os.path.join(filepath, result_file_name + '.xlsx')

        self.isotherm.to_json(res_path)

    def to_csv(self, filepath='default'):

        _, result_file_name, default_path = self._get_result_path()

        if not filepath:
            res_path = default_path + '.csv'
        else:
            res_path = os.path.join(filepath, result_file_name + '.csv')

        isotherm_to_csv(self.isotherm, res_path)

        return

    def to_excel(self, filepath=None):

        ##Getting file path
        _, result_file_name, default_path = self._get_result_path()

        if not filepath:
            res_path = default_path + '.xlsx'
        else:
            res_path = os.path.join(filepath, result_file_name + '.xlsx')

        # -- Instanciating the xlsx writer
        writer = pd.ExcelWriter(res_path, engine='xlsxwriter')  # Defining the xlsx writer
        workbook = writer.book  # Definint the workbook
        cell_format = workbook.add_format({'bold': True, 'italic': True}) # cell format "bold"
        cell_format_it = workbook.add_format({'italic': True}) # cell format "italic"

        # -- Creating isotherm sheet
        isotherm_sheet = 'Isotherm'
        df_isotherm = []
        for branch in ['ads', 'des']:
            isotherm_points = {
                'pressure_{}'.format(branch) : self.isotherm.pressure(branch=branch),
                'loading_{}'.format(branch) : self.isotherm.loading(branch=branch),
            }
            df = pd.DataFrame(isotherm_points)
            df_isotherm.append(df)

        isotherm_start_row = 11
        start_cols = [0, 2]
        for i, df in enumerate(df_isotherm):
            df.to_excel(writer, sheet_name=isotherm_sheet,
             startcol=start_cols[i], startrow=isotherm_start_row, index=False)

        # -- writing isotherm dictonary to excel sheet
        iso_dict = {}
        for r in self.isotherm.to_csv().split('\n')[:10]:
            key, val = r.split(',')
            iso_dict[key] = val
        worksheet_iso = writer.sheets[isotherm_sheet]

        row_num = 0  # The start column number
        for key, value in iso_dict.items(): # Convert non-list value to list
            worksheet_iso.write(row_num, 0, key, cell_format)
            worksheet_iso.write_string(row_num, 1, value)
            row_num += 1

        # -- plotting isotherm

        chart_isotherm = workbook.add_chart({'type' : 'scatter', 'subtype': 'smooth_with_markers'})

        rows_ads = self.isotherm.pressure(branch='ads').shape[0]
        rows_des = self.isotherm.pressure(branch='des').shape[0]

        chart_isotherm.add_series({
                                    'name': '{}-ads.'.format(str(self.isotherm.adsorbate)),
                                    'categories': [isotherm_sheet, isotherm_start_row+1, 0, isotherm_start_row+rows_ads+1, 0],
                                    'values':  [isotherm_sheet, isotherm_start_row+1, 1, isotherm_start_row+rows_ads+1, 1],
                                    'marker': {
                                                'type': 'circle',
                                                'size': 8,
                                                'fill' : {'color': '#144F82'},
                                                'border': {
                                                    'color': '#144F82',
                                                    'width' : 1.5},
                                                    },
                                                'line' : {'color' : '#144F82'}
                                })

        chart_isotherm.add_series({
                                    'name': '{}-des.'.format(str(self.isotherm.adsorbate)),
                                    'categories': [isotherm_sheet, 13, 2, isotherm_start_row+rows_des+1, 2],
                                    'values':  [isotherm_sheet, 13, 3, isotherm_start_row+rows_des+1, 3],
                                    'marker': {
                                                'type': 'circle',
                                                'size': 8,
                                                'fill' : {'color': 'white'},
                                                'border': {
                                                    'color': '#144F82',
                                                    'width' : 1.5},
                                                    },
                                                'line' : {
                                                    'color' : '#144F82',
                                                    'dash_type': 'dash'}
                                })

        chart_isotherm.set_x_axis({
                    'name': 'Pressure [p/p0]',
                    'name_font' : {'size' : 18},
                    'num_format' : '0.00',
                    'num_font':  {'size': 13},
                    'max' : 1,
                    'min' : 0,
            })

        chart_isotherm.set_y_axis({
                'name': 'Loading (ml/g [STP])',
                'name_font' : {'size' : 18},
                'num_font':  {'size': 14 },
                'major_gridlines': {'visible': False},
        })

        chart_isotherm.set_plotarea({
                'border': {'color': 'black', 'width': 1.25}
            })

        chart_isotherm.set_size({'x_scale': 1.45, 'y_scale': 1.6})

        worksheet_iso.insert_chart('H5', chart_isotherm)

        # -- Creating the BET sheet
        sheetname = 'BET'

        # -- full bet plot
        bet_start_row = 13

        main_bet_points = {
            'pressure' : self.isotherm.pressure(branch='ads'),
            'bet_points' : bet_transform(
                self.isotherm.pressure(branch='ads'),
                self.isotherm.loading(
                         branch='ads',
                         loading_unit='mol',
                         loading_basis='molar')
                         )
        }

        df_main_bet_points = pd.DataFrame(main_bet_points)
        df_main_bet_points.to_excel(writer, sheet_name=sheetname,
             startcol=0, startrow=bet_start_row, index=False)

        # -- selected bet plot from roq criteria
        bet_minimum = self.BET_results['minimum']
        bet_maximum = self.BET_results['maximum']

        selected_bet_point = {
            'pressure_selected' : self.isotherm.pressure(branch='ads')[bet_minimum:bet_maximum],
            'bet_points_selected' : bet_transform(
                self.isotherm.pressure(branch='ads'),
                self.isotherm.loading(
                         branch='ads',
                         loading_unit='mol',
                         loading_basis='molar')
                         )[bet_minimum:bet_maximum]
        }

        df_selected_bet_point = pd.DataFrame(selected_bet_point)
        df_selected_bet_point.to_excel(writer, sheet_name=sheetname,
             startcol=2, startrow=bet_start_row, index=False)

        # -- selected monolayer point
        bet_p_monolayer = self.BET_results['p_monolayer']
        bet_n_monolayer = self.BET_results['n_monolayer']

        bet_monolayer_point = {
            'pressure_monolayer' : [bet_p_monolayer],
            'bet_point_monolayer' : [bet_transform(bet_p_monolayer, bet_n_monolayer)]
        }

        df_monolayer_point = pd.DataFrame(bet_monolayer_point)
        df_monolayer_point.to_excel(writer, sheet_name=sheetname,
             startcol=4, startrow=bet_start_row, index=False)

        # -- instanciating the bet worksheet
        worksheet = writer.sheets[sheetname]

        # -- writing BET_results dictonary to excel sheet
        row_num = 0  # The start column number
        for key, value in self.BET_results.items(): # Convert non-list value to list
            worksheet.write(row_num, 0, key, cell_format)
            if not type(value) == list:
                value = [value]
            worksheet.write_row(row_num, 1, value)
            if key == 'bet_area':
                worksheet.write(row_num, 2, 'm^2/g', cell_format_it)
            if key == 'n_monolayer':
                worksheet.write_string(row_num, 2, 'mol/g', cell_format_it)
            row_num += 1

        # -- plotting BET
        chart = workbook.add_chart({'type' : 'scatter'})

        rows_bet_mp, _ = df_main_bet_points.shape
        chart.add_series({
                                    'name': 'All Points',
                                    'categories': [sheetname, bet_start_row+1, 0, bet_start_row+rows_bet_mp+1, 0],
                                    'values':  [sheetname, bet_start_row+1, 1, bet_start_row+rows_bet_mp+1, 1],
                                    'marker': {
                                                'type': 'circle',
                                                'size': 8,
                                                'fill' : {'color': 'white'},
                                                'border': {
                                                    'color': 'gray',
                                                    'width' : 1.5},
                                                }
                                })
        row_bet_sel, _ = df_selected_bet_point.shape
        chart.add_series({
                                    'name': 'Selected Points',
                                    'categories': [sheetname, bet_start_row+1, 2, bet_start_row+row_bet_sel+1, 2],
                                    'values':  [sheetname, bet_start_row+1, 3, bet_start_row+row_bet_sel+1, 3],
                                    'marker': {
                                                'type': 'circle',
                                                'size': 8,
                                                'fill': {'color': 'red'},
                                                'border' : {'color' : 'red'}
                                                },
                                    'trendline': {
                                        'type': 'linear',
                                        'name' : 'fit',
                                        'line': {
                                        'width': 1.2,
                                        'dash_type': 'long_dash'},
                                    },
                            })

        row_bet_mono, _ = df_monolayer_point.shape
        chart.add_series({
                                    'name': 'Monolayer Point',
                                    'categories' : [sheetname, bet_start_row+1, 4, bet_start_row+row_bet_mono+1, 4],
                                    'values':  [sheetname, bet_start_row+1, 5, bet_start_row+row_bet_mono+1, 5],
                                    'marker': {
                                                'type': 'x',
                                                'size': 8,
                                                'fill' : {'color' : 'black'},
                                                'border' : {'color' : 'black'}
                                                }
                                })

        chart.set_x_axis({
                    'name': 'Pressure (p/p0)',
                    'name_font' : {'size' : 18},
                    'num_format' : '0.00',
                    'num_font':  {'size': 13 },
                    'min' : 0,
                    'max' : self.BET_results['p_select_max']
            })

        chart.set_y_axis({
                'name': '1/[n(p0/p - 1)]',
                'name_font' : {'size' : 18},
                'num_font':  {'size': 13 },
                'min' : 0,
                'max' : 1.3*df_selected_bet_point.iloc[:, 1].max(),
                'major_gridlines': {'visible': False},
        })

        chart.set_plotarea({
                'border': {'color': 'black', 'width': 1.25}
            })

        chart.set_size({'x_scale': 1.45, 'y_scale': 1.6})
        worksheet.insert_chart('H5', chart)

        # -- Rouquerol

        roq_sheet = 'Rouquerol'

        roq_start_row = 0

        main_roq_points = {
            'pressure' : self.isotherm.pressure(branch='ads'),
            'roq_points' : roq_transform(
                self.isotherm.pressure(branch='ads'),
                self.isotherm.loading(
                         branch='ads',
                         loading_unit='mol',
                         loading_basis='molar')
                         )
        }

        df_main_roq_points = pd.DataFrame(main_roq_points)
        df_main_roq_points.to_excel(writer, sheet_name=roq_sheet,
             startcol=0, startrow=roq_start_row, index=False)

        # -- selected bet plot from roq criteria
        bet_minimum = self.BET_results['minimum']
        bet_maximum = self.BET_results['maximum']

        selected_roq_point = {
            'pressure_selected' : self.isotherm.pressure(branch='ads')[bet_minimum:bet_maximum],
            'roq_points_selected' : roq_transform(
                self.isotherm.pressure(branch='ads'),
                self.isotherm.loading(
                         branch='ads',
                         loading_unit='mol',
                         loading_basis='molar')
                         )[bet_minimum:bet_maximum]
        }

        df_selected_roq_point = pd.DataFrame(selected_roq_point)
        df_selected_roq_point.to_excel(writer, sheet_name=roq_sheet,
             startcol=2, startrow=roq_start_row, index=False)

        # -- selected monolayer point
        bet_p_monolayer = self.BET_results['p_monolayer']
        bet_n_monolayer = self.BET_results['n_monolayer']

        roq_monolayer_point = {
            'pressure_monolayer' : [bet_p_monolayer],
            'bet_point_monolayer' : [roq_transform(bet_p_monolayer, bet_n_monolayer)]
        }

        df_roq_monolayer_point = pd.DataFrame(roq_monolayer_point)
        df_roq_monolayer_point.to_excel(writer, sheet_name=roq_sheet,
             startcol=4, startrow=roq_start_row, index=False)

        # -- plotting roq plot
        chart_roq = workbook.add_chart({'type' : 'scatter'})

        rows_roq_mp, _ = df_main_roq_points.shape
        chart_roq.add_series({
                                    'name': 'All Points',
                                    'categories': [roq_sheet, roq_start_row+1, 0, roq_start_row+rows_roq_mp+1, 0],
                                    'values':  [roq_sheet, roq_start_row+1, 1, roq_start_row+rows_roq_mp+1, 1],
                                    'marker': {
                                                'type': 'circle',
                                                'size': 8,
                                                'fill' : {'color': 'white'},
                                                'border': {
                                                    'color': 'gray',
                                                    'width' : 1.5},
                                                }
                                })
        row_roq_sel, _ = df_selected_roq_point.shape
        chart_roq.add_series({
                                    'name': 'Selected Points',
                                    'categories': [roq_sheet, roq_start_row+1, 2, roq_start_row+row_roq_sel+1, 2],
                                    'values':  [roq_sheet, roq_start_row+1, 3, roq_start_row+row_roq_sel+1, 3],
                                    'marker': {
                                                'type': 'circle',
                                                'size': 8,
                                                'fill': {'color': 'red'},
                                                'border' : {'color' : 'red'}
                                                },
                            })

        row_roq_mono, _ = df_roq_monolayer_point.shape
        chart_roq.add_series({
                                    'name': 'Monolayer Point',
                                    'categories' : [roq_sheet, roq_start_row+1, 4, roq_start_row+row_roq_mono+1, 4],
                                    'values':  [roq_sheet, roq_start_row+1, 5, roq_start_row+row_roq_mono+1, 5],
                                    'marker': {
                                                'type': 'x',
                                                'size': 8,
                                                'fill' : {'color' : 'black'},
                                                'border' : {'color' : 'black'}
                                                }
                                })

        chart_roq.set_x_axis({
                    'name': 'Pressure (p/p0)',
                    'name_font' : {'size' : 18},
                    'num_format' : '0.00',
                    'num_font':  {'size': 13 },
                    'min' : 0,
                    'max' : 1.0
            })

        chart_roq.set_y_axis({
                'name': 'n(1 - p/p0)',
                'name_font' : {'size' : 18},
                'num_font':  {'size': 13 },
                'num_format' : '0.0E+00',
                'min' : 0,
                'major_gridlines': {'visible': False},
        })

        chart_roq.set_plotarea({
                'border': {'color': 'black', 'width': 1.25}
            })

        chart_roq.set_size({'x_scale': 1.45, 'y_scale': 1.6})

        roq_worksheet = writer.sheets[roq_sheet]
        roq_worksheet.insert_chart('H5', chart_roq)

        # -- Writing raw data

        raw_data_sheet = 'Raw Data'

        self.adsorption_data.to_excel(writer, sheet_name=raw_data_sheet,
             startcol=0, startrow=1, index=False)

        self.desorption_data.to_excel(writer, sheet_name=raw_data_sheet,
             startcol=5, startrow=1, index=False)

        raw_worksheet = writer.sheets[raw_data_sheet]
        raw_worksheet.write_string(0, 0, 'Adsorption', cell_format)
        raw_worksheet.write_string(0, 5, 'Desorption', cell_format)

        # -- writing BET_results dictonary to excel sheet
        row_num = 1  # The start column number
        raw_worksheet.write_string(0, 11, 'Measurment metadata', cell_format)
        for key, value in self.metadata.items(): # Convert non-list value to list
            raw_worksheet.write_string(row_num, 11, key, cell_format)
            raw_worksheet.write_string(row_num, 12, value)
            row_num += 1

        writer.save()
        return

    ### METHOD WHICH GETS THE RESULT PATH
    def _get_result_path(self):

        list_path = self.file.split('\\')
        default_result_folder = '\\'.join(list_path[:-1] + ['RESULTS'])
        file_name = list_path[-1].split('.')[0]
        result_file_name = file_name + '_result'

        if os.path.isdir(default_result_folder):
            result_path = os.path.join(default_result_folder, result_file_name)
        else:
            os.mkdir(default_result_folder)
            result_path = os.path.join(default_result_folder, result_file_name)

        return file_name, result_file_name, result_path


# coding=utf-8
import tensorflow as tf
import numpy as np
import sklearn as sk
import time
import matplotlib
import matplotlib.pyplot as plt
from sklearn.datasets.samples_generator import make_circles

import os
os.environ['TF_CPP_MIN_LOG_LEVEL']='2'

N = 210
K = 2
MAX_ITERS = 1000
cut = int(N*0.7)

start = time.time()
data, features = make_circles(n_samples=N,shuffle=True,noise=0.12,factor=0.4)
tr_data, tr_features = data[:cut], features[:cut]
te_data, te_feature = data[cut:], features[cut:]

fig, ax = plt.subplots()
ax.scatter(tr_data.transpose()[0], tr_data.transpose()[1], marker = 'o', s = 100, c = tr_features, cmap=plt.cm.coolwarm )
plt.show()

points = tf.Variable(data)
cluster_assignment = tf.Variable(tf.zeros([N], dtype=tf.int64))

sess = tf.Session()
sess.run(tf.global_variables_initializer())

test = []
for i, j in zip(te_data, te_feature): #元素个数与最短的列表一致
    distances = tf.reduce_sum(tf.square(i- tr_data), reduction_indices=1) #计算te_data中某个样本与tr_data中所有样本的距离值
    neighor = tf.argmin(distances,axis=0) # te_data中该样本与tr_data中距离最小的样本所在的索引
    # print("the shape of distance = ", sess.run(tf.shape(distances)))
    test.append(tr_features[sess.run(neighor)])

print(test)
fig, ax = plt.subplots()
ax.scatter(te_data.transpose()[0], te_data.transpose()[1], marker = 'o', s = 100, c = test, cmap=plt.cm.coolwarm )
plt.show()


def matchingStrings(strings, queries):
    count_sim = []
    count = 0
    for que in queries:
        for strg in strings:
            if que == strg:
                count += 1
        count_sim.append(count)
        count = 0
    return (count_sim)

strings_count = int(input().strip())

strings = []

for _ in range(strings_count):
    strings_item = input()
    strings.append(strings_item)

queries_count = int(input().strip())

queries = []

for _ in range(queries_count):
    queries_item = input()
    queries.append(queries_item)

# strings = ['def','de','fgh']
# queries = ['de','lmn','fgh']

res = matchingStrings(strings, queries)

print(res)


import pathlib
import time
from collections import defaultdict
from dataclasses import dataclass
from typing import Generator

from smb3parse.constants import OFFSET_SIZE
from smb3parse.data_points import LevelPointerData
from smb3parse.levels import HEADER_LENGTH, WORLD_COUNT
from smb3parse.levels.level_header import LevelHeader
from smb3parse.levels.world_map import WorldMap
from smb3parse.objects.object_set import MUSHROOM_OBJECT_SET, SPADE_BONUS_OBJECT_SET
from smb3parse.util import hex_int
from smb3parse.util.parser.cpu import NesCPU
from smb3parse.util.rom import Rom

@dataclass
class FoundLevel:
    level_offset_positions: list[int]
    enemy_offset_positions: list[int]

    world_number: int

    level_offset: int
    enemy_offset: int

    object_set_number: int
    object_data_length: int
    """Length of all Level Objects, including the Level Header, but without the delimiter at the end."""

    enemy_data_length: int
    """Length of all Enemy Objects, without the delimiter at the beginning and end."""

    found_in_world: bool
    found_as_jump: bool
    is_world_specific: bool

    def to_dict(self) -> dict[str, list[int] | int | bool]:
        ret_dict = vars(self)

        return ret_dict

    @staticmethod
    def from_dict(data: dict) -> "FoundLevel":
        return FoundLevel(
            data["level_offset_positions"],
            data["enemy_offset_positions"],
            data["world_number"],
            data["level_offset"],
            data["enemy_offset"],
            data["object_set_number"],
            data["object_data_length"],
            data["enemy_data_length"],
            data["found_in_world"],
            data["found_as_jump"],
            data.get("is_world_specific", data.get("is_generic", False)),  # backwards compatible
        )

@dataclass
class FoundLevelRecord:
    level_address: int
    level_address_offset: int

    enemy_address: int
    enemy_address_offset: int

    object_set: int

    found_in_world: bool = False
    found_as_jump: bool = False
    is_world_specific: bool = False

    @staticmethod
    def from_level_pointer(
        level_pointer: LevelPointerData,
        from_world: bool,
        from_jump: bool,
        world_specific: bool,
    ) -> "FoundLevelRecord":
        return FoundLevelRecord(
            level_pointer.level_address,
            level_pointer.level_offset_address,
            level_pointer.enemy_address,
            level_pointer.enemy_offset_address,
            level_pointer.object_set,
            from_world,
            from_jump,
            world_specific,
        )

def gen_levels_in_rom(
    rom: Rom,
) -> Generator[tuple[int, int], bool, tuple[defaultdict, dict[int, FoundLevel]]]:
    levels_by_address: dict[int, FoundLevel] = {}

    start = time.time()

    for world_num in range(WORLD_COUNT - 1):
        levels_in_world = 0

        world = WorldMap.from_world_number(rom, world_num + 1)

        found_level_records: list[FoundLevelRecord] = [
            (FoundLevelRecord.from_level_pointer(lp, True, False, False)) for lp in world.level_pointers
        ]

        # add airship
        found_level_records.append(
            FoundLevelRecord(
                world.data.airship_level_address,
                world.data.airship_level_offset_address,
                world.data.airship_enemy_address,
                world.data.airship_enemy_offset_address,
                world.data.airship_level_object_set,
                False,
                False,
                True,
            )
        )

        # add generic exit
        found_level_records.append(
            FoundLevelRecord(
                world.data.generic_exit_level_address,
                world.data.generic_exit_level_offset_address,
                world.data.generic_exit_enemy_address,
                world.data.generic_exit_enemy_offset_address,
                world.data.generic_exit_object_set,
                False,
                False,
                True,
            )
        )

        # add big ? level
        found_level_records.append(
            FoundLevelRecord(
                world.data.big_q_block_level_address,
                world.data.big_q_block_level_offset_address,
                world.data.big_q_block_enemy_address,
                world.data.big_q_block_enemy_offset_address,
                world.data.big_q_block_object_set,
                False,
                False,
                True,
            )
        )

        # add coin ship level
        found_level_records.append(
            FoundLevelRecord(
                world.data.coin_ship_level_address,
                world.data.coin_ship_level_offset_address,
                world.data.coin_ship_enemy_address,
                world.data.coin_ship_enemy_offset_address,
                world.data.coin_ship_level_object_set,
                False,
                False,
                True,
            )
        )

        # add special/white toad house level
        found_level_records.append(
            FoundLevelRecord(
                world.data.toad_warp_level_address,
                world.data.toad_warp_level_offset_address,
                0x0,  # enemy item data is used directly, not as an offset
                world.data.toad_warp_item_address,
                MUSHROOM_OBJECT_SET,
                False,
                False,
                True,
            ),
        )

        should_stop = False
        for record in found_level_records:
            if should_stop:
                return defaultdict(list), levels_by_address

            if record.level_address in levels_by_address:
                found_level = levels_by_address[record.level_address]

                assert record.level_address_offset not in found_level.level_offset_positions
                found_level.level_offset_positions.append(record.level_address_offset)

                assert record.enemy_address_offset not in found_level.enemy_offset_positions
                found_level.enemy_offset_positions.append(record.enemy_address_offset)

                found_level.found_in_world |= record.found_in_world
                found_level.found_as_jump |= record.found_as_jump
                found_level.is_world_specific |= record.is_world_specific

                continue

            if record.object_set == SPADE_BONUS_OBJECT_SET:
                continue

            print(
                f"W{world_num + 1}",
                hex(record.level_address),
                hex(record.enemy_address),
                record.object_set,
            )
            # traverse Jump Destinations by following the offsets in the header
            while True:
                levels_in_world += 1

                should_stop = yield world.number, levels_in_world

                if should_stop:
                    break

                parsed_level = NesCPU(rom).load_from_address(
                    record.object_set, record.level_address, record.enemy_address
                )

                found_level = FoundLevel(
                    [record.level_address_offset],
                    [record.enemy_address_offset],
                    world_num + 1,
                    record.level_address,
                    record.enemy_address,
                    record.object_set,
                    parsed_level.object_data_length,
                    parsed_level.enemy_data_length,
                    record.found_in_world,
                    record.found_as_jump,
                    record.is_world_specific,
                )

                levels_by_address[record.level_address] = found_level

                if not parsed_level.has_jump():
                    break

                header_of_old_level = LevelHeader(
                    rom,
                    rom.read(record.level_address, HEADER_LENGTH),
                    record.object_set,
                )

                level_address_position = record.level_address
                level_address = header_of_old_level.jump_level_address

                enemy_address_position = record.level_address + OFFSET_SIZE
                enemy_address = header_of_old_level.jump_enemy_address

                object_set_number = header_of_old_level.jump_object_set_number

                new_record = FoundLevelRecord(
                    level_address,
                    level_address_position,
                    enemy_address,
                    enemy_address_position,
                    object_set_number,
                )

                if 0 in [header_of_old_level.jump_level_offset, object_set_number]:
                    break

                if level_address in levels_by_address:
                    found_level = levels_by_address[level_address]
                    assert level_address_position not in found_level.level_offset_positions
                    found_level.level_offset_positions.append(level_address_position)
                    found_level.enemy_offset_positions.append(enemy_address_position)
                    found_level.found_as_jump = True
                    break

                new_record.found_in_world = False
                new_record.found_as_jump = True

                record = new_record

                print("    ", hex(level_address), object_set_number)

    print(time.time() - start)

    level_count = 0

    levels_per_object_set = defaultdict(list)

    for level_address in sorted(levels_by_address.keys()):
        found_level = levels_by_address[level_address]
        levels_per_object_set[found_level.object_set_number].append(level_address)

    for object_set, levels_addresses in sorted(levels_per_object_set.items()):
        level_count += len(levels_addresses)
        print(
            object_set,
            ": ",
            len(levels_addresses),
            ", ".join(
                f"{hex(level_address).upper().replace('X', 'x')}/"
                f"{len(levels_by_address[level_address].level_offset_positions)}"
                for level_address in sorted(levels_addresses)
            ),
        )

    print("---------------------", level_count, "------------------------")

    root_dir = pathlib.Path(__file__).parent.parent.parent.parent

    stock_level_file = root_dir / "data" / "levels.dat"

    missing = 0
    levels: dict[int, set[int]] = defaultdict(set)

    for line in stock_level_file.open("r").readlines():
        if not line:
            continue

        world_no, *_, level_address, _, object_set_no, _ = line.split(",")

        level_address = hex_int(level_address) - 9
        object_set_num = hex_int(object_set_no)

        if int(world_no) in [0, 9]:
            continue

        levels[object_set_num].add(level_address)

        if level_address not in levels_per_object_set[object_set_num]:
            missing += 1
            print(world_no, object_set_num, hex(level_address))

    print(missing, "levels")

    for object_set_num in range(1, 15):
        print(
            object_set_num,
            list(
                map(
                    hex,
                    set(levels_per_object_set[object_set_num]).difference(levels[object_set_num]),
                )
            ),
        )

    return levels_per_object_set, levels_by_address


# Definition for singly-linked list.
# class ListNode(object):
#     def __init__(self, x):
#         self.val = x
#         self.next = None

class Solution(object):
    def isPalindrome(self, head):
        """
        :type head: ListNode
        :rtype: bool
        """
        if not (head and head.next):
            return True
        slow = fast = head
        slow_pos = head.next
        slow_pre = None
        while fast and fast.next:
            fast = fast.next.next
            # reverse the first half
            slow.next = slow_pre
            slow_pre, slow, slow_pos = slow, slow_pos, slow_pos.next
        if fast:
            slow = slow.next
        # compare the second half to the reversed first half
        while slow and slow.val == slow_pre.val:
            slow = slow.next
            slow_pre = slow_pre.next
        return not slow


# -*- coding: utf-8 -*-
"""
Éditeur de Spyder

Renaud.
"""

import numpy as np
import matplotlib.pyplot as plt

l1=0.17
m1=2.8
l2=1.
m2=1.
g=9.81
Jeq=m1*l1**2+m2*l2**2
K=0.45
theta0=np.pi/2.
Cf=4

def F(y,t):
    mP=(l1*m1+(l1+l2)*m2)*g*np.cos(y[1])
    yl=(K*(180/np.pi)*(-y[1]+theta0)-mP-Cf*y[0])/Jeq,y[0]
    return yl

def euler(F,y0,t):
    N = len(t)
    y=np.zeros((len(t),2))
    y[:0] = y0
    for i in range(N-1):
        dt=(t[i+1]-t[i])
        s = [j*dt for j in F(y[i],t[i])]
        y[i+1] = y[i]+s
    return y

t = np.linspace(0,5,100)
y = euler(F,[0,0],t)
y1 = (180/np.pi)*y[:,1]
plt.plot(t,y1,label='Euler')
plt.legend(loc='upper left')
plt.show()


from sqlalchemy import Table

from flaskr.db import get_db, metadata, Base

class Favour(Base):
    __table__ = Table('favours', metadata, autoload=True)

    def __init__(self, author_id, post_id):
        self.author_id = author_id
        self.post_id = post_id

    def get(self):
        """
        Get the favour record with author_id and post_id in Favour instance.

        :return: one favour or None
        """
        favour = get_db().execute(
            'SELECT * FROM favours WHERE post_id = ? AND author_id = ?', (self.post_id, self.author_id)
        ).fetchone()
        return favour

    def insert(self):
        con = get_db()
        try:
            con.execute(
                'INSERT OR IGNORE INTO favours (post_id, author_id)'
                ' VALUES (?, ?)', (self.post_id, self.author_id)
            )
        except BaseException:
            con.rollback()
            raise
        else:
            con.commit()
        return

    def delete(self):
        con = get_db()
        try:
            con.execute(
                'DELETE FROM favours WHERE post_id = ? AND author_id = ?', (self.post_id, self.author_id)
            )
        except BaseException:
            con.rollback()
            raise
        else:
            con.commit()
        return


from django.contrib.gis.geos import GEOSGeometry

from rest_framework import response, status, views

from oppurtunity.models import GeoOpportunityZone

class OpportunityZoneView(views.APIView):
    """
    API for opportunity zone.
    """

    def post(self, request):
        data = request.data
        response_list = []
        geo_oppurtunity_objs_all = GeoOpportunityZone.objects.filter(state='New Jersey')
        for each in data:
            longitude = each.get('longitude', None)
            latitude = each.get('latitude', None)
            lookup_type = each.get('lookup_type')
            geom = None
            zones_list = []
            try:
                geom = GEOSGeometry("Point({} {})".format(longitude, latitude), srid=4269)
            except:
                pass
            if geom:
                if lookup_type == "contains":
                    geo_oppurtunity_objs = geo_oppurtunity_objs_all.filter(poly__contains=geom)
                else:
                    geo_oppurtunity_objs = geo_oppurtunity_objs_all.filter(poly__intersects=geom)
                if geo_oppurtunity_objs:
                    each.update({"is_opportunity_zone": True})
                else:
                    each.update({"is_opportunity_zone": False})
            response_list.append(each)
        return response.Response(response_list)


from sklearn.cluster import FeatureAgglomeration
from fastsklearnfeature.declarative_automl.optuna_package.optuna_utils import categorical
import numpy as np
from fastsklearnfeature.declarative_automl.optuna_package.optuna_utils import id_name

class FeatureAgglomerationOptuna(FeatureAgglomeration):
    def init_hyperparameters(self, trial, X, y):
        self.name = id_name('FeatureAgglomeration_')

        self.n_components_fraction = trial.suggest_uniform(self.name + 'n_components_fraction', 0.0, 1.0)

        self.linkage = trial.suggest_categorical(self.name + "linkage", ["ward", "complete", "average"])

        if self.linkage == 'ward':
            self.affinity = "euclidean"
        else:
            self.affinity = trial.suggest_categorical(self.name + "affinity", ["euclidean", "manhattan", "cosine"])

        self.pooling_func = trial.suggest_categorical(self.name + "pooling_func", [np.mean, np.median, np.max])

        self.sparse = False

    def fit(self, X, y=None, **params):
        self.n_clusters = max(1, int(self.n_components_fraction * X.shape[1]))
        #print('ncom: ' + str(self.n_clusters))
        return super().fit(X=X, y=y, **params)

    def generate_hyperparameters(self, space_gen, depending_node=None):
        self.name = id_name('FeatureAgglomeration_')

        space_gen.generate_number(self.name + 'n_components_fraction', 0.5, depending_node=depending_node)
        space_gen.generate_cat(self.name + "linkage", ["ward", "complete", "average"], "ward", depending_node=depending_node)
        space_gen.generate_cat(self.name + "affinity", ["euclidean", "manhattan", "cosine"], "euclidean", depending_node=depending_node)
        space_gen.generate_cat(self.name + "pooling_func", [np.mean, np.median, np.max], np.mean, depending_node=depending_node)


"""empty message

Revision ID: 06b06a865055
Revises:
Create Date: 2018-02-18 23:17:06.285139

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '06b06a865055'
down_revision = None
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('addresses',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('address', sa.String(length=60), nullable=True),
    sa.Column('state', sa.String(length=30), nullable=True),
    sa.Column('city', sa.String(length=30), nullable=True),
    sa.Column('zipcode', sa.String(length=20), nullable=True),
    sa.PrimaryKeyConstraint('id'),
    sa.UniqueConstraint('address', 'state', 'city', 'zipcode', name='addr_state_city_zip')
    )
    op.create_table('users',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('first_name', sa.String(length=60), nullable=True),
    sa.Column('last_name', sa.String(length=60), nullable=True),
    sa.Column('phone_number', sa.String(length=60), nullable=True),
    sa.PrimaryKeyConstraint('id'),
    sa.UniqueConstraint('first_name', 'last_name', 'phone_number', name='first_last_phone_uc')
    )
    op.create_index(op.f('ix_users_first_name'), 'users', ['first_name'], unique=False)
    op.create_index(op.f('ix_users_last_name'), 'users', ['last_name'], unique=False)
    op.create_index(op.f('ix_users_phone_number'), 'users', ['phone_number'], unique=False)
    op.create_table('restaurants',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('name', sa.String(length=60), nullable=True),
    sa.Column('category', sa.String(length=60), nullable=True),
    sa.Column('address_id', sa.Integer(), nullable=True),
    sa.ForeignKeyConstraint(['address_id'], ['addresses.id'], ),
    sa.PrimaryKeyConstraint('id'),
    sa.UniqueConstraint('name'),
    sa.UniqueConstraint('name', 'category', 'address_id', name='name_category_address')
    )
    op.create_table('ratings',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('cost', sa.Integer(), nullable=True),
    sa.Column('food', sa.Integer(), nullable=True),
    sa.Column('cleanliness', sa.Integer(), nullable=True),
    sa.Column('service', sa.Integer(), nullable=True),
    sa.Column('totalscore', sa.Float(), nullable=True),
    sa.Column('comment', sa.String(length=120), nullable=True),
    sa.Column('restaurant_id', sa.Integer(), nullable=True),
    sa.Column('user_id', sa.Integer(), nullable=True),
    sa.Column('date', sa.DateTime(), nullable=True),
    sa.ForeignKeyConstraint(['restaurant_id'], ['restaurants.id'], ),
    sa.ForeignKeyConstraint(['user_id'], ['users.id'], ),
    sa.PrimaryKeyConstraint('id'),
    sa.UniqueConstraint('restaurant_id', 'user_id', 'date', name='restaurant_user_date')
    )
    op.create_index(op.f('ix_ratings_date'), 'ratings', ['date'], unique=False)
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_index(op.f('ix_ratings_date'), table_name='ratings')
    op.drop_table('ratings')
    op.drop_table('restaurants')
    op.drop_index(op.f('ix_users_phone_number'), table_name='users')
    op.drop_index(op.f('ix_users_last_name'), table_name='users')
    op.drop_index(op.f('ix_users_first_name'), table_name='users')
    op.drop_table('users')
    op.drop_table('addresses')
    # ### end Alembic commands ###


#%%
import sys
sys.path.append('.')

from typer import Typer
from TenderCrab.DataModels import Session, TenderItem
from sqlalchemy import select, func
from parsel import Selector
import re
import logging
import typer

logging.basicConfig(filename='analyze.log', level=logging.INFO)

#%%
session = Session()

#%%
def get_buyer1(body: str):
    temp = re.findall(r'采购人[：:]([\u4e00-\u9fa5\w]+)', body)
    if temp:
        return temp[0]
    else:
        wrm = re.sub(r'<[^>]+>', ' ', body)
        temp = re.findall(r'采购人名?称?[：:]\s*([\u4e00-\u9fa5\w]+)', wrm)
        if temp:
            return temp[0]
        else:
            return None

def get_buyer2(body: str):
    temp = re.sub(r'<[^>]+>', '', body)
    temp1 = re.findall(r'采购人信息[\xa0\s]*名[\xa0\s]*称：([\u4e00-\u9fa5\w]+)', temp)
    if temp1:
        return temp1[0]
    else:
        return None

def update_buyer(batch:int=100, offset:int=0, total_count:int=0):
    global session
    if total_count == 0:
        stmt = select(func.count()).select_from(TenderItem)
        result = session.execute(stmt).fetchone()
        total_count = result[0]

    count = 0
    while count < total_count:
        stmt = select(TenderItem).limit(batch).offset(count + offset)
        result = session.execute(stmt).fetchall()
        count += batch
        for k, row in enumerate(result):
            temp = get_buyer1(row[0].body)
            if temp:
                row[0].buyer = temp
                continue
            temp = get_buyer2(row[0].body)
            if temp:
                row[0].buyer = temp
            else:
                logging.info(f"Failed to find buyer {row[0].id}")
        logging.info(f'{count} of {total_count} is completed.')
        session.commit()
#%%
if __name__ == "__main__":
    typer.run(update_buyer)

# %%
stmt = select(TenderItem).limit(20).offset(38)
result = session.execute(stmt).fetchall()


"""
Poszukaj innych funkcji niż upper i lower i spróbuj ze zdania:
"12 kilogramów jabłek, 10 kilogramów gruszek, 20 kilogramów śliwek"
policzyć ile łącznie razem ważą zakupione produkty (spacje przed kilogramami muszą być)
"""
sentence = '12 kilogramów jabłek, 10 kilogramów gruszek, 20 kilogramów śliwek'
words = sentence.split(' ')
total = 0
for word in words:
    if word.isnumeric():
        total += int(word)
print(f'Lączna waga towarów to {total}')
# words_list = words.split(' ')
# words_list.sort()
# words_list = words_list[0:3]
# value_1, value_2, value_3 = words_list
# sum_value = int(value_1 + value_2 + value_3)
# print(f'Produkty ważą {sum_value}')