# -*- coding: utf-8 -*-
import requests
import logging
import random
import time
import hashlib
import string
import uuid
import json
import glob
import os
import sys

#重要参数
Salt_Sign = 'PVeGWIZACpxXZ1ibMVJPi9inCY4Nd4y2' #米游社签到salt
Salt_BBS = 't0qEgfub6cvueAPgR5m9aQWWVciEer7v' #米游社讨论区专用salt
mysVersion = "2.38.1" #米游社版本
mysClient_type = '2' # 1:ios 2:Android

#日志输出配置
log = logger = logging
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s %(levelname)s %(message)s',
    datefmt='%Y-%m-%dT%H:%M:%S')

#全局变量
VERSION = "v1.6.1"
IdentifyChar = "⨌" #删帖用的特殊识别符号
PATH = os.path.dirname(os.path.realpath(__file__))
Multi_ConfigPath = PATH + "/MultiConfig/{}-config-{}.json"
Multi_CookiePath = PATH + "/MultiConfig/{}-cookie.json"

def LoadConfig(path=f"{PATH}/config.json"):
    '''加载配置文件'''
    with open(path, "r",encoding="utf-8") as f:
            data = json.load(f)
            f.close()
            return data

def LoadCookie(path=f"{PATH}/cookie.json"):
    '''加载缓存的cookie文件'''
    with open(path, "r",encoding="utf-8") as f:
            data = json.load(f)
            f.close()
            return data

def Write(login_ticket,stuid,stoken,path=f"{PATH}/cookie.json"):
    '''写入cookie缓存'''
    params = {"login_ticket": "{}".format(login_ticket), "stuid": "{}".format(stuid), "stoken": "{}".format(stoken)}
    with open(path, "w",encoding="utf-8") as r:
        json.dump(params,r)
        r.close()

def Cleared(path=f"{PATH}/cookie.json"):
    '''清空缓存cookie'''
    if path != f"{PATH}/cookie.json":
        os.remove(path)
    else:
        params = {"login_ticket": "", "stuid": "", "stoken": ""}
        with open(path, "w",encoding="utf-8") as r:
            json.dump(params,r)
            r.close()

def GameHeader(Referer="https://webstatic.mihoyo.com/"): #此referer为默认值
    '''游戏签到福利的header'''
    headers = {
        'Cookie': Cookie,
        'User-Agent': 'Mozilla/5.0 (Linux; Android 12; vivo-s7 Build/RKQ1.211119.001; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/105.0.5195.79 Mobile Safari/537.36 miHoYoBBS/2.38.1',
        'Referer': Referer,
        'DS': DS_BBS(),
        'Accept': 'application/json, text/plain, */*',
        'Accept-Encoding': 'gzip, deflate',
        'Origin': 'https://webstatic.mihoyo.com',
        'X-Requested-With': 'com.mihoyo.hyperion',
        'x-rpc-device_id': str(uuid.uuid3(uuid.NAMESPACE_URL, Cookie)),
        'x-rpc-device_name': 'vivo s7',
        'x-rpc-device_model': 'vivo-s7',
        'x-rpc-sys_version': '12',
        'x-rpc-channel': 'vivo',
        'x-rpc-client_type': mysClient_type,
        'x-rpc-app_version': mysVersion
    }
    return headers

def SleepTime():
    '''随机延迟时间'''
    if delay :
        time.sleep(random.randint(5,10))
    else:
        time.sleep(1)

def md5(text):
    '''md5加密'''
    md5 = hashlib.md5()
    md5.update(text.encode())
    return md5.hexdigest()

def randomStr(n):
    '''生成指定位数的随机数'''
    return (''.join(random.sample(string.digits + string.ascii_letters, n))).lower()

def DS_BBS():
    '''生成米游社DS'''
    n = Salt_Sign
    i = str(int(time.time()))
    r = randomStr(6)
    c = md5("salt=" + n + "&t=" + i + "&r=" + r)
    return "{},{},{}".format(i, r, c)

def DS_discuss(gid):
    '''生成讨论区DS'''
    n = Salt_BBS
    i = str(int(time.time()))
    r = str(random.randint(100001, 200000))
    b = json.dumps({"gids": gid})
    c = md5("salt=" + n + "&t=" + i + "&r=" + r + "&b=" + b + "&q=") #q值为空
    return "{},{},{}".format(i, r, c)

def GetAllRoles():
    '''获取账号的全部角色信息'''
    log.info('获取账号的全部角色信息中......')
    url = 'https://api-takumi.mihoyo.com/binding/api/getUserGameRolesByCookie'

    response = requests.get(url,headers=GameHeader())
    data = json.loads(response.text.encode('utf-8'))
    if '登录失效，请重新登录' in data["message"]:
        log.warning(data)
        log.warning('Cookie已失效,请重新抓取Cookie')
        sys.exit()
    SleepTime()
    return data

def CreateCookieCache(ConfigPath=f"{PATH}/config.json", CookiePath=f"{PATH}/cookie.json"):
    Cookie_url  = "https://webapi.account.mihoyo.com/Api/cookie_accountinfo_by_loginticket?login_ticket={}"
    Cookie_url2 = "https://api-takumi.mihoyo.com/auth/api/getMultiTokenByLoginTicket?login_ticket={}&token_types=3&uid={}"

    log.info("更新Cookie数据中......")
    CookieCache = LoadConfig(ConfigPath)
    if "login_ticket" in CookieCache["Cookie"]:
        c = CookieCache["Cookie"].split(";")
        for i in c:
            if i.split("=")[0] == " login_ticket": #login_ticket前面的空格是必须的
                LoginTicket = i.split("=")[1]
                break
        response = requests.get(url=Cookie_url.format(LoginTicket)) #获取stuid
        data = json.loads(response.text.encode('utf-8'))
        if "成功" in data["data"]["msg"]:
            stuid = data["data"]["cookie_info"]["account_id"]
            response = requests.get(url=Cookie_url2.format(LoginTicket, stuid)) #获取stoken
            data = json.loads(response.text.encode('utf-8'))
            stoken = data["data"]["list"][0]["token"]
            Write(LoginTicket,stuid,stoken,CookiePath)
            log.info('更新成功')
        else:
            log.warning(data)
            log.warning('Cookie已失效,请重新抓取Cookie')
            Cleared(CookiePath)
            sys.exit()
    else:
        log.warning('Cookie中没有"login_ticket"数据,请重新抓取Cookie!')
        sys.exit()

def Multi_Load():
    '''多账号加载'''
    MultiPath = []
    for m in range(len(glob.glob(Multi_ConfigPath.format("*","*")))):
        try:#从1开始轮询，避免顺序错误
            MultiPath.append(glob.glob(Multi_ConfigPath.format(m+1,"*"))[0])
        except:
            break
    MultiCookie = glob.glob(Multi_CookiePath.format("*"))
    TotalConfig = len(MultiPath)
    TotalCookie = len(MultiCookie)

    Cookie_url  = "https://webapi.account.mihoyo.com/Api/cookie_accountinfo_by_loginticket?login_ticket={}"
    Cookie_url2 = "https://api-takumi.mihoyo.com/auth/api/getMultiTokenByLoginTicket?login_ticket={}&token_types=3&uid={}"

    #获取备注信息
    MultiRemark = []
    for r in MultiPath:
        MultiRemark.append(r.split("/")[-1].split("-")[-1].split(".")[0])

    log.info('共找到 {} 个配置文件, {} 个缓存的Cookie数据'.format(TotalConfig,TotalCookie))

    #获取没有缓存cookie的账号序号并创建cookie缓存
    ConfigOrdinal = []
    for c1 in MultiPath:
        ConfigOrdinal.append(c1.split("/")[-1].split("-")[0])
    CookieOrdinal = []
    for c2 in MultiCookie:
        CookieOrdinal.append(c2.split("/")[-1].split("-")[0])
    if TotalCookie != TotalConfig:
        for D in ConfigOrdinal:
            if D not in CookieOrdinal:
                d = int(D)
                log.info('正在为 [{}] 更新cookie数据......'.format(MultiRemark[d-1]))
                CreateCookieCache(ConfigPath=Multi_ConfigPath.format(d, MultiRemark[d-1]), CookiePath=Multi_CookiePath.format(d))
                time.sleep(random.randint(2,5)) #写死休眠时间,防止触发风控
    return MultiPath,MultiRemark,TotalConfig

class BH2_Checkin():

    #api
    Referer_url     = 'https://webstatic.mihoyo.com/'
    Sign_url        = 'https://api-takumi.mihoyo.com/event/luna/sign' #POST json
    Check_url       = 'https://api-takumi.mihoyo.com/event/luna/info?lang=zh-cn&act_id={}&region={}&uid={}'

    #value
    Act_id          = 'e202203291431091'
    Game_biz        = 'bh2_cn'

    def bh2_sign(self,region,uid):
        sign_data = {
            "act_id": self.Act_id,
            "region": region,
            "uid": uid,
            "lang": "zh-cn"
        }
        log.info('崩坏2: 正在为舰长「{}」签到'.format(uid))
        response = requests.post(url=self.Sign_url, headers=GameHeader(Referer=self.Referer_url), json=sign_data)
        data = json.loads(response.text.encode('utf-8'))
        SleepTime()
        return data

    def run(self,list):
        region = list["region"]
        uid = list["game_uid"]

        for i in range(3):
            sign_data = self.bh2_sign(region,uid) #签到并返回数据
            try:
                #从专门的api检查签到结果
                response = requests.get(url=self.Check_url.format(self.Act_id,region,uid), headers=GameHeader())
                data = json.loads(response.text.encode('utf-8'))
                if "OK" in sign_data["message"]:
                    if data["data"]["is_sign"]: # "is_sign" 为布尔值
                        log.info('崩坏2: 签到成功')
                        break
                    else:
                        log.info('崩坏2: 第 {} 次请求成功但并未签上'.format(i + 1))
                elif "已签到" in sign_data["message"]:
                    log.info('崩坏2: 重复签到')
                    break
                else:
                    log.warning(sign_data)
                    log.warning('崩坏2: 签到出现错误,请及时检查')
                    break
            except:
                #直接输出签到api返回的签到结果
                log.warning(data)
                log.warning("未能从检查api得到签到结果,下面为签到api返回的结果,有概率漏签")
                if "OK" in sign_data["message"]:
                    log.info('崩坏2: 签到成功')
                elif '已签到' in sign_data["message"]:
                    log.info('崩坏2: 重复签到')
                else:
                    log.warning(sign_data)
                    log.warning('崩坏2: 签到出现错误,请及时检查')
                break

class BH3_Checkin():

    #api
    Referer_url     = 'https://webstatic.mihoyo.com/bbs/event/signin/bh3/index.html?bbs_auth_required=true&act_id={}&bbs_presentation_style=fullscreen&utm_source=bbs&utm_medium=mys&utm_campaign=icon'
    Sign_url        = 'https://api-takumi.mihoyo.com/event/luna/sign' #POST json
    Check_url       = 'https://api-takumi.mihoyo.com/event/luna/info?lang=zh-cn&act_id={}&region={}&uid={}'

    #value
    Act_id          = 'e202207181446311'
    Game_biz        = 'bh3_cn'

    def bh3_sign(self,region,uid):
        sign_data = {
            "act_id": self.Act_id,
            "region": region,
            "uid": uid,
            "lang": "zh-cn"
        }
        log.info('崩坏3: 正在为舰长「{}」签到'.format(uid))
        response = requests.post(url=self.Sign_url, headers=GameHeader(Referer=self.Referer_url.format(self.Act_id)), json=sign_data)
        data = json.loads(response.text.encode('utf-8'))
        SleepTime()
        return data

    def run(self,list):
        region = list["region"]
        uid = list["game_uid"]

        for i in range(3):
            sign_data = self.bh3_sign(region,uid) #签到并返回数据
            try:
                #从专门的api检查签到结果
                response = requests.get(url=self.Check_url.format(self.Act_id,region,uid), headers=GameHeader())
                data = json.loads(response.text.encode('utf-8'))
                if "OK" in sign_data["message"]:
                    if data["data"]["is_sign"]: # "is_sign" 为布尔值
                        log.info('崩坏3: 签到成功')
                        break
                    else:
                        log.info('崩坏3: 第 {} 次请求成功但并未签上'.format(i + 1))
                elif "已签到" in sign_data["message"]:
                    log.info('崩坏3: 重复签到')
                    break
                else:
                    log.warning(sign_data)
                    log.warning('崩坏3: 签到出现错误,请及时检查')
                    break
            except:
                #直接输出签到api返回的签到结果
                log.warning(data)
                log.warning("未能从检查api得到签到结果,下面为签到api返回的结果,有概率漏签")
                if "OK" in sign_data["message"]:
                    log.info('崩坏3: 签到成功')
                elif '已签到' in sign_data["message"]:
                    log.info('崩坏3: 重复签到')
                else:
                    log.warning(sign_data)
                    log.warning('崩坏3: 签到出现错误,请及时检查')
                break

class YS_Checkin():

    #api
    Referer_url     = 'https://webstatic.mihoyo.com/bbs/event/signin-ys/index.html?bbs_auth_required=true&act_id={}&utm_source=bbs&utm_medium=mys&utm_campaign=icon'
    Sign_url        = 'https://api-takumi.mihoyo.com/event/bbs_sign_reward/sign' #POST json
    Check_url       = 'https://api-takumi.mihoyo.com/event/bbs_sign_reward/info?act_id={}&region={}&uid={}'

    #value
    Act_id          = 'e202009291139501'
    Game_biz        = 'hk4e_cn'

    def ys_sign(self,region,uid):
        sign_data = {
            "act_id": self.Act_id,
            "region": region,
            "uid": uid
        }
        log.info('原神: 正在为旅行者「{}」签到'.format(uid))
        response = requests.post(url=self.Sign_url, headers=GameHeader(Referer=self.Referer_url.format(self.Act_id)), json=sign_data)
        data = json.loads(response.text.encode('utf-8'))
        SleepTime()
        return data

    def run(self,list):
        region = list["region"]
        uid = list["game_uid"]

        for i in range(3):
            sign_data = self.ys_sign(region,uid) #签到并返回数据
            try:
                #从专门的api检查签到结果
                response = requests.get(url=self.Check_url.format(self.Act_id,region,uid), headers=GameHeader())
                data = json.loads(response.text.encode('utf-8'))
                if "OK" in sign_data["message"]:
                    if data["data"]["is_sign"]: # "is_sign" 为布尔值
                        log.info('原神: 签到成功')
                        break
                    else:
                        log.info('原神: 第 {} 次请求成功但并未签上'.format(i + 1))
                elif "旅行者,你已经签到过了" in sign_data["message"]:
                    log.info('原神: 重复签到')
                    break
                else:
                    log.warning(sign_data)
                    log.warning('原神: 签到出现错误,请及时检查')
                    break
            except:
                #直接输出签到api返回的签到结果
                log.warning(data)
                log.warning("未能从检查api得到签到结果,下面为签到api返回的结果,有概率漏签")
                if "OK" in sign_data["message"]:
                    log.info('原神: 签到成功')
                elif "旅行者,你已经签到过了" in sign_data["message"]:
                    log.info('原神: 重复签到')
                else:
                    log.warning(sign_data)
                    log.warning('原神: 签到出现错误,请及时检查')
                break

class MihoyoBBS():
    #api
    Cookie_url          = "https://webapi.account.mihoyo.com/Api/cookie_accountinfo_by_loginticket?login_ticket={}"
    Cookie_url2         = "https://api-takumi.mihoyo.com/auth/api/getMultiTokenByLoginTicket?login_ticket={}&token_types=3&uid={}"
    Sign_url            = "https://bbs-api.mihoyo.com/apihub/app/api/signIn"  # POST json
    List_url            = "https://bbs-api.mihoyo.com/post/api/getForumPostList?forum_id={}&is_good=false&is_hot=false&page_size=20&sort_type=1"
    GetPostFull_url     = "https://bbs-api.mihoyo.com/post/api/getPostFull?post_id={}"
    Share_url           = "https://bbs-api.mihoyo.com/apihub/api/getShareConf?entity_id={}&entity_type=1"
    UpVote_url          = "https://bbs-api.mihoyo.com/apihub/sapi/upvotePost"  # POST json
    UserBusinesses_url  = "https://bbs-api.mihoyo.com/user/api/getUserBusinesses?uid={}" #获取"我的频道"信息
    Missions_url        = "https://bbs-api.mihoyo.com/apihub/api/getUserMissionsState?" #获取任务完成情况
    Draft_url           = "https://bbs-api.mihoyo.com/post/api/draft/save"  # POST json 草稿
    ReleasePost_url     = "https://bbs-api.mihoyo.com/post/api/releasePost/v2" # POST json 发帖
    ReleaseReply_url    = "https://bbs-api.mihoyo.com/post/api/releaseReply"# POST json 发评论
    SelfPostList_url    = "https://bbs-api.mihoyo.com/painter/api/user_instant/list?offset={}&size=20&uid={}" #offset初始值为0
    DeletePost_url      = "https://bbs-api.mihoyo.com/post/api/deletePost" # POST json 删帖

    #米游社分区
    BBS_List = [
        {
            "id": "1",
            "forumId": "1",
            "name": "崩坏3",
            "url": "https://bbs.mihoyo.com/bh3/"
        },
        {
            "id": "2",
            "forumId": "26",
            "name": "原神",
            "url": "https://bbs.mihoyo.com/ys/"
        },
        {
            "id": "3",
            "forumId": "30",
            "name": "崩坏2",
            "url": "https://bbs.mihoyo.com/bh2/"
        },
        {
            "id": "4",
            "forumId": "37",
            "name": "未定事件簿",
            "url": "https://bbs.mihoyo.com/wd/"
        },
        {
            "id": "5",
            "forumId": "34",
            "name": "大别野",
            "url": "https://bbs.mihoyo.com/dby/"
        },
        {
            "id": "6",
            "forumId": "52",
            "name": "星穹铁道",
            "url": "https://bbs.mihoyo.com/sr/"
        },
        {
            "id": "8",
            "forumId": "57",
            "name": "绝区零",
            "url": "https://bbs.mihoyo.com/zzz/"
        }
    ]

    def __init__(self):
        self.LoginTicket = CookieCache["login_ticket"]
        self.stuid = CookieCache["stuid"]
        self.stoken = CookieCache["stoken"]

        self.CheckMission = True #检查任务完成情况

        if self.LoginTicket == "" or self.stuid == "" or self.stoken == "":
            CreateCookieCache()

        self.headers = {
            "Cookie": f'login_ticket={self.LoginTicket};stuid={self.stuid};stoken={self.stoken}',
            'User-Agent': "okhttp/4.8.0",
            "DS": "", #随用随更新，保证实时性
            "x-rpc-client_type": mysClient_type,
            "x-rpc-app_version": mysVersion,
            "x-rpc-device_id": str(uuid.uuid3(uuid.NAMESPACE_URL, Cookie)),
            "x-rpc-device_name": "vivo s7",
            "x-rpc-device_model": "vivo-s7",
            "x-rpc-sys_version": "12",
            "x-rpc-channel": "miyousheluodi",
            "Accept-Encoding": "gzip",
            "Referer": "https://app.mihoyo.com",
            "Host": "bbs-api.mihoyo.com"
        }

        self.BBS_WhiteList = self.UserBusinesses()["data"]["businesses"]

    def UserBusinesses(self):
        log.info('获取米游社 “我的频道” 信息中......')
        self.headers["DS"] = DS_BBS()
        response = requests.get(url=self.UserBusinesses_url.format(self.stuid), headers=self.headers)
        data = json.loads(response.text.encode('utf-8'))
        if "OK" in data["message"]:
            SleepTime()
            return data
        elif "登录失效，请重新登录" in data["message"]:
            log.warning('Cookie已失效,请重新抓取Cookie')
            Cleared()
        else:
            log.warning(data)
            log.warning('获取账号 “我的频道” 信息失败,退出签到')
            sys.exit()

    def SignIn(self):
        log.info('讨论区签到中......')
        for i in self.BBS_List:
            if i["id"] in self.BBS_WhiteList:
                self.headers["DS"] = DS_discuss(gid=i["id"])
                response = requests.post(url=self.Sign_url, headers=self.headers, json={"gids": i["id"]})
                data = json.loads(response.text.encode('utf-8'))
                if "登录失效，请重新登录" not in data["message"]:
                    log.info(i["name"] + ": " + data["message"])
                    SleepTime()
                else:
                    log.warning(data)
                    log.warning('签到失败,你的Cookie可能已过期,请重新抓取Cookie')
                    Cleared()
                    sys.exit()

    def Only_MYB(self):
        '''米游币任务'''
        List = []
        #获取帖子列表
        self.headers["DS"] = DS_BBS()
        response = requests.get(url=self.List_url.format('26'), headers=self.headers) #获取的原神频道帖子
        data = json.loads(response.text.encode('utf-8'))
        #将获取到的帖子的id写入List
        PostSum = len(data["data"]["list"])
        for n in range(PostSum):
            List.append(data["data"]["list"][n]["post"]["post_id"])

        #看帖3篇
        Success = 0
        Count = 0
        log.info('浏览3篇帖子中......')
        while Success < 3 and Count < PostSum:
            self.headers["DS"] = DS_BBS()
            response = requests.get(url=self.GetPostFull_url.format(List[Count]), headers=self.headers)
            data = json.loads(response.text.encode('utf-8'))
            if "OK" in data["message"]:
                Count += 1
            elif "帖子不存在" in data["message"]:
                log.warning('帖子(ID:{})可能已被作者或管理员删除'.format(List[Count]))
                Count += 1
            else:
                log.warning(data)
                log.warning("浏览帖子出现问题,请及时检查")
                Count = 114514 #手动定义数值达到退出循环的效果
            #检查任务是否完成
            if Count >= 3 and self.CheckMission:
                try:
                    response = requests.get(url=self.Missions_url + "point_sn=myb", headers=GameHeader()) #使用的是游戏签到header!!!
                    data = json.loads(response.text.encode('utf-8'))
                    for i in data["data"]["states"]:
                        if i["mission_key"] == "view_post_0": #米游币每日任务-浏览3个帖子
                            Success = i["happened_times"]
                            break
                except:
                    log.warning("检查任务是否完成失败,跳过检查")
                    log.warning(data)
                    self.CheckMission = False
            else:
                Success = Count
            SleepTime()
        else:
            if self.CheckMission:
                log.info("检查结果: 浏览成功 {} 篇".format(Success))
            else:
                log.info('未能从检查api得到签到结果,有概率漏签')
            if Success < 3:
                if Count == 114514:
                    Count = ' "( !!出现错误!! )" '
                log.warning('尝试了{}篇帖子,浏览成功{}篇,未能完成任务'.format(Count,Success))

        #点赞5篇
        Success = 0
        Count = 0
        log.info('点赞5篇帖子中......')
        while Success < 5 and Count < PostSum:
            self.headers["DS"] = DS_BBS()
            response = requests.post(url=self.UpVote_url, headers=self.headers,json={"post_id": List[Count], "is_cancel": False})
            data = json.loads(response.text.encode('utf-8'))
            if "OK" in data["message"]:
                Count += 1
            elif "帖子不存在" in data["message"]:
                log.warning('帖子(ID:{})可能已被作者或管理员删除'.format(List[Count]))
                Count += 1
            else:
                log.warning(data)
                log.warning("点赞出现问题,请及时检查")
                Count = 114514 #手动定义数值达到退出循环的效果
            #检查任务是否完成
            if Count >= 5 and self.CheckMission:
                response = requests.get(url=self.Missions_url + "point_sn=myb", headers=GameHeader()) #使用的是游戏签到header!!!
                data = json.loads(response.text.encode('utf-8'))
                #上一步"看帖3篇"try过了
                for i in data["data"]["states"]:
                    if i["mission_key"] == "post_up_0": #米游币每日任务-完成5次点赞
                        Success = i["happened_times"]
                        break
            else:
                Success = Count
            SleepTime()
        else:
            if self.CheckMission:
                log.info("检查结果: 点赞成功 {} 篇".format(Success))
            else:
                log.info('未能从检查api得到签到结果,有概率漏签')
            if Success < 5:
                if Count == 114514:
                    Count = ' "( !!出现错误!! )" '
                log.warning('尝试了{}篇帖子,点赞成功{}篇,未能完成任务'.format(Count,Success))

        #分享1篇
        Success = 0
        Count = 0
        log.info('分享1篇帖子中......')
        while Success < 1 and Count < PostSum:
            self.headers["DS"] = DS_BBS()
            response = requests.get(url=self.Share_url.format(List[Count]), headers=self.headers)
            data = json.loads(response.text.encode('utf-8'))
            if "OK" in data["message"]:
                Count += 1
            elif "帖子不存在" in data["message"]:
                log.warning('帖子(ID:{})可能已被作者或管理员删除'.format(List[Count]))
                Count += 1
            else:
                log.warning(data)
                log.warning("点赞出现问题,请及时检查")
                Count = 114514 #手动定义数值达到退出循环的效果
            #检查任务是否完成
            if Count >= 1 and self.CheckMission:
                response = requests.get(url=self.Missions_url + "point_sn=myb", headers=GameHeader()) #使用的是游戏签到header!!!
                data = json.loads(response.text.encode('utf-8'))
                #上上步"看帖3篇"try过了
                for i in data["data"]["states"]:
                    if i["mission_key"] == "share_post_0": #米游币每日任务-分享帖子
                        Success = i["happened_times"]
                        break
            else:
                Success = Count
            SleepTime()
        else:
            if self.CheckMission:
                log.info("检查结果: 分享成功 {} 篇".format(Success))
            else:
                log.info('未能从检查api得到签到结果,有概率漏签')
            if Success < 1:
                if Count == 114514:
                    Count = ' !!出现错误!! '
                log.warning('尝试了分享{}篇帖子,居然一篇都没有成功,未能完成任务'.format(Count))

    def Channel_UpVote(self,channel):
        '''各频道点赞任务'''
        List = []
        log.info("正在执行「{}」频道 “点赞10篇帖子” 任务......".format(channel["name"]))
        #获取该频道帖子列表
        self.headers["DS"] = DS_BBS()
        response = requests.get(url=self.List_url.format(channel["forumId"]), headers=self.headers)
        data = json.loads(response.text.encode('utf-8'))
        #将获取到的帖子的id写入List
        PostSum = len(data["data"]["list"])
        for n in range(PostSum):
            List.append(data["data"]["list"][n]["post"]["post_id"])

        #点赞10篇
        Success = 0
        Count = 0
        while Success < 10 and Count < PostSum:
            self.headers["DS"] = DS_BBS()
            response = requests.post(url=self.UpVote_url, headers=self.headers,json={"post_id": List[Count], "is_cancel": False})
            data = json.loads(response.text.encode('utf-8'))
            if "OK" in data["message"]:
                Count += 1
            elif "帖子不存在" in data["message"]:
                log.warning('帖子(ID:{})可能已被作者或管理员删除'.format(List[Count]))
                Count += 1
            else:
                log.warning(data)
                log.warning("点赞出现问题,请及时检查")
                Count = 114514 #手动定义数值达到退出循环的效果
            #检查任务是否完成
            if Count >= 10 and self.CheckMission:
                try:
                    response = requests.get(url=self.Missions_url + "gids={}".format(channel["id"]), headers=GameHeader()) #使用的是游戏签到header!!!
                    data = json.loads(response.text.encode('utf-8'))
                    for i in data["data"]["states"]:
                        if i["mission_key"] == "post_up_{}".format(channel["id"]):
                            Success = i["happened_times"]
                            break
                except:
                    log.warning("检查任务是否完成失败,跳过检查")
                    self.CheckMission = False
            else:
                Success = Count
            SleepTime()
        else:
            if self.CheckMission:
                log.info("检查结果: 点赞成功 {} 篇".format(Success))
            else:
                log.info('未能从检查api得到签到结果,有概率漏签')
            if Success < 10:
                if Count == 114514:
                    Count = ' "( !!出现错误!! )" '
                log.warning('尝试了{}篇帖子,点赞成功{}篇,未能完成任务'.format(Count,Success))

    #以下为实验性功能
    #↓ ↓ ↓ ↓ ↓ ↓ ↓ ↓
    def RandomElements(self):
        '''随机标题与内容'''
        SubjectList = ["水帖一篇","水水水","今日份的水帖","水帖得经验","随手一水","灌水灌水","欸嘿水一帖","每日任务之水帖"]
        ContentList = ["水帖不知道写啥","求点赞评论收藏","为了经验大家快来水帖呀","今日份的经验我来啦","水帖多多经验多多","愿早日升级"]
        Reply1List = ["我回我自己","发帖回复一条龙","为了经验必须回帖","今日份的经验get","水个回复","回复一下愿早日升级"]
        Reply2List = ["每日任务需要的回复","什么大伟出奇迹","飞鱼丸与包菜与应急食品","勤劳的孩子升级快","Ok经验到手","回复一下加快升级"]
        Subject = SubjectList[random.randint(0,len(SubjectList) - 1)]
        Content = ContentList[random.randint(0,len(ContentList) - 1)]
        Reply1 = Reply1List[random.randint(0,len(Reply1List) - 1)]
        Reply2 = Reply2List[random.randint(0,len(Reply2List) - 1)]
        return Subject,Content,Reply1,Reply2

    def Draft(self,subject,content,forumId,id):
        '''写入草稿'''
        draft_data = {
            "collection_id": 0,
            "content": "\u003cp\u003e" + content + "\u003c/p\u003e",
            "cover": "",
            "draft_id": "",
            "forum_cate_id": "",
            "forum_id": forumId,
            "gids": id,
            "is_original": 0,
            "is_profit": False,
            "link_card_list": [],
            "republish_authorization": 1,
            "structured_content": "[{\"insert\":\"" + content + "\\n\"}]",
            "subject": subject,
            "topic_ids": ["877"], #"每日一水"话题分区id
            "view_type": 1
        }
        count = 0
        while count < 3:
            self.headers["DS"] = DS_BBS()
            response = requests.post(url=self.Draft_url, headers=self.headers, json=draft_data)
            data = json.loads(response.text.encode('utf-8'))
            if data["retcode"] == 0:
                return(data["data"]["draft_id"])
            elif data["retcode"] == -1:
                count += 1
                log.info('系统繁忙,1分钟后再次尝试')
                time.sleep(random.randint(55,65)) #写死休眠时间,防止触发风控
            else:
                log.warning(data)
                log.warning('写入草稿发生错误')
                return("err")
        else:
            log.warning(data)
            log.warning('写入草稿失败')
            return("err")

    def ReleasePost(self,subject,content,forumId,id,DraftId):
        '''发帖'''
        post_data = {
            "collection_id": 0,
            "content": "\u003cp\u003e" + content + "\u003c/p\u003e",
            "cover": "",
            "draft_id": DraftId,
            "forum_cate_id": "",
            "f_forum_id": forumId,
            "gids": id,
            "is_original": 0,
            "is_pre_publication": False,
            "is_profit": False,
            "post_id": "",
            "republish_authorization": 0,
            "structured_content": "[{\"insert\":\"" + content + "\\n\"}]",
            "subject": subject,
            "topic_ids": ["877"], #"每日一水"话题分区id
            "view_type": 1
        }
        count = 0
        while count < 3:
            self.headers["DS"] = DS_BBS()
            response = requests.post(url=self.ReleasePost_url, headers=self.headers, json=post_data)
            data = json.loads(response.text.encode('utf-8'))
            if data["retcode"] == 0:
                return(data["data"]["post_id"])
            elif data["retcode"] == -1:
                count += 1
                log.info('系统繁忙,1分钟后再次尝试')
                time.sleep(random.randint(55,65)) #写死休眠时间,防止触发风控
            else:
                log.warning(data)
                log.warning('发帖发生错误')
                return("err")
        else:
            log.warning(data)
            log.warning('发帖失败')
            return("err")

    def ReleaseReply(self,PostId,reply):
        '''发评论'''
        reply_data = {
            "content": reply,
            "post_id": PostId,
            "reply_id": "",
            "structured_content": "[{\"insert\":\"" + reply + "\"}]"
        }
        count = 0
        while count < 3:
            self.headers["DS"] = DS_BBS()
            response = requests.post(url=self.ReleaseReply_url, headers=self.headers, json=reply_data)
            data = json.loads(response.text.encode('utf-8'))
            if data["retcode"] == 0:
                return data["message"]
            elif "帖子不存在" in data["message"]:
                log.warning('帖子不存在,无法评论')
                return("err")
            elif data["retcode"] == -1: #这条if未经验证,仅为推测,"帖子不存在"的retcode值也为-1
                count += 1
                log.info('系统繁忙,1分钟后再次尝试')
                time.sleep(random.randint(55,65)) #写死休眠时间,防止触发风控
            else:
                log.warning(data)
                log.warning('发评论发生错误')
                return("err")
        else:
            log.warning(data)
            log.warning('评论失败')
            return("err")

    def DeletePost(self):
        #获取需要删除的帖子
        SelfPost = []
        offset,limit = 0,0
        last = False
        while last == False and limit < 5:
            limit += 1
            self.headers["DS"] = DS_BBS()
            response = requests.get(url=self.SelfPostList_url.format(offset,self.stuid), headers=self.headers)
            data = json.loads(response.text.encode('utf-8'))
            if data["retcode"] == 0:
                offset = data["data"]["next_offset"]
                last = data["data"]["is_last"]
                for list in data["data"]["list"]:
                    PostSubject = list["post"]["post"]["subject"]
                    #双重识别,避免误删
                    if IdentifyChar in PostSubject:
                        if PostSubject.split(IdentifyChar)[-1] == "(1/2)" or PostSubject.split(IdentifyChar)[-1] == "(2/2)":
                            SelfPost.append(list["post"]["post"]["post_id"])
            else:
                log.info(data)
            SleepTime()
        #开始删帖
        for id in SelfPost:
            delete_data = {
                "operate_type": 0,
                "post_id": id
                }
            self.headers["DS"] = DS_BBS()
            response = requests.post(url=self.DeletePost_url, headers=self.headers, json=delete_data)
            data = json.loads(response.text.encode('utf-8'))
            if data["retcode"] == 0:
                log.info('删除帖子(id:{}) 成功'.format(id))
            else:
                log.info(data)
            SleepTime()

    def Channel_Publish(self,channel):
        '''执行各频道发帖&发评论任务'''
        for i in range(2): #发帖2篇
            subject,content,reply1,reply2 = self.RandomElements()
            subject += " {}({}/2)".format(IdentifyChar, i+1)

            #写入草稿
            DraftId = self.Draft(subject,content,channel["forumId"],channel["id"])
            time.sleep(random.randint(12,15)) #写死休眠时间,防止触发风控

            #发帖子
            if DraftId == "err":
                return("err")
            else:
                PostId = self.ReleasePost(subject,content,channel["forumId"],channel["id"],DraftId)
                time.sleep(random.randint(20,30)) #写死休眠时间,防止触发风控

            #给每篇自己的帖子回复2次
            if PostId == "err":
                Reply1Data = "未发帖"
                Reply2Data = "未发帖"
            else:
                Reply1Data = self.ReleaseReply(PostId,reply1)
                time.sleep(random.randint(150,180)) #写死休眠时间,防止触发风控
                Reply2Data = self.ReleaseReply(PostId,reply2)

            #改为全角,保证对齐,强迫症直呼满足
            PerfectName = channel["name"] #传递给中间参数，避免修改源数据
            if PerfectName == "崩坏3":
                PerfectName = "崩坏３"
            elif PerfectName == "崩坏2":
                PerfectName = "崩坏２"
            elif PerfectName == "原神":
                PerfectName = "原　神"

            log.info('频道:{:　<4}, 次数:{}, DraftID:{}, PostID:{}, 评论:{},{}'.format(PerfectName, i+1, DraftId, PostId, Reply1Data, Reply2Data))
            time.sleep(random.randint(300,320)) #亲测发帖间隔5分钟以上较为安全不太会触发风控
    #↑ ↑ ↑ ↑ ↑ ↑ ↑ ↑
    #以上为实验性功能

    def run(self):
        if Enable["BBS"]:
            log.info('开始执行米游币任务......')
            self.SignIn() #讨论区签到
            self.Only_MYB() #3次浏览,5次点赞,1次分享

        if Enable["ChannelUpVote"]:
            log.info('开始执行各频道点赞任务......')
            for channel in self.BBS_List:
                if channel["id"] in self.BBS_WhiteList:
                    self.Channel_UpVote(channel)

        if Enable["DeleteOldPost"]:
            log.info('开始删除今天之前水的贴子')
            self.DeletePost()

        if Enable["ChannelPublish"]:
            log.info('开始执行各频道发帖&评论任务......')
            log.info('!!!作者亲测发帖间隔5分钟以上较为安全不太会触发风控,故完成本项任务需要巨长时间!!!')
            log.info('预估完成本项任务需要 {} 分钟,请一定要耐心等待'.format(len(self.BBS_WhiteList) * 2 * 8)) #乘数的第一个为每频道发帖次数，第二个为间隔时间
            for channel in self.BBS_List:
                if channel["id"] in self.BBS_WhiteList:
                    result = self.Channel_Publish(channel)
                    if result == "err":
                        log.warning('退出执行各频道发帖&评论任务')
                        break

def StartRun():
    if delay:
        log.info('已启用随机延迟,请耐心等待')

    #实验性功能警告
    if Enable["ChannelPublish"] or Enable["DeleteOldPost"]:
        log.info('!!!您已开启实验性功能,可能存在未知bug,且会有各种不可预估的风险(如禁言,封号),请酌情选择使用!!!')

    #游戏每日签到
    if Enable["BH2"] or Enable["BH3"] or Enable["YS"]:
        for list in GetAllRoles()["data"]["list"]:
            if list["game_biz"] == "bh2_cn" and list["game_uid"] not in Game_BlackList["BH2"] and Enable["BH2"]:
                BH2_Checkin().run(list) #崩坏2每日签到
            if list["game_biz"] == "bh3_cn" and list["game_uid"] not in Game_BlackList["BH3"] and Enable["BH3"]:
                BH3_Checkin().run(list) #崩坏3福利补给
            elif list["game_biz"] == "hk4e_cn" and list["game_uid"] not in Game_BlackList["YS"] and Enable["YS"]:
                YS_Checkin().run(list) #原神每日签到

    #米游社签到
    if Enable["BBS"] or Enable["ChannelUpVote"] or Enable["ChannelPublish"] or Enable["DeleteOldPost"]:
        MihoyoBBS().run()

if __name__ == '__main__':
    if "updata" in sys.argv: #手动更新cookie
        log.info('请选择为哪个模式的config更新cookie(1:单账号  2:多账号)')
        mode = int(input("请输入对应的数字："))
        if mode == 1:
            CreateCookieCache()
        elif mode == 2:
            number = int(input('请输入多账号config文件的序号：'))
            CreateCookieCache(ConfigPath=glob.glob(Multi_ConfigPath.format(number,"*"))[0], CookiePath=Multi_CookiePath.format(number))

    elif "multi" in sys.argv: #多账号
        log.info(f'欢迎使用 MihoyoBBS-AutoSign {VERSION} (多账号版)')

        #读取多账号配置
        MultiPath,MultiRemark,TotalConfig = Multi_Load()
        for n in range(TotalConfig):
            log.info('{:=^46}'.format(f'正在执行第 {n+1} 个账号(备注信息:{MultiRemark[n]})'))
            try:
                config = LoadConfig(Multi_ConfigPath.format(n+1,MultiRemark[n]))
                delay = config["Delay"]
                Cookie = config["Cookie"]
                Game_BlackList = config["Game_BlackList"]
                Enable = config["Enable"]
                CookieCache = LoadCookie(Multi_CookiePath.format(n+1))
            except:
                log.warning('有配置信息未能读取,请检查配置文件是否有错误或是否为最新版')
                sys.exit()
            StartRun()

    else:#单账号
        log.info(f'欢迎使用 MihoyoBBS-AutoSign {VERSION}')

        #读取配置文件
        try:
            config = LoadConfig()
            delay = config["Delay"]
            Cookie = config["Cookie"]
            Game_BlackList = config["Game_BlackList"]
            Enable = config["Enable"]
            CookieCache = LoadCookie()
        except:
            log.warning('有配置信息未能读取,请检查配置文件是否有错误或是否为最新版')
            sys.exit()
        StartRun()

    log.info('任务全部完成\n')


"""
    Description : This Program detect diabetes using Machine Learning and Python
"""
# run block of code and catch warnings
import warnings

with warnings.catch_warnings():
    warnings.filterwarnings("ignore")

import pandas as pd
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
import streamlit as st

st.write("""
# Diabetes Detection
Application of Python in Real-World Situations
""")

data = pd.read_csv("diabetes.csv")

st.subheader("Data Information")

st.dataframe(data)

st.write(data.describe())

chart = st.bar_chart(data)

X = data.iloc[: , 0:8].values
Y = data.iloc[:, -1].values

x_train,x_test,y_train,y_test = train_test_split(X,Y,test_size = 0.25, random_state = 0)

def get_user_input():
    pregnancies = st.sidebar.slider("Pregnancies",0,17,3)
    glucose = st.sidebar.slider("Glucose",0,199,117)
    blood_pressure = st.sidebar.slider("Blood Pressure",0,122,72)
    skin_thickness = st.sidebar.slider("Skin Thickness",0,99,23)
    insulin = st.sidebar.slider("Insulin",0.0,846.0,30.0)
    bmi = st.sidebar.slider("BMI",0.0,67.1,32.0)
    diabetes_pedigree_function = st.sidebar.slider("Diabetes Pedigree Function",0.078,2.42,0.3725)
    age = st.sidebar.slider("Age",0,200,29)

    user_data = {"pregnancies": pregnancies,
                 "glucose" : glucose,
                 "blood_pressure" : blood_pressure,
                 "skin_thickness" : skin_thickness,
                 "insulin" : insulin,
                 "bmi" : bmi,
                 "diabetes_pedigree_function" : diabetes_pedigree_function,
                 "age" : age
                 }
    features = pd.DataFrame(user_data, index = [0])
    return features

user_input = get_user_input()

st.subheader("User Input :")
st.write(user_input)

model = RandomForestClassifier(n_estimators = 100)
model.fit(x_train,y_train)

#st.subheader("Model Test Accuracy Score:")
#st.write(str(accuracy_score(y_test) , model.predict(x_test))*100 + "%")

predictions = model.predict(user_input)

if str(predictions) == '[0]':
    st.subheader("Predictions")
    st.write("The person is not diabetic.")
elif str(predictions) == '[1]':
    st.subheader("Predictions")
    st.write("The person is diabetic.")


import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torchvision.utils as utils
from torchvision import datasets, transforms
from torch.utils.data import DataLoader

device = 'cuda' if torch.cuda.is_available() else 'cpu'
batch_size = 500

# The output of torchvision datasets are PILImage images of range [0, 1].
# We transform them to Tensors of normalized range [-1, 1].
transform = transforms.Compose(
    [transforms.ToTensor(),
     transforms.Normalize((0.5), (0.5))])

# MNIST Dataset
train_dataset = datasets.MNIST(root='data/', train=True, transform=transform, download=True)
test_dataset = datasets.MNIST(root='data/', train=False, transform=transform)

# Data Loader
train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True)
test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False)

class Model(nn.Module):
    def __init__(self):
        super(Model, self).__init__()
        self.l1 = nn.Linear(784, 512)
        self.l2 = nn.Linear(512, 240)
        self.l3 = nn.Linear(240, 10)

    def forward(self, input):
        output = input.view(-1, 784) # Flatten the data (n, 1, 28, 28)-> (n, 784)
        output = F.relu(self.l1(output))
        output = F.relu(self.l2(output))
        output = self.l3(output)
        return output

model = Model()
model.to(device)

criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.003)

def train(epoch):
    model.train()

    for batch_idx, (data, label) in enumerate(train_loader):
        data, label = data.to(device), label.to(device)

        optimizer.zero_grad()
        output = model(data)
        loss = criterion(output, label)
        loss.backward()
        optimizer.step()

        if batch_idx % 10 == 0:
            print('Train Epoch: {} | Batch Status: {}/{} ({:.0f}%) | Loss: {:.6f}'.format(
                epoch, batch_idx * len(data), len(train_loader.dataset),
                100. * batch_idx / len(train_loader), loss.item()))

def test():
    model.eval()
    test_loss = 0
    correct = 0

    for data, label in test_loader:
        data, label = data.to(device), label.to(device)

        output = model(data)
        test_loss += criterion(output, label).item()
        pred = output.data.max(1, keepdim=True)[1]
        correct += pred.eq(label.data.view_as(pred)).cpu().sum()

    test_loss /= len(test_loader.dataset)
    print(f'===========================\nTest set: Average loss: {test_loss:.4f}, Accuracy: {correct}/{len(test_loader.dataset)} '
          f'({100. * correct / len(test_loader.dataset):.0f}%)')

for epoch in range(3):
    train(epoch)
    test()


from typing import Literal, TypedDict
from urllib.parse import urljoin

from scrapy import Request, Spider
from scrapy.selector import Selector

from csv_writer import CSVWriter

PartOfSpeech = Literal["Verb", "Adjektiv", "Substantiv"]

class LemmaFilters(TypedDict, total=False):
    part_of_speech: set[PartOfSpeech]
    starts_with: str
    does_not_start_with: list[str]

LemmaFeature = Literal["url", "title", "part_of_speech", "hyphenation"]

SOFT_HYPHEN = "\xad"

class DudenSpider(Spider):
    """Extract Information from duden.de

    Spider has 2 levels: 1st one handles search results, 2nd one lemma page.
    """

    name = "duden spide"
    base_url = "https://www.duden.de/"
    filters: LemmaFilters
    features: list[LemmaFeature]

    def parse(
        self, search_result_page
    ):  # pylint: disable=arguments-renamed, arguments-differ
        lemma_outline_path = "*//section[@class='vignette']"
        outlines = search_result_page.xpath(lemma_outline_path)
        for lemma_outline in outlines:
            if self._lemma_outline_is_relevant(lemma_outline):
                url = urljoin(
                    self.base_url,
                    self._extract_relative_lemma_url(lemma_outline),
                )
                yield Request(
                    url=url,
                    callback=self.parse_lemma_page,
                )
        next_relative_url = self._extract_next_page_url(search_result_page)
        if next_relative_url:
            yield Request(
                url=urljoin(
                    search_result_page.request.url,
                    next_relative_url,
                ),
                callback=self.parse,
            )

    @staticmethod
    def _extract_next_page_url(search_result_page) -> str:
        next_page_link_path = "//a[@class='pager__item' and @rel='next']/@href"
        result = search_result_page.xpath(next_page_link_path).extract()
        if not result:
            return None

        return result[0]

    @staticmethod
    def _clean_soft_hyphens(word: str) -> str:
        return word.replace(SOFT_HYPHEN, "")

    def _lemma_outline_is_relevant(self, lemma_outline: Selector) -> bool:
        """Outline is relevant if it does not contradict filters"""

        lemma_title = self._extract_lemma_title(lemma_outline)
        if "starts_with" in self.filters:
            if not any(
                lemma_title.startswith(mandatory_start)
                for mandatory_start in self.filters["starts_with"]
            ):
                return False

        if "does_not_start_with" in self.filters:
            if any(
                lemma_title.startswith(forbidden_beginning)
                for forbidden_beginning in self.filters["does_not_start_with"]
            ):
                return False

        if "part_of_speech" in self.filters:
            description = self._extract_lemma_description(lemma_outline)
            if not any(
                description.startswith(relevant_part_of_speech)
                for relevant_part_of_speech in self.filters["part_of_speech"]
            ):
                return False

        return True

    def _extract_lemma_title(self, lemma_outline: Selector) -> str:
        title_path = "h2[@class='vignette__title']/a/strong/text()"
        result = lemma_outline.xpath(title_path).extract()[0]
        return self._clean_soft_hyphens(result)

    @staticmethod
    def _extract_lemma_description(lemma_outline: Selector) -> str:
        description_path = "p[@class='vignette__snippet']/text()"
        return lemma_outline.xpath(description_path).extract()[0].strip()

    @staticmethod
    def _extract_relative_lemma_url(lemma_outline: Selector) -> str:
        url_path = "a[@class='vignette__link']/@href"
        relative_url = lemma_outline.xpath(url_path).extract()[0]
        return relative_url

    def parse_lemma_page(self, lemma_page) -> None:
        result = {}
        if "url" in self.features:
            result["url"] = lemma_page.request.url
        if "title" in self.features:
            result["title"] = self._extract_title(lemma_page)
        if "hyphenation" in self.features:
            result["hyphenation"] = self._extract_hyphenation(lemma_page)
        if "part_of_speech" in self.features:
            result["part_of_speech"] = self._extract_pos(lemma_page)
        self.csv_writer.writerow(result)

    @staticmethod
    def _extract_pos(lemma_page) -> str | None:
        pos_path = (
            "//article/dl["
            "dt["
            "@class='tuple__key' "
            "and contains(text(), 'Wortart')"
            "]"
            "]/dd[@class='tuple__val']/text()"
        )
        pos = lemma_page.xpath(pos_path).extract()
        if not pos:
            return None
        return pos[0]

    @staticmethod
    def _extract_raw_title(lemma_page) -> str:
        title_path = "//div[@class='lemma']/h1/span/text()"
        return lemma_page.xpath(title_path).extract()[0]

    def _extract_title(self, lemma_page) -> str:
        raw_title = self._extract_raw_title(lemma_page)
        return self._clean_soft_hyphens(raw_title)

    def _extract_hyphenation(self, lemma_page) -> str:
        raw_title = self._extract_raw_title(lemma_page)
        hyphenation = raw_title.replace(SOFT_HYPHEN, " | ")
        return hyphenation

def make_duden_spider(
    output_path: str,
    search_term: str,
    features: list[LemmaFeature],
    filters: LemmaFilters | None = None,
) -> DudenSpider:
    filters = filters or {}
    ConfiguredDudenSpider = DudenSpider
    ConfiguredDudenSpider.filters = filters
    ConfiguredDudenSpider.features = features

    ConfiguredDudenSpider.start_urls = [
        f"https://www.duden.de/suchen/dudenonline/{search_term}"
    ]
    ConfiguredDudenSpider.csv_writer = CSVWriter(output_path, fields=features)
    return ConfiguredDudenSpider


import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
from torch.autograd import Variable
from torch.utils.data import dataloader
from torch.backends import cudnn
import random
from tqdm import *
import decimal
from copy import deepcopy
from dataset import return_similar_word_to_vector
from transformer import EncoderLayer
from transformer import PositionalEncoder
from copy import deepcopy

# torch.has_cudnn = False

def frange(x, y, jump):
  while x < y:
    yield x
    x += jump

def squash(tensor, dim=-1):
    squared_norm = (tensor ** 2).sum(dim=dim, keepdim=True)
    scale = squared_norm / (1 + squared_norm)
    return scale * tensor / torch.sqrt(squared_norm)

def load_saved_model(num_of_topics=11, hidden_size=128, topic_hidden_size=32, drop_out_prob=0.6,
                     model_path='./topic-attention'):
    model = TopicAttention(num_of_topics=num_of_topics, hidden_size=hidden_size,
                           topic_hidden_size=topic_hidden_size, drop_out_prob=drop_out_prob)
    model.load_state_dict(torch.load(model_path))
    model.eval()
    return model

def examine_context_vectors(num_of_topics, model_path):
    model = load_saved_model(num_of_topics, model_path)
    for vector in model.attn_context:
        print(return_similar_word_to_vector(vector.detach().numpy()))

def get_sentence_weights(model_path, sentence):
    model = load_saved_model()
    sentence = Variable(torch.from_numpy(sentence)).float()
    if torch.has_cudnn:
        sentence.cuda()
    weights, probs = model.return_weights(sentence.unsqueeze(0))
    return weights, probs

class topkCE(nn.Module):
    def __init__(self, k=0.7):
        super(topkCE, self).__init__()
        self.loss = nn.NLLLoss()
        self.k = k
        self.softmax = nn.LogSoftmax(dim=1)
        return

    def forward(self, input, target):
        softmax_result = self.softmax(input)

        loss = Variable(torch.Tensor(1).zero_())
        if torch.has_cudnn:
            loss = loss.cuda()
        for idx, row in enumerate(softmax_result):
            gt = target[idx]
            pred = torch.unsqueeze(row, 0)
            cost = self.loss(pred, gt.unsqueeze(0))
            loss = torch.cat((loss, cost.unsqueeze(0)), 0)
        loss = loss[1:]
        if self.k == 1.0:
            valid_loss = loss
        else:
            index = torch.topk(loss, int(self.k * loss.size()[0]))
            valid_loss = loss[index[1]]
        return torch.mean(valid_loss)

class TopicAttentionLayer(nn.Module):
    def __init__(self, input_size, num_of_topics):
        super(TopicAttentionLayer, self).__init__()
        self.attention_context = nn.ModuleList([nn.Linear(input_size, input_size) for _ in range(num_of_topics)])
        self.attention_linear_trans = nn.ModuleList([nn.Linear(input_size, input_size)
                                                       for _ in range(num_of_topics)])
        self.num_of_topics = num_of_topics
        self.attention_drop_out = nn.Dropout(.1)

    def forward(self, x, mask=None):
        topic_output = []
        for i in range(self.num_of_topics):
            energy = self.attention_context[i](x)
            if mask is not None:
                energy = energy.masked_fill(mask, -np.inf)
            probs = F.softmax(energy, dim=1)
            probs = self.attention_drop_out(probs)
            out = torch.sum(x * probs, dim=1)

            out = self.attention_linear_trans[i](out)
            out = F.relu(out)

            topic_output.append(out.unsqueeze(1))
        return torch.cat(topic_output, dim=1)

    def regularization_loss(self):
        attn_contexts = torch.cat([self.attention_context[i].weight.view(-1).unsqueeze(0) for i in range(self.num_of_topics)], dim=0)
        normalized_context_values = F.normalize(attn_contexts, dim=1)
        reg_term = torch.mm(normalized_context_values, normalized_context_values.t())
        identity = Variable(torch.eye(self.num_of_topics, self.num_of_topics))
        if torch.has_cudnn:
            identity = identity.cuda()
        return torch.norm(reg_term - identity, p=2.0)

class MemNet(nn.Module):
    def __init__(self, input_size):
        super(MemNet, self).__init__()
        self.query = nn.Parameter(torch.randn(input_size), requires_grad=True)

        self.linear_transform = torch.nn.Conv1d(input_size, input_size, kernel_size=5, padding=2)
        self.bn = torch.nn.BatchNorm1d(input_size)
        self.drop_out = nn.Dropout(.6)

        self.layer_norm = nn.LayerNorm(input_size)

        self.halt_prob = nn.Linear(input_size, 2)
        self.max_hop = 10

    def forward(self, x, mask):
        query = torch.cat([self.query.unsqueeze(0) for _ in range(x.size(0))], dim=0).contiguous()
        halted = torch.zeros(x.size(0)).byte().contiguous()
        halted = Variable(halted)
        if torch.has_cudnn:
            halted = halted.cuda()

        halting_hop = [self.max_hop] * x.size(0)
        n_hop = 0
        while not (halted.any() or n_hop == self.max_hop):
            energies = torch.bmm(x, query.unsqueeze(-1)).squeeze(-1)
            energies = energies.masked_fill(mask.squeeze(-1), -np.inf)
            probs = F.softmax(energies, dim=-1)

            x = F.leaky_relu(self.bn(self.linear_transform(x.transpose(1, 2))).transpose(1, 2))
            x = x.masked_fill(mask, .0)

            o = torch.bmm(x.transpose(1, 2), probs.unsqueeze(-1)).squeeze(-1)
            halt_prob = F.softmax(self.halt_prob(o + query), dim=-1)
            query = o + query * (1 - halted).unsqueeze(1).float()

            for b in range(len(halted)):
                halted[b] = halted[b] or (torch.argmax(halt_prob[b], dim=-1) == 1)
                if halting_hop[b] == self.max_hop and halted[b]:
                    halting_hop[b] = n_hop
            n_hop += 1
        # if not self.training:
        #     print(max_prob)
        #     print(min_prob)
        #     print(sum(halting_hop) / len(halting_hop))
        return query

class TopicAttention(nn.Module):
    def __init__(self, num_of_topics=11, hidden_size=150, input_size=300, classification_size=12, topic_hidden_size=20,
                 drop_out_prob=0.6, sentence_length=65, embeddings=None):
        super(TopicAttention, self).__init__()
        torch.manual_seed(0)
        np.random.seed(0)
        if torch.has_cudnn:
            torch.cuda.manual_seed(0)
        cudnn.benchmark = True
        random.seed(0)
        self.num_of_topics = num_of_topics
        self.sentence_length = sentence_length
        self.classification_size = classification_size
        self.topic_hidden_size = topic_hidden_size
        self.hidden_size = hidden_size
        self.input_size = input_size

        self.vocab_size = len(embeddings)

        self.embeddings = nn.Embedding(num_embeddings=self.vocab_size, embedding_dim=300, padding_idx=self.vocab_size - 1)
        self.embeddings.weight = nn.Parameter(torch.from_numpy(embeddings).float(), requires_grad=False)

        self.dim_reduction = nn.Linear(300, 128)
        self.pos_emb = PositionalEncoder(300, 66, 128, self.vocab_size - 1)

        self.layer_norm = torch.nn.LayerNorm(128)

        self.mem_net = nn.ModuleList([
            MemNet(128) for _ in range(num_of_topics)
        ])
        self.output_layer = torch.nn.Linear(num_of_topics * 128, classification_size)
        self.drop_out = nn.Dropout(p=drop_out_prob)

        self.topic_hidden = nn.ModuleList([
                nn.Linear(128, topic_hidden_size) for _ in range(num_of_topics)])
        self.attn_aspect_context = nn.Parameter(torch.randn(classification_size, topic_hidden_size))
        self.linear_aspect = nn.ModuleList([nn.Linear(topic_hidden_size * num_of_topics, 2 * topic_hidden_size)
                                            for _ in range(self.classification_size)])
        self.encoder = torch.nn.GRU(input_size=input_size, hidden_size=64, batch_first=True, bidirectional=True)

    def forward(self, x_indices, lengths, validate=False):
        mask = Variable(torch.ones(x_indices.size(0), x_indices.size(1), 1))
        if torch.has_cudnn:
            mask = mask.cuda()
        max_len = lengths[torch.argmax(lengths)].item()
        for i, l in enumerate(lengths):
            if l < max_len:
                mask[i, l:] = 0
        mask = mask.byte()
        # attention_mask = 1 - torch.bmm(mask, mask.transpose(1, 2))
        mask = 1 - mask

        x = self.embeddings(x_indices) + self.pos_emb(x_indices)
        x = self.dim_reduction(x)
        x = x.masked_fill(mask, .0)
        x = self.drop_out(x)

        # x = torch.nn.utils.rnn.pack_padded_sequence(x, lengths, batch_first=True)
        # x, _ = self.encoder(x)
        # x, lengths = torch.nn.utils.rnn.pad_packed_sequence(x, batch_first=True)
        # x = self.layer_norm(x)

        x = torch.cat([self.mem_net[i](x, mask) for i in range(self.num_of_topics)], dim=-1)
        x = torch.sigmoid(self.output_layer(x))
        return x, self.regularization_loss()

    def regularization_loss(self):
        attn_contexts = torch.cat([self.mem_net[i].query.unsqueeze(0) for i in range(self.num_of_topics)], dim=0)
        normalized_context_values = F.normalize(attn_contexts, dim=1)
        reg_term = torch.mm(normalized_context_values, normalized_context_values.t())
        identity = Variable(torch.eye(self.num_of_topics, self.num_of_topics))
        if torch.has_cudnn:
            identity = identity.cuda()
        return torch.norm(reg_term - identity, p=2.0)

    def return_weights(self, x):
        weights = []
        existance = []
        x, hidden = self.rnn(x)
        for i in range(self.num_of_topics):
            context = torch.stack([self.attn_context[i]] * x.size(0))
            energy = torch.bmm(x, context.unsqueeze(-1))
            probs = F.softmax(energy, dim=1)
            weights.append(probs)
            out = torch.bmm(x.transpose(1, 2), probs).squeeze(-1)
            out = self.topic_hidden[i](out)
            out = squash(out)
            existance.append(torch.norm(out[0]))
        return weights, existance

class TopicAttentionWithoutSquash(nn.Module):
    def __init__(self, num_of_topics=6, hidden_size=150, input_size=300, classification_size=5, topic_hidden_size=20,
                 drop_out_prob=0.6, sentence_length=65):
        super(TopicAttentionWithoutSquash, self).__init__()
        torch.manual_seed(0)
        np.random.seed(0)
        if torch.has_cudnn:
            torch.cuda.manual_seed(0)
        cudnn.benchmark = True
        random.seed(0)
        self.num_of_topics = num_of_topics
        self.sentence_length = sentence_length
        self.classification_size = classification_size
        self.topic_hidden_size = topic_hidden_size
        self.drop_out = nn.Dropout(p=drop_out_prob)

        self.rnn = nn.GRU(input_size=input_size,
                          hidden_size=hidden_size,
                          num_layers=1, batch_first=True, bidirectional=True)
        self.attn_context = nn.Parameter(torch.randn(num_of_topics, hidden_size * 2))
        self.topic_hidden = nn.ModuleList([
                nn.Linear(hidden_size * 2, topic_hidden_size) for _ in range(num_of_topics)])
        self.output_layer = nn.Linear(topic_hidden_size * num_of_topics, classification_size, bias=True)

        self.attn_aspect_context = nn.Parameter(torch.randn(classification_size, topic_hidden_size))
        self.linear_aspect = nn.ModuleList([nn.Linear(topic_hidden_size * num_of_topics, 2 * topic_hidden_size)
                                            for _ in range(self.classification_size)])

        self.reconstruction_layer = nn.Sequential(
            nn.Linear(topic_hidden_size * num_of_topics, 2 ** 10),
            nn.ReLU(),
            nn.Linear(2 ** 10, 2 ** 12),
            nn.ReLU(),
            nn.Linear(2 ** 12, sentence_length * input_size),
            nn.Sigmoid()
        )
        self.reconstruction_loss_function = nn.MSELoss()

    def forward(self, x, validate=False):
        x = self.drop_out(x)
        self.rnn.flatten_parameters()
        x, hidden = self.rnn(x)
        x = self.drop_out(x)
        x_ = None
        for i in range(self.num_of_topics):
            context = torch.stack([self.attn_context[i]] * x.size(0))
            energy = torch.bmm(x, context.unsqueeze(-1))
            probs = F.softmax(energy, dim=1)
            out = torch.bmm(x.transpose(1, 2), probs).squeeze(-1)
            out = self.topic_hidden[i](out)
            out = F.relu(out)
            if x_ is None:
                x_ = out
            else:
                x_ = torch.cat((x_, out), dim=1)
        x = self.output_layer(x_)
        x = torch.sigmoid(x)
        return x, self.regularization_loss()

    def regularization_loss(self):
        normalized_context_values = F.normalize(self.attn_context, dim=1)
        reg_term = torch.mm(normalized_context_values, normalized_context_values.t())
        identity = Variable(torch.eye(self.num_of_topics, self.num_of_topics))
        if torch.has_cudnn:
            identity = identity.cuda()
        return torch.norm(reg_term - identity)

    def reconstruction_loss(self, hidden_topic, init_input):
        recon_input = self.reconstruction_layer(hidden_topic)
        init_input_flat = None
        for i in range(init_input.size(1)):
            if init_input_flat is None:
                init_input_flat = init_input[:, i]
            else:
                init_input_flat = torch.cat((init_input_flat, init_input[:, i]), dim=1)
        return self.reconstruction_loss_function(recon_input, init_input_flat)

    def return_weights(self, x):
        weights = []
        existance = []
        x, hidden = self.rnn(x)
        for i in range(self.num_of_topics):
            context = torch.stack([self.attn_context[i]] * x.size(0))
            energy = torch.bmm(x, context.unsqueeze(-1))
            probs = F.softmax(energy, dim=1)
            weights.append(probs)
            out = torch.bmm(x.transpose(1, 2), probs).squeeze(-1)
            out = self.topic_hidden[i](out)
            out = squash(out)
            existance.append(torch.norm(out[0]))
        return weights, existance

class VanillaAttention(nn.Module):
    def __init__(self, hidden_size=150, input_size=300, classification_size=5, drop_out_prob=0.6, embeddings=None):
        super(VanillaAttention, self).__init__()
        torch.manual_seed(0)
        np.random.seed(0)
        if torch.has_cudnn:
            torch.cuda.manual_seed(0)
        cudnn.benchmark = True
        random.seed(0)

        self.vocab_size = len(embeddings)

        self.embeddings = nn.Embedding(num_embeddings=self.vocab_size, embedding_dim=300, padding_idx=self.vocab_size - 1)
        self.embeddings = self.embeddings.from_pretrained(torch.from_numpy(embeddings).float(), freeze=True)

        self.drop_out = nn.Dropout(drop_out_prob)
        self.rnn = nn.GRU(input_size=input_size,
                          hidden_size=hidden_size,
                          num_layers=1, batch_first=True, bidirectional=True)
        self.attn_context = nn.Linear(2 * hidden_size, 1)
        self.hidden_layer = nn.Linear(2 * hidden_size, hidden_size)
        self.output_layer = nn.Linear(hidden_size, classification_size)

    def forward(self, x, validate=False):
        x = self.embeddings(x)
        x = self.drop_out(x)
        x, _ = self.rnn(x)
        x = self.drop_out(x)

        energy = self.attn_context(x)
        probs = F.softmax(energy, dim=1)
        x = torch.bmm(x.transpose(1, 2), probs).squeeze(-1)
        x = self.hidden_layer(x)
        x = F.relu(x)
        x = self.output_layer(x)
        if not self.training:
            x = torch.sigmoid(x)
        return x

class Net:
    def __init__(self, epochs, train_loader, test_loader, validation_loader, learning_rate, model_type,
                 mode, min_delta, patience, num_of_topics=6, hidden_size=150, input_size=300,
                 topic_hidden_size=20, drop_out_prob=0.6, embeddings=None):
        if model_type == 'topic-attention':
            self.model = TopicAttention(num_of_topics=num_of_topics, hidden_size=hidden_size, input_size=input_size,
                                        classification_size=train_loader.get_num_of_classes(),
                                        topic_hidden_size=topic_hidden_size, drop_out_prob=drop_out_prob,
                                        sentence_length=train_loader.get_sentence_length(),
                                        embeddings=embeddings)
        elif model_type == 'vanilla-attention':
            self.model = VanillaAttention(hidden_size=hidden_size, input_size=input_size,
                                          classification_size=train_loader.get_num_of_classes(),
                                          drop_out_prob=drop_out_prob, embeddings=embeddings)
        elif model_type == 'topic-attention-without-squash':
            self.model = TopicAttentionWithoutSquash(num_of_topics=num_of_topics, hidden_size=hidden_size, input_size=input_size,
                                                     classification_size=train_loader.get_num_of_classes(),
                                                     topic_hidden_size=topic_hidden_size, drop_out_prob=drop_out_prob,
                                                     sentence_length=train_loader.get_sentence_length(), embeddings=embeddings)
        if torch.has_cudnn:
            self.model.cuda()

        self.model_type = model_type
        self.num_epochs = epochs
        # self.LossFunction = topkCE(k=k)
        # self.loss_function = nn.CrossEntropyLoss()
        # self.loss_function = nn.MSELoss()
        # self.validate_loss = nn.MSELoss()
        self.loss_function = nn.BCELoss()
        self.validate_loss = nn.BCELoss()

        self.optimizer = torch.optim.Adam(self.model.parameters(), lr=learning_rate, weight_decay=1e-4)
        # self.optimizer = torch.optim.Adadelta(self.model.parameters(), weight_decay=1e-5)
        self.early_stopping = EarlyStopping(mode, min_delta, patience)
        # lr_decay = lambda epoch: 1 / (1 + epoch * 0.2)
        # self.optimizer = torch.optim.SGD(self.model.parameters(), lr=learning_rate, momentum=0.9)
        # self.scheduler = torch.optim.lr_scheduler.LambdaLR(self.optimizer, lr_lambda=lr_decay)
        self.train_loader = train_loader
        self.test_loader = test_loader
        self.validation_loader = validation_loader

        self.validation_data_loader = dataloader.DataLoader(dataset=self.validation_loader,
                                                            batch_size=128,
                                                            shuffle=False,
                                                            num_workers=0,
                                                            collate_fn=collate_fn)
        self.train_data_loader = dataloader.DataLoader(dataset=self.train_loader,
                                                       batch_size=128,
                                                       shuffle=True,
                                                       num_workers=0,
                                                       collate_fn=collate_fn)
        self.test_data_loader = dataloader.DataLoader(dataset=self.test_loader,
                                                      batch_size=128,
                                                      shuffle=False,
                                                      num_workers=0,
                                                      collate_fn=collate_fn)

    def to_var(self, x):
        # x = torch.from_numpy(x).float()
        try:
            x = x.float()
        except AttributeError:
            x = torch.from_numpy(x).float()
        if torch.has_cudnn:
            x = Variable(x).cuda()
        else:
            x = Variable(x)
        return x

    def train(self, batch_size, validate=False):
        validate_loss = []
        train_loss = []

        for epoch in tqdm(range(self.num_epochs)):
            self.model.train()
            total_loss = []
            reg_total_loss = []
            for i, (data, labels, lengths) in enumerate(self.train_data_loader):
                reg_loss = 0.0
                data = self.to_var(data).long()
                labels = self.to_var(labels)
                self.optimizer.zero_grad()
                if self.model_type == 'topic-attention' or self.model_type == 'topic-attention-without-squash':
                    outputs, reg_loss = self.model(data, lengths)
                elif self.model_type == 'vanilla-attention':
                    outputs = self.model(data)
                # print(outputs)
                label_loss = self.loss_function(outputs, labels.squeeze(1))
                if self.model_type == 'topic-attention' or self.model_type == 'topic-attention-without-squash':
                    loss = label_loss + reg_loss
                elif self.model_type == 'vanilla-attention':
                    loss = label_loss

                loss.backward()
                # torch.nn.utils.clip_grad_norm_(self.model.parameters(), 1.0)
                self.optimizer.step()
                total_loss.append(float(label_loss))
                reg_total_loss.append(float(reg_loss))

            # print('Epoch [%d/%d], Avr Loss: %.4f, Avr Reg Loss: %.4f'
            #       % (epoch + 1, self.num_epochs, sum(total_loss) / len(total_loss), sum(reg_total_loss) / len(reg_total_loss)))

            if validate is True:
                valid_loss = self.validate()
                validate_loss.append(valid_loss)
                train_loss.append(float(sum(total_loss) / len(total_loss)))
            else:
                # valid_f1 = self.eval_validation_data()[0]
                valid_loss = self.validate()
                result = self.test()
                print('F1: %.4f, precision: %.4f, recall: %.4f' % (result[2], result[0], result[1]))
                print('train loss: ' + str(float(sum(total_loss) / len(total_loss))))
                print('validation loss: ' + str(valid_loss))
                # print(valid_loss)
                train_loss.append(float(sum(total_loss) / len(total_loss)))
                if self.early_stopping.step(valid_loss, deepcopy(self.model)) is True:
                    print('Early stopping at ' + str(epoch))
                    self.model = self.early_stopping.best_model
                    result = self.test()
                    print('F1: ' + str(result[0]))
                    print('P: ' + str(result[1]))
                    print('R: ' + str(result[2]))
                    # break
        if validate is True:
            return train_loss, validate_loss
        else:
            torch.save(self.model.state_dict(), './' + self.model_type)
            return result

    def test(self):
        self.model.eval()
        pred_labels = []
        true_labels = []
        for data, labels, lengths in self.test_data_loader:
            data = data.long()
            labels = labels.long()
            lengths = lengths.long()
            if torch.has_cudnn:
                data = data.cuda()
                labels = labels.cuda()
                lengths = lengths.cuda()
            # print(data)
            if self.model_type == 'topic-attention' or self.model_type == 'topic-attention-without-squash':
                outputs, _ = self.model(data, lengths)
            elif self.model_type == 'vanilla-attention':
                outputs = self.model(data)
            pred_labels += deepcopy(list(outputs.cpu().detach().numpy()))
            true_labels += deepcopy(list(labels.cpu().squeeze(1).detach().numpy()))
        pred_labels = list(pred_labels)
        # print(pred_labels)
        true_labels = list(true_labels)
        threshold = self.eval_validation_data()[1]
        print('Best threshold value: ' + str(threshold))
        return self.calculate_F1(pred_labels, true_labels, threshold)

    def eval_validation_data(self):
        self.model.eval()
        pred_labels = []
        true_labels = []
        best_result = [0.0, 0]
        for data, labels, lengths in self.validation_data_loader:
            data = data.long()
            labels = labels.long()
            lengths = lengths.long()
            if torch.has_cudnn:
                data = data.cuda()
                labels = labels.cuda()
                lengths = lengths.cuda()
            if self.model_type == 'topic-attention' or self.model_type == 'topic-attention-without-squash':
                outputs, _ = self.model(data, lengths)
            elif self.model_type == 'vanilla-attention':
                outputs = self.model(data)
            pred_labels += deepcopy(list(outputs.cpu().detach().numpy()))
            true_labels += deepcopy(list(labels.cpu().squeeze(1).detach().numpy()))
        for threshold in list(frange(0, 1, decimal.Decimal('0.01'))):
            precision, recall, f1 = self.calculate_F1(pred_labels, true_labels, threshold)
            if f1 > best_result[0]:
                best_result[0] = f1
                best_result[1] = threshold
        print('validation f1: ' + str(best_result[0]))
        return best_result

    def calculate_F1(self, pred_labels, true_labels, threshold):
        TP = 0
        FP = 0
        FN = 0
        for idx in range(len(pred_labels)):
            output = pred_labels[idx]
            label = true_labels[idx]
            for i in range(len(list(output))):
                if output[i] >= threshold:
                    if label[i] == 1:
                        TP += 1
                    else:
                        FP += 1
                else:
                    if label[i] == 1:
                        FN += 1
        try:
            precision = TP / (TP + FP)
            recall = TP / (TP + FN)
            f1 = 2 * (precision * recall) / (precision + recall)
        except ZeroDivisionError:
            precision = .0
            recall = .0
            f1 = .0
        # if len(pred_labels) == 642:
        #     print(str(TP) + ' ' + str(FP) + ' ' + str(FN))
        return precision, recall, f1

    def validate(self):
        self.model.eval()
        valid_loss = 0
        count = 0
        for data, labels, lengths in self.validation_data_loader:
            data = data.long()
            labels = labels.float()
            lengths = lengths.long()
            if torch.has_cudnn:
                data = data.cuda()
                labels = labels.cuda()
                lengths = lengths.cuda()
            if self.model_type == 'topic-attention' or self.model_type == 'topic-attention-without-squash':
                outputs, _ = self.model(data, lengths, validate=True)
            elif self.model_type == 'vanilla-attention':
                outputs = self.model(data)
            valid_loss += self.validate_loss(outputs, labels.squeeze(1)).item()
            count += 1
        valid_loss = float(valid_loss) / count
        return valid_loss

class EarlyStopping(object):
    def __init__(self, mode='min', min_delta=0, patience=10):
        self.mode = mode
        self.min_delta = min_delta
        self.patience = patience
        self.best = None
        self.num_bad_epochs = 0
        self.is_better = None
        self._init_is_better(mode, min_delta)

        if patience == 0:
            self.is_better = lambda a, b: True

    def step(self, metrics, model):
        if self.best is None:
            self.best = metrics
            self.best_model = model
            return False

        if np.isnan(metrics):
            return True

        if self.is_better(metrics, self.best):
            self.num_bad_epochs = 0
            self.best = metrics
            self.best_model = model
        else:
            self.num_bad_epochs += 1

        if self.num_bad_epochs >= self.patience:
            return True

        return False

    def _init_is_better(self, mode, min_delta):
        if mode not in {'min', 'max'}:
            raise ValueError('mode ' + mode + ' is unknown!')
        if mode == 'min':
            self.is_better = lambda a, best: a < best - min_delta
        if mode == 'max':
            self.is_better = lambda a, best: a > best + min_delta

    def return_best_model(self):
        return self.best_model

def collate_fn(data):
    sentence_lengths = [data[i][2] + 1 for i in range(len(data))]
    sorted_indices = np.argsort(sentence_lengths)[::-1]

    max_sentence_length = max(sentence_lengths)

    sentence_rep = np.array([data[idx][0][:max_sentence_length] for idx in sorted_indices])
    # try:
    label = np.array([data[i][1] for i in sorted_indices])
    sentence_lengths = np.array([sentence_lengths[idx] for idx in sorted_indices])

    sentence_rep = np.stack(sentence_rep)
    label = np.stack(label)
    sentence_lengths = np.stack(np.array(sentence_lengths))

    return torch.from_numpy(sentence_rep), torch.from_numpy(label), torch.from_numpy(sentence_lengths)


import random

print('Ввидите кол-во клеток')
cells_count = int(input())
cells_list = [random.randint(-1, 6) for _ in range(cells_count)]
result = ''
for index in range(cells_count):
    value = cells_list[index]
    print(f'Экффективность {index}, клетки: {value}')
    if value < index:
        result += ' ' + str(value)

print('Неподходящие значения:', result)


from solver import *

prob = load_prob('case_opt1')

prob.plot3()
ax = prob.plot3()

lim = 0.05

ax.set_xlim3d(0, lim)
ax.set_ylim3d(0, lim)
ax.set_zlim3d(0, lim)

pl.show()


class Node:
    def __init__(self,Data):
        self.next= None
        self.Data=Data

class LinkedList:
    def __init__(self) :
        self.Head=None
    def CreateAtStart(self,newData):
        p=Node(newData)
        if self.Head==None:
            self.Head=p
        else:
            self.next=self.Head
            self.Head=p

    def CreateAtEnd(self,newData):
        p=Node(newData)
        if self.Head==None:
            self.Head=p
        else:
            temp=self.Head
            while temp.next:
                temp=temp.next
            temp.next=p

    def DeleteAtStart(self):
        if self.Head==None:
            print("Empty Linked List")
        else:
            p=self.Head
            self.Head=self.Head.next
            del p

    def CreateAtGivenLOcation(self,loc,newData):
        p=Node(newData)
        if self.Head==None and loc==1:
            self.Head=p
        else:
            i=1
            count=0
            q=self.Head
            while i<loc-1 and q !=None:
                q=q.next
                i=i+1
            if q==None:
                print("Invalid Location")
            else:
                p.next=q.next
                q.next=p

    def DeleteAtEnd(self):
        pass
    def DeleteAtGivenLocation(self):
        pass


# -*- coding: utf-8 -*-
"""
Created on Mon Apr 27 23:16:23 2020

@author: lizijian
"""

import numpy as np
from constants import *
from environment import MECsystem
from decision import Agent
from utils import plot_learning
import matplotlib.pyplot as plt
import logging

logging.basicConfig(filename='testing.log', level=logging.ERROR,
                    format='%(message)s')

UEnet = Agent(alpha=0.000025, beta=0.00025, input_dims = 8, tau=0.001, \
              env=None, batch_size=64, layer1_size=500, layer2_size=300,
              n_actions=1)
env = MECsystem(apply_num, UEnet)
MECSnet = Agent(alpha=alpha, beta=beta, input_dims=input_dims, tau=0.001,
                env=env, batch_size=batch_size, layer1_size=layer1_size,
                layer2_size=layer2_size, n_actions=n_actions)

np.seed(0)

MECSnet.load_modules()

done = False
reward_history = []
task_fail_count = 0
time_history = []
obs = env.reset()
print('started')
k = 0
while not done:
    k += 1
    act = MECSnet.choose_action(obs, with_noise=False)
    new_state, reward, done, info = env.step(act)
    reward_history.append(reward)
    obs = new_state
    if k % 100 == 0:
        print(k)
        print('already done {} %'.format(env.time/time_total*100))

reward_mean = np.mean(reward_history)
reward_var = np.var(reward_history)
delay_mean = np.mean(delay_history)
delay_var = np.var(delay_history)
energy_mean = np.mean(energy_history)
energy_var = np.var(energy_history)

logging.error("module:\nreward:{}(+-{})\ndelay:{}(+-{})\nenergy{}(+-{})\n " \
              .format(reward_mean, reward_var, delay_mean, delay_var,
                      energy_mean, energy_var, task_fail_count))