# Weighting Mesopotamia

# You are a government leader of Mesopotamia, and Mesopotamia is facing an
# influx of immigrants, many of whom die, and you want to determine what
# factors contribute to people dying so that immigrants can make an informed
# decision.
#
# The initial settlement is conducting the first 5 year experiment. At the
# end of the five years, they have their data telling them,the DNA sequence
# of each individual and whether they survived for five years or not. The
# data may look something like the following (training set). Note:
# Mesopotamians had different symbols to represent a DNA strand.
#
# After the first five years, a new group of settlers come in (test set).
# Their DNA sequence is written down, and the settlers predict whether they
# will live or die based on the DNA sequences and results of the previous
# experiment. (explained below)
#
# The results are looked at, after five years, and weights are given to
# each expert depending on whether it predicted correctly.
# This civilization has the following type of data: [“MYDNASTRANDS”, “live”]

###############################################################################

from math import *
from random import *

############ Training and Test Set ####################
def getDataFromFile(filename):
    with open(filename, 'r') as f:
        data = eval(f.read()) # NOTE: DO NOT USE eval unless you are absolutely
                              # sure that your file is uncorruptable!!!
    return data

############# KNN to get weights of branches ###########

# This distance formula is using the four letters of a string starting at
# the expert's index to calculate the distance between DNA strings.
# For example ABCD, TYUY we would have ord(A)-ord(T) + ord(B)-ord(Y) ...
def distance(dna1, dna2, expertIndex):
    newDna1 = dna1[expertIndex*4:(expertIndex+1)*4]
    newDna2 = dna2[expertIndex*4:(expertIndex+1)*4]
    total = 0
    for index in range(len(newDna1)):
        total += abs(ord(newDna1[index]) - ord(newDna2[index]))
    return total

# Returns the K closest neighbors to the current test instance
def getNeighbors(trainingSet, testInstance, k, expertIndex):
    # Gather the dna strand of the test instance, and initialize the list
    # of distances.
    distances = []
    testDNA = testInstance[0]

    # Here we want to calculate the distance between our current point, and
    # the test point. (Use the helper function)
    # Note, the distance function doesn't require x, y points but the
    # dna strands and expert index.

    ###################################################################
    # Fill in here
    ###################################################################

    # For our design we will be storing the distances in a 2D List so we
    # can easily sort them below

    distances.sort()
    neighbors = []
    # Here we would like to return only the k closest training instances.
    # So we will be appending the top 3 distances

    ###################################################################
    # Fill in here
    ###################################################################

    return neighbors

# Helper function for sorting the votes in a dictionary
def reverseSort(votes):
    return sorted(votes)[::-1]

# Sort the votes from the dictionary into a 2d list
def sortVotes(typeVotes):
    votes = []
    for key in typeVotes:
        votes.append([typeVotes[key], key])
    votes = reverseSort(votes)
    return votes[0]

# Returns True, if the majority of the closest neighbors live,
# false otherwise
def getLabel(neighbors):
    typeVotes = {}
    # Here we will iterate through neighbors and retrieve the true or false
    # type of each neighbor and add it to the dictionary type votes.

    ###################################################################
    # Fill in here
    ###################################################################

    typeVotes = sortVotes(typeVotes)
    return typeVotes[1]

def knnMain(trainingSet, testSet, experts):
    k = 3
    penalty = .5
    # Iterate through each instance in the testSet. For each testInstance, we
    # will also iterate through each expert. From there we will find the
    # neighbors, labels, and actual label based on that testInstance-expert
    # scenario, and downweight the current expert if the label came back
    # incorrect.

    ###################################################################
    # Fill in here
    ###################################################################

    return experts

# Gather test and training sets
trainingSet = getDataFromFile("data.txt")
testSet = []
for i in range(len(trainingSet)//2):
    testSet.append(trainingSet.pop())

#experts =      ["Speed", "Height", "Weight"]
expertWeights = [1,       1,        1]
knnMain(trainingSet, testSet, expertWeights)
print(expertWeights)

# Challenge 0: For the first step, complete the getNeighbors function which
# will iterate over the trainingSet call the distance function, and then
# add the distance and traingingInstance to a list similar to last time.
# Be careful, and make sure to send the correct information in the distance
# function. Then similarly to our original KNN, retrieve the top k instances
# from the created 2-D list of distances and points.

# Challenge 1: For the next step, we will be completing the getLabel function.
# We are trying to predict the label ‘live’ or ‘die’, so similarly to the
# previous KNN, we will iterate through the neighbors, and add the labels to
# our dictionary accordingly.

# Challenge 2: For the final step, you will want to follow a similar format
# to our previous iteration of KNN. Except in this case (a) you will want to
# iterate through each expert over each test instance and (b) after you have
# successfully found the label for the given expert:instance, you want to check
# if it is correct.
# If it is correct, then leave it, if it is not correct, then you will
# have to downweight your current expert.

# Make sure you are referencing the previous KNN assignment while you do this!

machine_learning_curriculum/3.2 K Nearest Neighbors 2/studentMain.py


# -*- coding: utf-8 -*-

import asyncio
import discord
from discord.ext import commands

from youtube_dl import YoutubeDL

from apiclient.discovery import build

import os
import shutil
import re
import random

import m10s_util as ut

"""
上のモジュールをインストールすること！

m10s_music.py
制作:mii-10#3110(Discord)
"""

ytdlopts = {
    'format': 'bestaudio/best',
    'outtmpl': 'musicfile/%(id)s',
    'restrictfilenames': True,
    # 'dump_single_json' :  True,
    # 'extract_flat' : True,
    'nocheckcertificate': True,
    'ignoreerrors': False,
    'logtostderr': False,
    'quiet': True,
    'no_warnings': True,
    'default_search': 'auto',
    'source_address': '0.0.0.0'  # ipv6 addresses cause issues sometimes
}

class music(commands.Cog):
    """music in discord.py"""

    def __init__(self, bot):
        self.bot = bot
        self.youtube = build('youtube', 'v3', developerKey=bot.GAPI_TOKEN)
        self.ytdl = YoutubeDL(ytdlopts)
        if not ("qu" in dir(bot) and "lp" in dir(bot) and "mp" in dir(bot)):
            self.bot.qu = {}
            self.bot.lp = {}
            self.bot.mp = {}

    async def gvinfo(self, url, dl=False):
        loop = self.bot.loop or asyncio.get_event_loop()
        dt = await loop.run_in_executor(None, lambda: self.ytdl.extract_info(url, download=dl))
        return dt

    async def gpdate(self, url, dl=True, utype="Youtube"):
        v = await self.gvinfo(url, dl)
        if utype == "Youtube":
            return {
                "type": "download" if dl else "stream",
                "video_id": v['id'],
                "video_url": v['webpage_url'],
                "video_title": v['title'],
                "video_thumbnail": v['thumbnail'],
                "video_up_name": v["uploader"],
                "video_up_url": v["uploader_url"],
                "video_source": "Youtube"
            }
        elif utype == "niconico":
            return {
                "type": "download" if dl else "stream",
                "video_id": v['id'],
                "video_url": v['webpage_url'],
                "video_title": v['title'],
                "video_thumbnail": v['thumbnail'],
                "video_up_name": v["uploader"],
                "video_up_url": "https://www.nicovideo.jp/user/"+v["uploader_id"],
                "video_source": "niconico"
            }
        elif utype == "soundcloud":
            return {
                "video_id": v['id'],
                "video_url": v['webpage_url'],
                "video_title": v['title'],
                "video_thumbnail": v['thumbnail'],
                "video_up_name": v["uploader"],
                "video_up_url": re.match(r"(https://soundcloud\.com/.+?/)", v['webpage_url']).group(0),
                "video_source": "SoundCloud"
            }

    @commands.command(name="join", aliases=["invc"])
    async def join_(self, ctx):
        if ctx.author.voice:
            if ctx.voice_client:
                await ctx.send(f"{ctx.voice_client.channel.name}に接続しました。")
            else:
                try:
                    await ctx.author.voice.channel.connect()
                except asyncio.TimeoutError:
                    await ctx.send("接続のタイムアウト！")
        else:
            await ctx.send("あなたがボイスチャンネルに接続していません！")

    @commands.command(name="stop", aliases=["leave"])
    async def stop_(self, ctx):
        if ctx.voice_client and ctx.author.voice:
            if ctx.voice_client.channel == ctx.author.voice.channel:
                try:
                    await self.bot.mp[str(ctx.guild.id)].delete()
                except:
                    await ctx.send("操作パネルを削除できませんでした。")
                self.bot.qu[str(ctx.guild.id)] = None
                self.bot.mp[str(ctx.guild.id)] = None
                self.bot.lp[str(ctx.guild.id)] = None
                await ctx.voice_client.disconnect()
                await ctx.send("切断しました。")

    @commands.command(name="pause")
    async def pause_(self, ctx):
        if ctx.voice_client and ctx.author.voice:
            ctx.voice_client.pause()
            await ctx.send("一時停止しました。ボイスチャットを出ても構いません。")
            await self.panel_update(ctx)

    @commands.command(name="play", aliases=["p"])
    async def play_(self, ctx, *, text: str=""):
        if not ctx.voice_client:
            await ctx.invoke(self.bot.get_command("join"))
            if not ctx.voice_client:
                return
        if ctx.voice_client.is_paused():
            await ctx.send("再生を再開しました。")
            ctx.voice_client.resume()
            await self.panel_update(ctx)
            return
        if text == "":
            try:
                m = await ut.wait_message_return(ctx, "検索するワード または 再生するURLを送信してください。", ctx.channel)
            except asyncio.TimeoutError:
                await ctx.send("タイムアウトしました。")
                return
            else:
                vurls = [m.content]
        async with ctx.typing():
            try:
                vurls = []
                vdl = True
                if (text.startswith("<http://") and text.endswith(">")) or (text.startswith("<https://") and text.endswith(">")):
                    vurls = [text[1:-1]]
                elif text.startswith("http://") or text.startswith("https://"):
                    vurls = [text]
                elif text.startswith("memo:"):
                    self.bot.cursor.execute(
                        "select * from users where id=?", (ctx.author.id,))
                    pf = self.bot.cursor.fetchone()
                    mn = text[5:]
                    if pf["memo"] is not None and pf["memo"].get(mn,None) is not None:
                        vurls = pf["memo"][mn].split("\n")
                    else:
                        await ctx.send("> 音楽再生\n　該当名称のメモが見つかりません。")
                        return
                else:
                    search_response = self.youtube.search().list(
                        part='snippet',
                        q=text,
                        type='video'
                    ).execute()
                    vid = search_response['items'][0]['id']['videoId']
                    if vid:
                        vurls = [f"https://www.youtube.com/watch?v={vid}"]
                    else:
                        return await ctx.send("動画が見つかりませんでした。")
                if vurls == []:
                    return
                for vurl in vurls:
                    vinfo = await self.gvinfo(vurl, False)
                    if vinfo.get("extractor", "").startswith("youtube"):
                        if vinfo.get("_type", None) == "playlist":
                            tks = []
                            for c in vinfo["entries"]:
                                tks.append(self.gpdate(
                                    f"https://www.youtube.com/watch?v={c['id']}", vdl))
                            iqlt = [i for i in await asyncio.gather(*tks) if i]
                            if self.bot.qu.get(str(ctx.guild.id), None):
                                await ctx.send(f"キューにプレイリスト内の動画{len(iqlt)}本を追加します。")
                                self.bot.qu[str(ctx.guild.id)] = self.bot.qu[str(
                                    ctx.guild.id)] + iqlt
                                await self.panel_update(ctx)
                            else:
                                await ctx.send(f"プレイリストより、{len(iqlt)}本の再生を開始します。")
                                self.bot.qu[str(ctx.guild.id)] = iqlt
                                await asyncio.sleep(0.3)
                                self.bot.loop.create_task(self.mplay(ctx))
                        else:
                            iqim = await self.gpdate(vurl, vdl)
                            if self.bot.qu.get(str(ctx.guild.id), None):
                                await ctx.send(f"`{iqim['video_title']}`をキューに追加します。")
                                self.bot.qu[str(ctx.guild.id)] = self.bot.qu[str(
                                    ctx.guild.id)] + [iqim]
                                await self.panel_update(ctx)
                            else:
                                await ctx.send(f"`{iqim['video_title']}`の再生を開始します。")
                                self.bot.qu[str(ctx.guild.id)] = [iqim]
                                await asyncio.sleep(0.3)
                                self.bot.loop.create_task(self.mplay(ctx))
                    elif vinfo.get("extractor", "") == "niconico":
                        iqim = await self.gpdate(vurl, vdl, "niconico")
                        if self.bot.qu.get(str(ctx.guild.id), None):
                            await ctx.send(f"`{iqim['video_title']}`をキューに追加します。")
                            self.bot.qu[str(ctx.guild.id)] = self.bot.qu[str(
                                ctx.guild.id)] + [iqim]
                            await self.panel_update(ctx)
                        else:
                            await ctx.send(f"`{iqim['video_title']}`の再生を開始します。")
                            self.bot.qu[str(ctx.guild.id)] = [iqim]
                            await asyncio.sleep(0.3)
                            self.bot.loop.create_task(self.mplay(ctx))
                    elif vinfo.get("extractor", "").startswith("soundcloud"):
                        if vinfo.get("_type", None) == "playlist":

                            tks = []
                            for c in vinfo["entries"]:
                                tks.append(self.gpdate(c["url"], vdl, "soundcloud"))
                            iqlt = [i for i in await asyncio.gather(*tks) if i]
                            if self.bot.qu.get(str(ctx.guild.id), None):
                                await ctx.send(f"キューにプレイリスト内の動画{len(iqlt)}本を追加します。")
                                self.bot.qu[str(ctx.guild.id)] = self.bot.qu[str(
                                    ctx.guild.id)] + iqlt
                                await self.panel_update(ctx)
                            else:
                                await ctx.send(f"プレイリストより、{len(iqlt)}本の再生を開始します。")
                                self.bot.qu[str(ctx.guild.id)] = iqlt
                                await asyncio.sleep(0.3)
                                self.bot.loop.create_task(self.mplay(ctx))

                        else:
                            iqim = await self.gpdate(vurl, vdl, "soundcloud")
                            if self.bot.qu.get(str(ctx.guild.id), None):
                                await ctx.send(f"`{iqim['video_title']}`をキューに追加します。")
                                self.bot.qu[str(ctx.guild.id)] = self.bot.qu[str(
                                    ctx.guild.id)] + [iqim]
                                await self.panel_update(ctx)
                            else:
                                await ctx.send(f"`{iqim['video_title']}`の再生を開始します。")
                                self.bot.qu[str(ctx.guild.id)] = [iqim]
                                await asyncio.sleep(0.3)
                                self.bot.loop.create_task(self.mplay(ctx))
                    else:
                        await ctx.send("now,the video can't play the bot")
            except:
                import traceback
                await ctx.send(f"> traceback\n```{traceback.format_exc(4)}```")

    async def mplay(self, ctx, vl=0.5):
        v = None
        if not self.bot.lp.get(str(ctx.guild.id), None):
            self.bot.lp[str(ctx.guild.id)] = False
        if not self.bot.mp.get(str(ctx.guild.id), None):
            ebd = discord.Embed(title="思惟奈ちゃん-ミュージック操作パネル", color=self.bot.ec)
            ebd.add_field(name="再生中の曲:", value="未読み込み")
            ebd.add_field(name="次の曲:", value="未読み込み")
            ebd.add_field(name="ループ:", value="未読み込み")
            ebd.add_field(name="ボリューム:", value="未読み込み")
            m = await ctx.send(embed=ebd)
            self.bot.mp[str(ctx.guild.id)] = m
            await m.add_reaction("▶")
            await m.add_reaction("⏸")
            await m.add_reaction("⏹")
            await m.add_reaction("⏭")
            await m.add_reaction("🔁")
            await m.add_reaction("🔀")
            await m.add_reaction("🔼")
            await m.add_reaction("🔽")
            await m.add_reaction("⬇")
            try:
                await m.pin()
            except:
                pass
        while self.bot.qu[str(ctx.guild.id)]:
            ctx.voice_client.play(discord.PCMVolumeTransformer(discord.FFmpegPCMAudio(
                f'musicfile/{self.bot.qu[str(ctx.guild.id)][0]["video_id"]}'), volume=v or vl))
            await self.panel_update(ctx)
            try:
                while ctx.voice_client.is_playing() or ctx.voice_client.is_paused():
                    await asyncio.sleep(1)
                    v = ctx.voice_client.source.volume
            except AttributeError:
                pass
            if self.bot.lp[str(ctx.guild.id)]:
                self.bot.qu[str(ctx.guild.id)].append(self.bot.qu[str(ctx.guild.id)][0])
            self.bot.qu[str(ctx.guild.id)].pop(0)
        await ctx.invoke(self.bot.get_command("stop"))

    @commands.command()
    async def skip(self, ctx):
        if ctx.author.voice and ctx.voice_client.is_playing():
            lp = self.bot.lp[str(ctx.guild.id)]
            self.bot.lp[str(ctx.guild.id)] = False
            ctx.voice_client.stop()
            self.bot.lp[str(ctx.guild.id)] = lp
            await ctx.send("曲をスキップしました。")

    @commands.command(aliases=["vol"])
    async def chvol(self, ctx, vol: float):
        if ctx.author.voice and ctx.voice_client.is_playing():
            ctx.voice_client.source.volume = vol/100.0
            await ctx.send("ボリュームを調節しました。")
            await self.panel_update(ctx)

    @commands.command(aliases=["np"])
    async def playingmusic(self, ctx):
        if ctx.voice_client is None:
            return await ctx.send("ボイスチャンネルに参加していません。")
        if ctx.voice_client.is_playing():
            e = discord.Embed(
                title="再生中の曲", description=f'[{self.bot.qu[str(ctx.guild.id)][0]["video_title"]}]({self.bot.qu[str(ctx.guild.id)][0]["video_url"]})\nアップロードチャンネル:[{self.bot.qu[str(ctx.guild.id)][0]["video_up_name"]}]({self.bot.qu[str(ctx.guild.id)][0]["video_up_url"]})\nソース:{self.bot.qu[str(ctx.guild.id)][0]["video_source"]}')
            e.set_thumbnail(
                url=self.bot.qu[str(ctx.guild.id)][0]["video_thumbnail"])
            await ctx.send(embed=e)
        else:
            await ctx.send("再生中の曲はありません。")

    @commands.command(aliases=["plist", "queue"])
    async def view_q(self, ctx, pg=1):
        if ctx.voice_client is None:
            return await ctx.send("ボイスチャンネルに参加していません。")
        if ctx.voice_client.is_playing():
            page = pg-1
            pls = [self.bot.qu[str(ctx.guild.id)][5*i:5*(i+1)]
                   for i in range(int(len(self.bot.qu[str(ctx.guild.id)])/5)+1)]
            e = discord.Embed(
                title="キューの中身", description=f"全{len(self.bot.qu[str(ctx.guild.id)])}曲")
            for i in pls[page]:
                e.add_field(
                    name=i["video_title"], value=f'[動画]({i["video_url"]})/[アップロードチャンネル]({i["video_up_url"]})\nソース:{i["video_source"]}')
            e.set_footer(text=f"page:{page+1}/{len(pls)}")
            msg = await ctx.send(embed=e)
            await msg.add_reaction(self.bot.get_emoji(653161518195671041))  # ←
            await msg.add_reaction(self.bot.get_emoji(653161518170505216))  # →
            while True:
                try:
                    r, u = await self.bot.wait_for("reaction_add", check=lambda r, u: r.message.id == msg.id and u.id == ctx.message.author.id, timeout=30)
                except:
                    break
                try:
                    await msg.remove_reaction(r, u)
                except:
                    pass
                if str(r) == str(self.bot.get_emoji(653161518170505216)):  # →
                    if page == len(pls)-1:
                        page = 0
                    else:
                        page += 1
                elif str(r) == str(self.bot.get_emoji(653161518195671041)):  # ←
                    if page == 0:
                        page = len(pls)-1
                    else:
                        page -= 1
                e = discord.Embed(
                    title="キューの中身", description=f"全{len(self.bot.qu[str(ctx.guild.id)])}曲")
                for i in pls[page]:
                    e.add_field(
                        name=i["video_title"], value=f'[動画]({i["video_url"]})/[アップロードチャンネル]({i["video_up_url"]})\nソース:{i["video_source"]}')
                e.set_footer(text=f"page:{page+1}/{len(pls)}")
                await msg.edit(embed=e)
        else:
            await ctx.send("現在キューには何もありません。")

    @commands.command(aliases=["loop", "repeat"])
    async def loop_q(self, ctx, torf: bool=None):
        if ctx.author.voice:
            if torf is None:
                await ctx.send(f"今のキューのループ状態:{self.bot.lp[str(ctx.guild.id)]}")
            else:
                self.bot.lp[str(ctx.guild.id)] = torf
                await ctx.send(f"きりかえました。\n今のキューのループ状態:{self.bot.lp[str(ctx.guild.id)]}")
                await self.panel_update(ctx)

    @commands.command()
    async def pupdate(self, ctx):
        await self.panel_update(ctx)

    async def panel_update(self, ctx):
        ebd = discord.Embed(title="思惟奈ちゃん-ミュージック操作パネル",
                            description=f"キューの曲数:{len(self.bot.qu[str(ctx.guild.id)])}曲\nリアクションで操作でき、そのたびに操作パネルが更新されます。\n▶:(一時停止中)再生の再開,⏸:(再生中)一時停止,⏹:ストップ,⏭:スキップ,🔁:ループ切替,🔀次以降の曲のシャッフル,🔼:ボリュームを上げる,🔽:ボリュームを下げる,⬇:パネルを下に持ってくる", color=self.bot.ec)
        if ctx.voice_client.is_paused():
            ebd.add_field(name="現在一時停止中",
                          value="再開には`s-play`か▶リアクション", inline=False)
        ebd.add_field(
            name="再生中の曲:", value=f"[{self.bot.qu[str(ctx.guild.id)][0]['video_title']}]({self.bot.qu[str(ctx.guild.id)][0]['video_url']})(from {self.bot.qu[str(ctx.guild.id)][0]['video_source']})")
        if len(self.bot.qu[str(ctx.guild.id)]) > 1:
            ebd.add_field(
                name="次の曲:", value=f"[{self.bot.qu[str(ctx.guild.id)][1]['video_title']}]({self.bot.qu[str(ctx.guild.id)][0]['video_url']})(from {self.bot.qu[str(ctx.guild.id)][1]['video_source']})")
        elif self.bot.lp[str(ctx.guild.id)]:
            ebd.add_field(
                name="次の曲:", value=f"[{self.bot.qu[str(ctx.guild.id)][0]['video_title']}]({self.bot.qu[str(ctx.guild.id)][0]['video_url']})(from {self.bot.qu[str(ctx.guild.id)][0]['video_source']})(スキップでキューから削除され、再生が止まります。)")
        else:
            ebd.add_field(name="次の曲:", value=f"再生終了")
        ebd.add_field(name="ループ:", value=self.bot.lp[str(ctx.guild.id)])
        try:
            ebd.add_field(name="ボリューム:", value=int(
                ctx.voice_client.source.volume*100))
        except:
            ebd.add_field(name="ボリューム:", value="現在アクセス不可")
        ebd.set_thumbnail(
            url=self.bot.qu[str(ctx.guild.id)][0]["video_thumbnail"])
        try:
            await self.bot.mp[str(ctx.guild.id)].edit(embed=ebd)
        except AttributeError:
            # パネルメッセージが、まだない状態の可能性があるため
            pass

    @commands.command(name="shuffle")
    async def shuffle_(self, ctx):
        if self.bot.qu.get(str(ctx.guild.id), None) is not None and len(self.bot.qu[str(ctx.guild.id)]) > 2:
            tmplist = self.bot.qu[str(ctx.guild.id)][1:]
            random.shuffle(tmplist)
            self.bot.qu[str(ctx.guild.id)] = [self.bot.qu[str(ctx.guild.id)][0]] + tmplist
            await ctx.send("> シャッフル\n　シャッフルしました。再生パネルや`s-view_q`でご確認ください。")
        else:
            await ctx.send("> シャッフルエラー\n　シャッフルに必要要件を満たしていません。(VCで音楽再生中で、3曲以上キューに入っている)")

    @commands.Cog.listener()
    async def on_raw_reaction_add(self, pr):
        if self.bot.mp.get(str(pr.member.guild.id), None) is None:
            return
        if pr.user_id != pr.member.guild.me.id and self.bot.mp[str(pr.guild_id)].id == pr.message_id:
            ch = self.bot.get_channel(pr.channel_id)
            msg = await ch.fetch_message(pr.message_id)
            try:
                await msg.remove_reaction(pr.emoji, pr.member)
            except:
                pass
            msg.author = pr.member
            ctx = await self.bot.get_context(msg)
            r = pr
            u = pr.member
            if str(r.emoji) == "▶":
                await ctx.invoke(self.bot.get_command("play"))
            elif str(r.emoji) == "⏸":
                await ctx.invoke(self.bot.get_command("pause"))
            elif str(r.emoji) == "⏹":
                await ctx.invoke(self.bot.get_command("stop"))
            elif str(r.emoji) == "⏭":
                await ctx.invoke(self.bot.get_command("skip"))
            elif str(r.emoji) == "🔁":
                if self.bot.lp[str(u.guild.id)]:
                    await ctx.invoke(self.bot.get_command("loop"), False)
                else:
                    await ctx.invoke(self.bot.get_command("loop"), True)
            elif str(r.emoji) == "🔀":
                await ctx.invoke(self.bot.get_command("shuffle"))
            elif str(r.emoji) == "🔼":
                await ctx.invoke(self.bot.get_command("chvol"), int(ctx.voice_client.source.volume*100+10))
            elif str(r.emoji) == "🔽":
                await ctx.invoke(self.bot.get_command("chvol"), int(ctx.voice_client.source.volume*100-10))
            elif str(r.emoji) == "⬇":
                op = self.bot.mp[str(u.guild.id)]
                self.bot.mp[str(u.guild.id)] = await msg.channel.send(embed=self.bot.mp[str(u.guild.id)].embeds[0])
                await op.delete()
                m = self.bot.mp[str(u.guild.id)]
                await m.add_reaction("▶")
                await m.add_reaction("⏸")
                await m.add_reaction("⏹")
                await m.add_reaction("⏭")
                await m.add_reaction("🔁")
                await m.add_reaction("🔀")
                await m.add_reaction("🔼")
                await m.add_reaction("🔽")
                await m.add_reaction("⬇")
                try:
                    await m.pin()
                except:
                    pass

    @commands.Cog.listener()
    async def on_voice_state_update(self, member, before, after):
        try:
            if not member.guild.voice_client.is_paused() and [i for i in member.guild.me.voice.channel.members if not i.bot] == []:
                try:
                    await self.bot.mp[str(member.guild.id)].delete()
                except:
                    await self.bot.mp[str(member.guild.id)].channel.send("操作パネルを削除できませんでした。")
                self.bot.qu[str(member.guild.id)] = None
                self.bot.mp[str(member.guild.id)] = None
                self.bot.lp[str(member.guild.id)] = None
                await member.guild.voice_client.disconnect()
        except:
            pass
        try:
            if self.bot.voice_clients == []:
                shutil.rmtree("musicfile/")
                os.makedirs('musicfile/', exist_ok=True)
        except:
            pass

def setup(bot):
    bot.add_cog(music(bot))


from stack import Stack

import requests
import random
from json import dumps, loads
import time

maze_map = loads(open('map', 'r').read())

visited = set()

tmap = {}

BASE_URL = "https://lambda-treasure-hunt.herokuapp.com/api"
headers = {"Authorization": "Token ce973da1f80b171cc2de6e4310f51fd2600d2614"}

r = requests.get(url=BASE_URL+"/adv/init", headers=headers)
data = r.json()
time.sleep(data['cooldown'] + 1)

def search_map(room):
    room_id = str(room['room_id'])
    if room_id not in visited:
        tmap[room_id] = room
        print(room)
        visited.add(room_id)
        print(visited)
        room_map = maze_map[room_id]
        for r in room['exits']:
            print(room_map)
            if room_map[r] not in visited:
                print(True)
                body = {"direction": r, "next_room_id": room_map[r]}
                r = requests.post(url=BASE_URL+"/adv/move",
                                  json=body, headers=headers)
                next_room = r.json()
                time.sleep(next_room['cooldown'] + 1)

                for item in next_room['items']:

                    if 'treasure' in item:
                        body = { "name": item}
                        r = requests.post(url=BASE_URL+"/adv/take",
                                  json=body, headers=headers)
                        print('took', item)
                        print(r.json())
                        time.sleep(r.json['cooldown'])

                search_map( next_room)

                with open('search', 'w') as f:
                    f.truncate()
                    f.write(dumps(tmap))
                    f.close()

search_map(data)


from urllib import request, error,parse
from http import cookiejar
import ssl

#创建cookiejar的实例

cookie = cookiejar.CookieJar()
#生成cookie的管理器
cookie_handle = request.HTTPCookieProcessor(cookie)
#创建http请求的管理器
http_handle = request.HTTPHandler()
#创建HTTPS管理器
context = ssl._create_unverified_context()
https_handle = request.HTTPSHandler(context=context)

#创建请求管理器
opener = request.build_opener(http_handle, https_handle, cookie_handle)

def login():

    '''
    负责初次登录
    需要用户名密码，用来获取cookie凭证
    :return:
    '''
    #此URL需要从登录的form的action属性中提取
    # url = "https://weibo.com/login.php?url=https%3A%2F%2Fweibo.com%2Fu%2F3811132737%2Fhome%3Fwvr%3D5"
    url = "https://login.sina.com.cn/sso/login.php?client=ssologin.js(v1.4.19)"
    #微博要验证。。。

    data = {
        "username":"1056838379@qq.com",
        "password":""
    }

    context = ssl._create_unverified_context()

    #数据进行编码
    data = parse.urlencode(data)
    #创建一个请求
    req = request.Request(url, data=data.encode())
    #使用opener发起请求

    rsp = opener.open(req)
    # rsp = request.urlopen(req, context=context)

def getHomePage():

    # url = "https://weibo.com/login.php?url=https%3A%2F%2Fweibo.com%2Fu%2F3811132737%2Fhome%3Fwvr%3D5"
    url = "https://login.sina.com.cn/sso/login.php?client=ssologin.js(v1.4.19)"
    #如果已经执行了login函数，则opener自动已经包含了相应的cookie

    try:
        # context = ssl._create_unverified_context()
        rsp = opener.open(url)

        # req = request.Request(url)

        # rsp = request.urlopen(req, context=context)

        html = rsp.read().decode()
        with open("rsp3.html", "w") as f:
            f.write(html)

    except error.URLError as err:
        print(err)
    except error.HTTPError as err:
        print(err)

if __name__ == '__main__':
    login()
    getHomePage()


"""
Created on June 4th, 2019. This script compares surface ozone data with hourly resolution from Summit (SUM) station
in Greenland with the ethane/methane and acetylene/methane ratios developed and analyzed in 'ratioComparison.py.

The ozone data used here is courtesy of NOAA ESRL GMD. See the citation below.

McClure-Begley, A., Petropavlovskikh, I., Oltmans, S., (2014) NOAA Global Monitoring Surface Ozone Network.
Summit, 2012-2018. National Oceanic and Atmospheric Administration, Earth Systems Research Laboratory
Global Monitoring Division. Boulder, CO. 5/30/2019. http://dx.doi.org/10.7289/V57P8WBF
"""

import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
from sklearn.linear_model import LinearRegression

def ozonePlot():
    # Importing Data
    root = r'C:\Users\ARL\Desktop\J_Summit\analyses\HarmonicFit\textfiles'                      # root source
    ozone = pd.read_csv(root + r'\ozone.txt', encoding='utf8', delim_whitespace=True,
                        header=None)                                                            # import as csv
    ethane = pd.read_csv(root + r'\test1.txt', encoding='utf8', delim_whitespace=True)
    ace = pd.read_csv(root + r'\aceDefault.txt', encoding='utf8', delim_whitespace=True)
    ozone.columns = ['date', 'value', 'function', 'resid', 'resid_smooth']
    print('Data Imported...')

    # Graphing with Seaborn -- Setup Subplots
    sns.set()
    f, ax = plt.subplots(ncols=2, nrows=2)                                                      # seaborn setup
    sns.despine(f)                                                                              # remove right/top axes
    flatui = ["#9b59b6", "#3498db", "#95a5a6", "#e74c3c", "#34495e", "#2ecc71"]                 # color palette
    sns.set_palette(flatui)
    plt.subplots_adjust(left=None, bottom=None, right=None, top=None, wspace=None, hspace=0.5)  # adjust plot spacing

    # ----
    # Background Ozone Data and Fitted Harmonic Functions
    ax1 = sns.scatterplot(x='date', y='value', data=ozone, ax=ax[0, 0],
                          alpha=0.7, s=10, legend='brief', label='Ozone Data')
    ax2 = sns.lineplot(x='date', y='function', data=ozone, ax=ax[0, 0],
                       linewidth=2, label='Fitted Curve')

    ax1.set_title('Ozone Surface MR from Summit', fontsize=12)
    ax1.set_xlabel('Decimal Year')
    ax1.set_ylabel('Mixing Ratio [ppb]')
    ax1.legend()
    print('Plot 1 Created')

    # ----
    # Ozone Residuals with smoothed residual curve
    ax3 = sns.scatterplot(x='date', y='resid', data=ozone, ax=ax[0, 1], alpha=0.7, s=10,
                          label='Ozone Residuals', legend='brief')
    ax4 = sns.lineplot(x='date', y='resid_smooth', data=ozone, ax=ax[0, 1], linewidth=2,
                       label='Smoothed Residual Curve')
    ax3.set_title('Ozone Residuals w/ Smoothed Curve')
    ax3.set_xlabel('Decimal Year')
    ax3.set_ylabel('Mixing Ratio [ppb]')
    ax3.legend()
    print('Plot 2 Created')

    # ----
    # Create new dataframes for residual comparison
    titles_old = ['date', 'value', 'residuals']
    ace_titles_new = ['date_ace', 'value_ace', 'resid_ace']
    eth_titles_new =['date_eth', 'value_eth', 'resid_eth']

    # drop unused columns for speed
    ethDrop = ethane.drop(labels=(ethane.columns[np.logical_and((np.logical_and(ethane.columns != titles_old[0],
                                                                                ethane.columns != titles_old[1])),
                                                                ethane.columns != titles_old[2])]), axis=1)
    aceDrop = ace.drop(labels=(ace.columns[np.logical_and((np.logical_and(ace.columns != titles_old[0],
                                                                          ace.columns != titles_old[1])),
                                                          ace.columns != titles_old[2])]), axis=1)

    ethDrop.columns = eth_titles_new                                                            # rename columns
    aceDrop.columns = ace_titles_new

    # Trim the data and assign equivalent date conditions
    earlyVals = ~(ozone['date'] < ethDrop['date_eth'][0])                                       # early ozone vals
    ozone = ozone[earlyVals]                                                                    # remove those vals
    ozone = ozone.reset_index()                                                                 # reset index
    ozone = ozone.drop('index', axis=1)                                                         # remove unnece column

    ozoneEthane = pd.concat([ozone, ethDrop], sort=False, axis=1)                               # combine datasets
    ozoneData = pd.concat([ozoneEthane, aceDrop], sort=False, axis=1)
    ozoneData['date_ace'] = ozoneData['date_eth']

    """
    This for loop is brutally slow, is there any way to fix this? Perhaps with a seperate function but that would take
    the same length. I'm not sure if theres a built in function to reduce the size of an array by averging values into
    the size of another array
    """
    dataClean = []
    tolerence = 1 / 365                                                                         # ozone valus within day
    for index, value in ozoneData.iterrows():
        high = value.date_eth + tolerence                                                       # upper date lim
        low = value.date_eth - tolerence                                                        # lower date lim
        indices = (ozoneData['date'] <= high) & (ozoneData['date'] >= low)                      # indices between
        ozoneAv = np.nanmean(ozoneData['resid'][indices].values)                                # average of resids
        if ~np.isnan(value.resid_eth):
            dataClean.append([ozoneAv, value.resid_eth, value.resid_ace])                       # append clean mat

    ozoneFinal = pd.DataFrame(dataClean)                                                        # dataframe it
    ozoneFinal = ozoneFinal.dropna(axis=0, how='any')                                           # drop nans
    ozoneFinal.columns = ['OzoneResid', 'EthaneResid', 'AceResid']                              # rename columns

    # perform linear regression statistics
    x1 = np.array(ozoneFinal['EthaneResid']).reshape(-1, 1)
    y1 = np.array(ozoneFinal['OzoneResid'])
    model1 = LinearRegression().fit(x1, y1)                                                     # fit model
    rSquare1, intercept1, slope1 = model1.score(x1, y1), model1.intercept_, model1.coef_        # statistics

    x2 = np.array(ozoneFinal['AceResid']).reshape(-1, 1)
    y2 = np.array(ozoneFinal['OzoneResid'])
    model2 = LinearRegression().fit(x2, y2)                                                     # fit model
    rSquare2, intercept2, slope2 = model2.score(x2, y2), model2.intercept_, model2.coef_        # statistics

    # ----
    # Ethane Residuals v. Ozone Residuals
    ax5 = sns.regplot(x='EthaneResid', y='OzoneResid', data=ozoneFinal, ax=ax[1, 0],
                      line_kws={'label': 'rSquared: {:1.5f}\n Slope: {:1.5f}\n'.format(rSquare1, slope1[0])})
    ax5.set_title('Ethane/Ch4 Ratio Residuals v. Ozone Residuals')
    ax5.set_xlabel('Ethane/Ch4 Ratio Residuals [ppb]')
    ax5.set_ylabel('Ozone Residuals [ppb]')
    ax5.set(xlim=(-.0004, .0006))
    ax5.set(ylim=(-25, 25))
    ax5.legend()
    ax5.get_lines()[0].set_color('red')
    print('Plot 3 Completed')

    # ----
    # Acetylene Residuals v. Ozone Residuals
    ax6 = sns.regplot(x='AceResid', y='OzoneResid', data=ozoneFinal, ax=ax[1, 1],
                      line_kws={'label': 'rSquared: {:1.5f}\n Slope: {:1.5f}\n'.format(rSquare2, slope2[0])})
    ax6.set_title('Acetylene/Ch4 Ratio Residuals v. Ozone Residuals')
    ax6.set_xlabel('Acetylene/Ch4 Ratio Residuals [ppb]')
    ax6.set_ylabel('Ozone Residuals [ppb]')
    ax6.legend()
    ax6.get_lines()[0].set_color('red')
    print('Plot 4 Completed')

    plt.show()

    return ozone

if __name__ == '__main__':
    ozonePlot()


#! /usr/bin/env python
import gevent.monkey
gevent.monkey.patch_all()
import gevent
import websocket
import argparse
import msgpack
import json
import irc.bot
import irc.strings

class WSIRCRelayBot(irc.bot.SingleServerIRCBot):
    def __init__(self, channel, nickname, server, endpoint, port):
        irc.bot.SingleServerIRCBot.__init__(self, [(server, port)], nickname, nickname)
        self.ws = websocket.create_connection(endpoint)
        self.channel = channel
        gevent.spawn(self.listener)

    def listener(self):
        while True:
            print('Listening')
            msg = msgpack.loads(self.ws.recv())
            print(msg)
            if msg[0] == 'blob' and msg[1] == 'irc':
                self.connection.privmsg(self.channel, msg[2])

    def on_nicknameinuse(self, c, e):
        c.nick(c.get_nickname() + "_")

    def on_welcome(self, c, e):
        c.join(self.channel)

    def on_pubmsg(self, c, e):
        self.ws.send(msgpack.dumps(['blob', 'irc', json.dumps({'msg': e.arguments[0], 'channel': e.target, 'source': e.source})]), opcode=2)

def main():
    parser = argparse.ArgumentParser(description='WearScript IRC Bot')
    parser.add_argument('endpoint')
    parser.add_argument('server')
    parser.add_argument('channel')
    parser.add_argument('nickname')
    parser.add_argument('--port', default=6667, type=int)
    bot = WSIRCRelayBot(**vars(parser.parse_args()))
    bot.start()

if __name__ == "__main__":
    main()


"""
  Run upgrader
"""
portal = context.getPortalObject()
portal_alarms = portal.portal_alarms

after_method_id = 'Base_postCheckConsistencyResult'
def launchSenseAlarm(alarm_id, after_tag=[]):
  """ Get the alarm and use sense"""
  upgrader_alarm = getattr(portal_alarms, alarm_id, None)
  if upgrader_alarm is not None:
    # call solve method
    kw = {"tag": alarm_id,
      "after_method_id": after_method_id}
    if len(after_tag) > 0:
      kw["after_tag"] = after_tag
    method_id = upgrader_alarm.getActiveSenseMethodId()
    if method_id not in (None, ''):
      method = getattr(upgrader_alarm.activate(**kw), method_id)
      method()
    return [alarm_id,]
  return after_tag

previous_tag = launchSenseAlarm('upgrader_check_pre_upgrade')

previous_tag = launchSenseAlarm('upgrader_check_upgrader',
                                   after_tag=previous_tag)

previous_tag = launchSenseAlarm('upgrader_check_post_upgrade',
                 after_tag=previous_tag)

active_process = context.newActiveProcess()
context.activate(after_tag=previous_tag,
  after_method_id=after_method_id).Alarm_postFullUpgradeNeed(
  active_process=active_process.getRelativeUrl())


# _*_ coding:utf-8 _*_
import numpy as np
def getA(n):
    A=np.eye(n)
    A=A+np.mat(np.tril(-np.ones(n),-1))
    return A

def HilbertMatrix(n):
    H=np.mat(np.zeros((n,n)))
    for i in range(n):
        for j in range(n):
            H[i,j]=1/(i+j+3)
    return H

def cond(A):
    def RowSumNorm(A):
        A=np.abs(A)
        B=A.sum(axis=1)
        return np.max(B)
    return RowSumNorm(A)*RowSumNorm(A.I)

def getb(A):
    n=len(A)
    x=np.random.rand(n,1)
    return x,A*x

def PartialPivotingLU(A):
    n=len(A)
    p=[i for i in range(n)]  #记录排列阵
    for i in range(n-1):
        a=A[i,i]
        for j in range(i+1,n):  #选取列中最大元素
            if abs(a) < abs(A[j,i]):
                a=A[j,i]
                p[i]=j
        for k in range(n):
            a=A[p[i],k]
            A[p[i],k]=A[i,k]
            A[i,k]=a
        if A[i,i]!=0:
            for j in range(i+1,n):
                A[j,i]=A[j,i]/A[i,i]
            for j in range(i+1,n):
                for k in range(i+1,n):
                    A[j,k]=A[j,k]-A[j,i]*A[i,k]
        else:
            print('矩阵奇异')
            return 0
    return A,p

def PartialPivotingSolve(A,b):
    A=A.astype(np.float64)
    b=b.astype(np.float64)
    if np.shape(b)[0]==1:
        b=b.T
    A,p=PartialPivotingLU(A)
    n=len(A)
    for i in range(n-1):
        b[[i,p[i]]]=b[[p[i],i]]
    for i in range (1,n):
        for k in range(i):
            b[i]=b[i]-A[i,k]*b[k]
    for i in range(n):
        b[n-1-i]=b[n-1-i]/A[n-1-i,n-1-i]
        for k in range(i):
            b[n-1-i]=b[n-1-i]-A[n-1-i,n-1-k]*b[n-1-k]/A[n-1-i,n-1-i]
    return b

def CompletePivotingLU(A):
    n=len(A)
    p=[i for i in range(n)]  #记录排列阵
    q=[i for i in range(n)]
    for i in range(n-1):
        a=A[i,i]
        for j in range(i,n):  #选取子矩阵中最大元素
            for k in range(i,n):
                if abs(a) < abs(A[j,k]):
                    a=A[j,k]
                    p[i]=j
                    q[i]=k
        for j in range(n):
            a=A[p[i],j]
            A[p[i],j]=A[i,j]
            A[i,j]=a
        for j in range(n):
            a=A[j,q[i]]
            A[j,q[i]]=A[j,i]
            A[j,i]=a
        for j in range(i+1,n):
            A[j,i]=A[j,i]/A[i,i]
        for j in range(i+1,n):
            for k in range(i+1,n):
                A[j,k]=A[j,k]-A[j,i]*A[i,k]
    return A,p,q

def CompletePivotingSolve(A,b):
    A=A.astype(np.float64)
    b=b.astype(np.float64)
    if np.shape(b)[0]==1:
        b=b.T
    A,p,q=CompletePivotingLU(A)
    n=len(A)
    for i in range(n-1):
        b[[i,p[i]]]=b[[p[i],i]]
    for i in range (1,n):
        for k in range(i):
            b[i]=b[i]-A[i,k]*b[k]
    for i in range(n):
        b[n-1-i]=b[n-1-i]/A[n-1-i,n-1-i]
        for k in range(i):
            b[n-1-i]=b[n-1-i]-A[n-1-i,n-1-k]*b[n-1-k]/A[n-1-i,n-1-i]
    for i in range(1,n+1):
        b[[n-i,q[n-i]]]=b[[q[n-i],n-i]]
    return b
H=HilbertMatrix(12)
A=getA(60)
print(cond(H))
print(cond(A))
BackErrp=0
ForErrp=0
BackErrc=0
ForErrc=0
BackErrp2=0
ForErrp2=0
BackErrc2=0
ForErrc2=0
for i in range(10):
    y,c=getb(H)
    y_p=PartialPivotingSolve(H,c)
    y_c=CompletePivotingSolve(H,c)
    BackErrp+=np.linalg.norm(c-H*y_p,2)/10
    ForErrp+=np.linalg.norm(y-y_p,2)/10
    BackErrc+=np.linalg.norm(c-H*y_c,2)/10
    ForErrc+=np.linalg.norm(y-y_c,2)/10
    A=getA(60)
    x,b=getb(A)
    x_p=PartialPivotingSolve(A,b)
    x_c=CompletePivotingSolve(A,b)
    BackErrp2+=np.linalg.norm(b-A*x_p,2)/10
    ForErrp2+=np.linalg.norm(x-x_p,2)/10
    BackErrc2+=np.linalg.norm(b-A*x_c,2)/10
    ForErrc2+=np.linalg.norm(x-x_c,2)/10
print('BackErrp=',BackErrp)
print('ForErrp=',ForErrp)
print('BackErrc=',BackErrc)
print('ForErrc=',ForErrc)
print('BackErrp=',BackErrp2)
print('ForErrp=',ForErrp2)
print('BackErrc=',BackErrc2)
print('ForErrc=',ForErrc2)


TYPE_INT = 1 << 0
TYPE_BOOL = 1 << 1
TYPE_STRING = 1 << 2

options_hotspot = {

    # Garbage First (G1) Garbage Collection Options
    'UseG1GC': TYPE_BOOL,
    'MaxGCPauseMillis': TYPE_INT,
    'InitiatingHeapOccupancyPercent': TYPE_INT,
    'NewRatio': TYPE_INT,
    'SurvivorRatio': TYPE_INT,
    'MaxTenuringThreshold': TYPE_INT,
    'ParallelGCThreads': TYPE_INT,
    'ConcGCThreads': TYPE_INT,
    'G1ReservePercent': TYPE_INT,
    'G1HeapRegionSize': TYPE_INT,

    # Performance Options
    "AggressiveOpts":TYPE_BOOL,
    "CompileThreshold":TYPE_INT,
    "LargePageSizeInBytes":TYPE_INT,
    'MaxHeapFreeRatio':TYPE_INT,
    'MinHeapFreeRatio':TYPE_INT,
    'NewRatio':TYPE_INT,
    'NewSize':TYPE_INT,
    'ReservedCodeCacheSize':TYPE_INT,
    'SurvivorRatio':TYPE_INT,
    'TargetSurvivorRatio':TYPE_INT,
    'ThreadStackSize':TYPE_INT,
    'UseBiasedLocking':TYPE_BOOL,
    'UseFastAccessorMethods':TYPE_BOOL,
    'UseISM':TYPE_BOOL

}

impact_options = {
    'InitiatingHeapOccupancyPercent':[10,20,30,40,50,60,70,80,90],
    'NewRatio':[10,20,30,40,50,60,70,80,90],
    'SurvivorRatio':[10,20,30,40,50,60,70,80,90],
    'MaxTenuringThreshold':[10,15,20],
    'ParallelGCThreads':[1,2,3,4],
    'ParallelGCThreads':[1,2,3,4],
    'ConcGCThreads':[1,2,3,4],
    'G1ReservePercent':[10,15,20]

}

def test_stat():
    print(options_hotspot.keys())

if __name__ == '__main__':
    test_stat()


import os
from google.api_core.exceptions import InvalidArgument
import dialogflow
from flask import Flask, render_template, request, jsonify
from google.api_core.gapic_v1 import method
from db_utils import insert_conversations, get_conversations, get_conversations_by_date, get_misses_by_date, get_misses
from file_ops import write_fallback_csv

os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = 'private_key.json'
DIALOGFLOW_PROJECT_ID = 'newagent-jojg'
DIALOGFLOW_LANGUAGE_CODE = 'en'
SESSION_ID = 'me'

#text_to_be_analyzed = "how does covid spread?"

welcome_message = '''Hello! I am a covid bot. I can answer questions related to covid.
You can ask me questions regarding the symptoms, precautions and other general information on covid'''

app = Flask(__name__,
            template_folder='client/templates',
            static_folder='client/static')

def get_response(text_to_be_analyzed):
    session_client = dialogflow.SessionsClient()
    session = session_client.session_path(DIALOGFLOW_PROJECT_ID, SESSION_ID)
    text_input = dialogflow.types.TextInput(text=text_to_be_analyzed, language_code=DIALOGFLOW_LANGUAGE_CODE)
    query_input = dialogflow.types.QueryInput(text=text_input)
    try:
        response = session_client.detect_intent(session=session, query_input=query_input)
        insert_conversations(text_to_be_analyzed, response.query_result.intent.display_name)
        '''if response.query_result.intent.display_name == "Default Fallback Intent":
            write_fallback_csv(text_to_be_analyzed)'''

        return response.query_result.fulfillment_text
    except InvalidArgument:
        raise

@app.route('/')
def index():
    return render_template('chat.html', message=welcome_message)

@app.route('/get_reply', methods=["POST"])
def get_reply():
    text = request.form["text"]
    reply = get_response(text)
    return jsonify({'text': reply})

@app.route('/conversations', methods=["GET", "POST"])
def conversations():
    if request.method == 'GET':
        messages = get_conversations()
        return render_template('conversations.html', messages=messages)
    elif request.method == 'POST':
        from_date = request.form['fromdate']
        to_date = request.form['todate']
        messages = get_conversations_by_date(from_date, to_date)
        return render_template('conversations.html', messages=messages)

@app.route('/misses', methods=["GET", "POST"])
def misses():
    if request.method == 'GET':
        messages = get_misses()
        return render_template('misses.html', messages=messages)
    elif request.method == 'POST':
        from_date = request.form['fromdate']
        to_date = request.form['todate']
        messages = get_misses_by_date(from_date, to_date)
        return render_template('misses.html', messages=messages)

if __name__ == '__main__':
    app.run(debug=True, port=3001)


'''
128. Longest Consecutive Sequence

Given an unsorted array of integers, find the length of the longest consecutive elements sequence.

Your algorithm should run in O(n) complexity.

Example:

Input: [100, 4, 200, 1, 3, 2]
Output: 4
Explanation: The longest consecutive elements sequence is [1, 2, 3, 4]. Therefore its length is 4.
Accepted
278,627
Submissions
629,656

'''

# 2020/04/30, hashtable, too hard
# 2021/06/06，时隔一年完全不记得了。

'''
Runtime: 60 ms, faster than 45.86% of Python3 online submissions for Longest Consecutive Sequence.
Memory Usage: 15.1 MB, less than 7.41% of Python3 online submissions for Longest Consecutive Sequence.
'''

class Solution:
    def longestConsecutive(self, nums: List[int]) -> int:
        nums = set(nums)
        res = 0
        for num in nums:
            if num - 1 not in nums:
                start = num
                curr = 1
                while start + 1 in nums:
                    start += 1
                    curr += 1
                res = max(res, curr)

        return res


import logging
import string
from datetime import datetime

import requests
from bs4 import BeautifulSoup as bs

log_filename = datetime.now().strftime('logs/stock_logs_%H:%M_%d-%m-%Y.log')
logging.basicConfig(filename=log_filename, level=logging.INFO,
                    format='%(asctime)s %(levelname)s %(message)s')
logger = logging.getLogger('analyzer.py')

def nasdaq():
    char = string.ascii_uppercase
    stock_list = []
    logger.info('Fetching tickers for all NASDAQ stocks')
    print('Fetching tickers for all NASDAQ stocks')
    for x in char:
        url = f'http://www.eoddata.com/stocklist/NASDAQ/{x}.htm'
        r = requests.get(url)
        scrapped = bs(r.text, "html.parser")
        d1 = scrapped.find_all('tr', {'class': 'ro'})
        d2 = scrapped.find_all('tr', {'class': 're'})
        for link in d1:
            stock_list.append(f"{(link.get('onclick').split('/')[-1]).split('.')[0]}")
        for link in d2:
            stock_list.append(f"{(link.get('onclick').split('/')[-1]).split('.')[0]}")

    return stock_list

if __name__ == '__main__':
    from pprint import pprint

    pprint(nasdaq())


from rest_framework import serializers
from .models import *

# 序列化器
# http://www.iamnancy.top/djangorestframework/Serializers/

class AccountBasicSerializer(serializers.ModelSerializer):
    class Meta:
        model = AccountBasic
        fields = ('uid', 'username', 'mobile', 'integral', 'createTime', 'campusCert', 'messageNotice',
                  'nickname', 'avatar', 'gender', 'birthday', 'introduction', 'position', 'campus', 'university')
        read_only_fields = ('username', 'mobile', 'integral', 'createTime')

    def validate_nickname(self, value):
        '''昵称，2-10 位'''
        if len(value) < 2 or len(value) > 10:
            raise serializers.ValidationError("昵称须为 2-10 个字")
        return value

    def validate_avatar(self, value):
        '''头像，以 avatar/ 开头；或者是 url 地址，以 http 开头'''
        if value.find('avatar/') != 0 and value.find('http') != 0:
            raise serializers.ValidationError("非法的头像")
        return value

    def validate_introduction(self, value):
        '''个人介绍，100 字以内'''
        if len(value) > 100:
            raise serializers.ValidationError("个人介绍最多 100 字")
        return value

    def validate_position(self, value):
        '''位置，20 字以内'''
        if len(value) > 20:
            raise serializers.ValidationError("位置最多 20 字")
        return value

    def validate_university(self, value):
        '''大学，20 字以内'''
        if len(value) > 20:
            raise serializers.ValidationError("大学最多 20 字")
        return value

    def to_representation(self, instance):
        # 检查该账号绑定的 OpeniID
        ret = super().to_representation(instance)
        # 查询绑定的 OpenID
        uid = ret['uid']
        record = WechatBasic.objects.filter(uid=uid)
        openid_arr = []
        if len(record) > 0:
            for item in record:
                openid_arr.append(item.openid)
        ret['wechat'] = openid_arr

        # 将校园认证的 cid 转换为认证信息
        if instance.campusCert is not None:
            ret['campusCert'] = {
                'libCardID': instance.campusCert.libCardID,
                'libUserName': instance.campusCert.libUserName
            }
        return ret

class UserInfoSerializer(serializers.ModelSerializer):
    class Meta:
        model = AccountBasic
        fields = ('uid', 'nickname', 'avatar', 'gender',
                  'campus', 'campusCert', 'introduction')

    def to_representation(self, instance):
        ret = super().to_representation(instance)
        if ret['introduction'] is None and ret['gender'] != '0':
            if ret['gender'] == '1':
                ret['introduction'] = '做一个安静的美男子'
            elif ret['gender'] == '2':
                ret['introduction'] = '做一个安静的小仙女'
        return ret

class WechaSettingSerializer(serializers.ModelSerializer):
    class Meta:
        model = WechatSetting
        fields = ('key', 'value')

class IssueBasicDetailSerializer(serializers.ModelSerializer):
    class Meta:
        model = IssueBasic
        fields = ('iid', 'title', 'description', 'view',
                  'status', 'createTime', 'updateTime')

class IssueBasicListSerializer(serializers.ModelSerializer):
    class Meta:
        model = IssueBasic
        fields = ('iid', 'title', 'view', 'status')

class WechatEntranceSerializer(serializers.ModelSerializer):
    class Meta:
        model = WechatEntrance
        fields = ('name', 'icon', 'tip', 'url')

class WechatBannerSerializer(serializers.ModelSerializer):
    class Meta:
        model = WechatBanner
        fields = ('title', 'image', 'url')


from __future__ import division
import time
import scipy.io
import tensorflow as tf
import tensorflow.contrib.slim as slim
import numpy as np
import glob
from astropy.io import fits
import matplotlib.pyplot as plt
import os
import sys
from astropy.stats import biweight_location
from astropy.stats import mad_std
from astropy.stats import sigma_clipped_stats
from datetime import datetime
from datetime import timedelta
os.environ['CUDA_DEVICE_ORDER'] = 'PCI_BUS_ID'
os.environ['CUDA_VISIBLE_DEVICES'] = str(3)   # The id of your GPU

# folders names change them according yours
# -------------------------------
folder_num = str('Network1')
# create_image_noise = False
input_dir = './dataset/in_data_poisson/'   # noisy images
gt_dir = './dataset/out_data/'             # ground truth
checkpoint_dir = './astro_result' + folder_num + '/'
result_dir = './astro_result' + folder_num + '/'   # results folder
test_net = './astro_result' + folder_num + '/test_net/'
name_txt_file = result_dir + "test" + folder_num + ".txt"
path_val = "./dataset/validation/"
path_noise_val = "./dataset/validation/noise/"

# -------------------------------

# other parameters
# -------------------------------
ps = 256  # patch size for training
save_freq = 500
ron = 3   # e
dk = 7   # e/hr/pix;
# -------------------------------

if not os.path.isdir(checkpoint_dir):
                os.makedirs(checkpoint_dir)

def lrelu(x):
    return tf.maximum(x * 0.2, x)

def upsample_and_concat(x1, x2, output_channels, in_channels, exp_time=None, exp=False):
    pool_size = 2
    deconv_filter = tf.Variable(tf.truncated_normal([pool_size, pool_size, output_channels, in_channels], stddev=0.02))
    deconv = tf.nn.conv2d_transpose(x1, deconv_filter, tf.shape(x2), strides=[1, pool_size, pool_size, 1])
    if not exp:
        deconv_output = tf.concat([deconv, x2], 3)
        deconv_output.set_shape([None, None, None, output_channels * 2])
    if exp:
        cons = tf.fill(tf.shape(deconv), exp_time)
        c = tf.slice(cons, [0, 0, 0, 0], [-1, -1, -1, 1])
        deconv_output = tf.concat([deconv, x2, c], 3)
        deconv_output.set_shape([None, None, None, output_channels * 2 + 1])
    return deconv_output

def network(input, e):
    conv1 = slim.conv2d(input, 32, [3, 3], rate=1, activation_fn=lrelu, scope='g_conv1_1')
    conv1 = slim.conv2d(conv1, 32, [3, 3], rate=1, activation_fn=lrelu, scope='g_conv1_2')
    pool1 = slim.max_pool2d(conv1, [2, 2], padding='SAME')

    conv2 = slim.conv2d(pool1, 64, [3, 3], rate=1, activation_fn=lrelu, scope='g_conv2_1')
    conv2 = slim.conv2d(conv2, 64, [3, 3], rate=1, activation_fn=lrelu, scope='g_conv2_2')
    pool2 = slim.max_pool2d(conv2, [2, 2], padding='SAME')

    conv3 = slim.conv2d(pool2, 128, [3, 3], rate=1, activation_fn=lrelu, scope='g_conv3_1')
    conv3 = slim.conv2d(conv3, 128, [3, 3], rate=1, activation_fn=lrelu, scope='g_conv3_2')
    pool3 = slim.max_pool2d(conv3, [2, 2], padding='SAME')

    conv4 = slim.conv2d(pool3, 256, [3, 3], rate=1, activation_fn=lrelu, scope='g_conv4_1')
    conv4 = slim.conv2d(conv4, 256, [3, 3], rate=1, activation_fn=lrelu, scope='g_conv4_2')
    pool4 = slim.max_pool2d(conv4, [2, 2], padding='SAME')

    conv5 = slim.conv2d(pool4, 512, [3, 3], rate=1, activation_fn=lrelu, scope='g_conv5_1')
    conv5 = slim.conv2d(conv5, 512, [3, 3], rate=1, activation_fn=lrelu, scope='g_conv5_2')

    up6 = upsample_and_concat(conv5, conv4, 256, 512, exp_time=e, exp=True)
    conv6 = slim.conv2d(up6, 256, [3, 3], rate=1, activation_fn=lrelu, scope='g_conv6_1')
    conv6 = slim.conv2d(conv6, 256, [3, 3], rate=1, activation_fn=lrelu, scope='g_conv6_2')

    up7 = upsample_and_concat(conv6, conv3, 128, 256)
    conv7 = slim.conv2d(up7, 128, [3, 3], rate=1, activation_fn=lrelu, scope='g_conv7_1')
    conv7 = slim.conv2d(conv7, 128, [3, 3], rate=1, activation_fn=lrelu, scope='g_conv7_2')

    up8 = upsample_and_concat(conv7, conv2, 64, 128)
    conv8 = slim.conv2d(up8, 64, [3, 3], rate=1, activation_fn=lrelu, scope='g_conv8_1')
    conv8 = slim.conv2d(conv8, 64, [3, 3], rate=1, activation_fn=lrelu, scope='g_conv8_2')

    up9 = upsample_and_concat(conv8, conv1, 32, 64)
    conv9 = slim.conv2d(up9, 32, [3, 3], rate=1, activation_fn=lrelu, scope='g_conv9_1')
    conv9 = slim.conv2d(conv9, 32, [3, 3], rate=1, activation_fn=lrelu, scope='g_conv9_2')

    conv10 = slim.conv2d(conv9, 1, [1, 1], rate=1, activation_fn=None, scope='g_conv10')
    return conv10

def fits_read(name):
    f = fits.open(name)
    if "EXPTIME" in f[0].header:
        e = f[0].header["EXPTIME"]
    else:
        e = 0
    return e

def save_fits(image, name, path):
    hdu = fits.PrimaryHDU(image)
    hdul = fits.HDUList([hdu])
    hdul.writeto(path + name + '.fits')

def squeeze_img(array, fil, ps=256):
    """change dimension of network output"""
    out = np.squeeze(array, axis=0)
    o = np.zeros((ps, ps, 1))
    o[:, :, 0] = out[:, :, fil]
    out = np.squeeze(o, axis=2)
    return out

def valid_img(epoch, cnt):
    """test network during training"""
    if not os.path.isdir(test_net + '%04d' % epoch):
        os.makedirs(test_net + '%04d' % epoch)
    path = "./dataset/validation/noise/"
    data_list = glob.glob(path + "*.fits")
    v_num = np.random.randint(0, len(data_list) - 6)
    for name in data_list[v_num: v_num+1]:
        one_image(name, test_net + '%04d/' % epoch, cnt)
    print("validation done")

def gen_image(img, fil, i, j, exp_time=None, ps=256, end_i=False, end_j=False):
    i_end = i + ps
    j_end = j + ps
    if end_i:
        i_end = None
    if end_j:
        j_end = None
    in_imag = img[i: i_end, j: j_end]
    in_img = np.zeros((ps, ps, 1))
    in_img[:, :, 0] = in_imag
    in_patch = np.expand_dims(in_img, axis=0)
    output = sess.run(out_image, feed_dict={in_image: in_patch, ex: 2})
    output = squeeze_img(output, 0)
    return output

def one_image(name, sv_path, cnt):
    """iterate whole image, predict output and make one big image"""
    ps = 256
    fil = 0
    img = fits.getdata(name, ext=0)
    step = 256
    div = ps/step
    W, H = img.shape[0: 2]
    w_diff = (ps - (W % 256))
    w_c = ps + int(np.ceil(w_diff/2))
    w_f = ps + int(np.floor(w_diff/2))
    h_diff = (ps - (H % 256))
    h_c = ps + int(np.ceil(h_diff/2))
    h_f = ps + int(np.floor(h_diff/2))
    w = W + w_c+w_f
    h = H + h_c+h_f
    image = np.zeros((w, h, 1))
    image[w_c: w_c + W, h_c: h_c + H, 0] = img
    output = np.zeros((w, h))
    print(w/ps, h/ps)
    for i in range(0, w, step):
        for j in range(0, h, step):
            in_patch = image[i: i+ps, j: j+ps]
            in_patch = np.expand_dims(in_patch, axis=0)
            new_image = sess.run(out_image, feed_dict={in_image: in_patch, ex: 2})
            output[i:i+ps, j: j+ps] += new_image[0, :, :, 0]
    output = (output/div**2)[w_c: w_c + W, h_c:h_c + H]
    save_fits(output, name[27:-5] + "_" + str(cnt) + "_output", sv_path)

def text_write(file, epoch, cnt, loss, time, name):
    """save parameters into text file"""
    file = open(file, "a")
    file.write(str(epoch) + "\t" + str(cnt) + "\t" + str(loss) + "\t" + str(time) + "\t" + str(name) + "\n")
    file.close

def black_level(arr, max_num, level=0.1):
    """Prevent to have an image with more than some percentage of zeroes as input
       level - percentage <0,1>; 0.1/10% default"""
    arr = list(np.hstack(arr))
    per = arr.count(0.00000000e+00)/len(arr)
    if max_num > 10:
        level = 0.3
    if per < level or max_num > 15:
        return True
    else:
        return False

def poisson_noise(name, ratio=2, xx=None, yy=None, ps=None, ron=3, dk=7, sv_name=None, path=None, patch_noise=False, ret=True, save=False):
    """add noise into image"""
    img_data = fits.getdata(name, ext=0)   # e/sec
    if patch_noise:
        img_data = img_data[xx:xx + ps, yy:yy + ps]
    width, height = img_data.shape[0:2]
    exp_time = int(name[-10:-5])
    img = img_data * exp_time   # e
    DN = np.random.poisson(np.sqrt(dk*exp_time/ratio), (width, height))
    RON = np.random.poisson(ron, (width, height))
    SN = np.random.poisson(np.sqrt(img/ratio))
    noise_img = (img/ratio + SN + RON + DN)/(exp_time/ratio)
    noise_img = np.where(img_data == 0.00000000e+00, 0.00000000e+00, noise_img)
    if save:
        save_fits(noise_img, sv_name, path)
    if ret:
        return noise_img

sess = tf.Session()
in_image = tf.placeholder(tf.float32, [None, None, None, 1])
gt_image = tf.placeholder(tf.float32, [None, None, None, 1])
ex = tf.placeholder(tf.float32, name="TIME")
out_image = network(in_image, ex)

G_loss = tf.reduce_mean(tf.abs(out_image - gt_image))
t_vars = tf.trainable_variables()

lr = tf.placeholder(tf.float32)
G_opt = tf.train.AdamOptimizer(learning_rate=lr).minimize(G_loss)
saver = tf.train.Saver()
sess.run(tf.global_variables_initializer())
ckpt = tf.train.get_checkpoint_state(checkpoint_dir)

if ckpt:
    print('loaded ' + ckpt.model_checkpoint_path)
    saver.restore(sess, ckpt.model_checkpoint_path)
g_loss = []
data_list = glob.glob(gt_dir + "*.fits")
allfolders = glob.glob('./astro_result' + folder_num + '/*0')
lastepoch = 0
for folder in allfolders:
    lastepoch = np.maximum(lastepoch, int(folder[-4:]))

learning_rate = 1e-4
for epoch in range(lastepoch, 8001):
    num = 0
    if os.path.isdir('./astro_result' + folder_num + '/%04d' % epoch):
        continue
    cnt = 0
    if epoch > 2000:
        learning_rate = 1e-5

    for name in np.random.permutation(data_list):
        st = time.time()
        num += 1
        out = fits.getdata(name, ext=0)
        ex_time = float(int(name[-10:-5])//2)   # exposure time is in the name of images, also in fits header

        H, W = out.shape[0], out.shape[1]
        zero_level = False
        max_num = 0
        while not zero_level:
            xx = np.random.randint(0, H - ps)
            yy = np.random.randint(0, W - ps)
            arr = out[xx:xx + ps, yy:yy + ps]
            zero_level = black_level(arr, max_num)
            max_num += 1
        # ground truth
        out_img = np.zeros((ps, ps, 1))
        out_img[:, :, 0] = out[xx:xx + ps, yy:yy + ps]
        gt_patch = np.expand_dims(out_img, axis=0)

        # input image
        in_name = glob.glob(input_dir + name[19:-11] + "*.fits")
        in_ = fits.getdata(in_name[0], ext=0)
        in_imag = in_[xx:xx + ps, yy:yy + ps]
        del in_
        in_img = np.zeros((ps, ps, 1))
        in_img[:, :, 0] = in_imag
        in_patch = np.expand_dims(in_img, axis=0)

        cnt += 1
        _, G_current, output = sess.run([G_opt, G_loss, out_image],
                                        feed_dict={in_image: in_patch, ex: 2, gt_image: gt_patch, lr: learning_rate})

        g_loss.append(G_current)
        print("%d %d Loss=%.3f Time=%.3f" % (epoch, cnt, np.mean(g_loss), time.time() - st))

        if cnt % 50 == 0:
            text_write(name_txt_file, epoch, cnt, np.mean(G_current), time.time() - st, name[19:])

        if epoch % save_freq == 0 and cnt % 50 == 0:
            if not os.path.isdir(result_dir + '%04d' % epoch):
                os.makedirs(result_dir + '%04d' % epoch)
            sv_name = "/" + name[19:-5]
            output = squeeze_img(output, 0)
            save_fits(out[xx:xx + ps, yy:yy + ps], sv_name, result_dir + '%04d' % epoch)
            save_fits(output, sv_name + "_output", result_dir + '%04d' % epoch)
            save_fits(in_imag, sv_name + "_noise", result_dir + '%04d' % epoch)
        if epoch % (save_freq*3) == 0 and cnt % 100 == 0:
            valid_img(epoch, cnt)
    saver.save(sess, checkpoint_dir + 'model.ckpt')
    if epoch >= 1000 and epoch % (save_freq*2) == 0:
        saver.save(sess, checkpoint_dir + 'model%04d.ckpt' % epoch)


"""
Further concern:

gamma的最优化是随机投点的

用每一层（比如w1）的频率分布图来visualize参数的变化。挺好。画均值标准差，也不错，求和感觉有点太笨了。or plot unit值也可以 (tensorboard ?)
gamma的图何不如画在一张图里。

don't use sigmoid, try relu ?

输入应该以0为中心。也就是应该标准化。!!!!!!!!!!!!

均匀分布与正态分布

神经网络参数只更新3次就结束了，会不会有点问题。

调整BN、dropout、lr？

没有分batch

hidden state的数量

lr ?

如果r是随机投点的话，设置初值没什么用！
如果r是梯度下降，那设置初值才有一些用！

reduction=sum

0313代办
1.检查self.eye的设置是不是全部合理  ok
2.检查数值稳定性
3.还是写一个日志吧。logging一下。不然好像太笨了
4.第二次剥离降维时，网络还是要更新吗。
5.何种论文来参照一下，科学论文的方式。
6.这个I-rr'到底是啥。
8.结构到底是什么。  做到什么地步-----

1.introduction literature review
2.介绍我的方法
3.simulation加结果  8-10页

#随机投点时，那些个点的分布。是正弦曲线的样子

#固定数据，固定r的生成时，居然还是出现了不一样的收敛结果，并且收敛结果大大相反！（虽然都是正确的，第一次先降r5，第二次先降r4)
问题出在神经网络参数的随机初始化

# 如果一开始tensor就建立在cuda上？

如果x1,x2,x3都有用，但我们强行投影，能否得出，强行投影的loss大于直接网络的loss？这个很重要。

#停止降维-并不足以识别出他

# 信噪比真的很影响，标准差之比10:2的时候稳的一匹，10:5的时候还将就，10:10彻底拉胯。
# 学习率也真的很重要，在某次结果不好是因为学习率太小，晕了

# 不正交的问题！
就比如降10次维吧，降到第五次的时候，r5和r1,r2,r3,r4的正交性保持的很好，但是到后面，就没那么好了，r8和r2的内积可以达到0.2了（一般是1e-5这个量级）
算法没有问题的，关键就在于比如 r2=(I-r1r1')r0，这么一算，理想情况下r1'r2是等于0，但是不是严格等于呀！慢慢地误差就积累了。
以往那个误差我们可以强制要求r的模长为1，现在这个误差咋办？总不能强制去要求eye的模长吧。可以吗。

check 正交性画个矩阵

# model给出的yhat是batch_size*1的结果
# 你的y也必须要是batch_size*1，而不能是batch_size!!!不然的话，去算yhat与y的损失函数时，会张成一个矩阵去算，却不会报错，太坑了！

#W的容忍度目前还是太大了吧，不要平滑了，比最小大几次就爬，另外可不可以返回原来的状态？
#对，不要平滑。比之前的最小点高三次就爬（连续高三次还是累计高三次？感觉连续高三次好一点，累计高三次有点小离谱，万一某一次是偶然跌到谷底，那不就凉了。

"""

import torch
import copy

import datetime
from torch import nn
import numpy as np
import torch.nn.functional as F
from service.utils import MLP

class Monitor:
    def __init__(self,r_patience=100,W_patience=3,W_MAperiod=5):
        """

        :param r_patience: gamma的模连续稳定多少次，就不等了，停止更新gamma
        :param W_patience: eval_loss连续上升多少次，就不等了，停止更新W。如果W_patience==-1，则不对W做限制
        """

        self.r_patience=r_patience
        self.W_patience=W_patience
        self.W_MAperiod=W_MAperiod

        self.gamma_only = []  #所有gamma的统计信息
        self.update_r_times = 1    #服务于gamma_only的变量，更新了几次r
        self.all_stats=[]     #所有统计信息

        self.num_of_dimension_reduction=1  #目前是第几次降维，每train一次就代表一次降维，这个东西就会加一。
        self.all_gamma_and_eye={}  #所有gamma和eye的统计信息  {1: {'r':tensor, 'eye':tesor} }

    def check_orthogonal(self):
        """
        检查每个gamma之间的正交性，检查用，不参与运算
        :return: p*p矩阵，第[i,j]个元素代表ri和rj的内积
        """
        gammas=[]
        for k in sorted(self.all_gamma_and_eye):
            gammas.append(self.all_gamma_and_eye[k]['gamma'])
        gamma_matrix=torch.cat(gammas,dim=1)
        return gamma_matrix.t().matmul(gamma_matrix)

    def reset_monitor_variable(self):
        self.r_repeat_times=0  #r的模当前维持了多少次，与r_patience比较
        self.update_r_times=1

        self.W_up_times=0   #eval_loss当前上升了多少次，与W_patience比较
        self.evalloss_MovingAverage=[]  #滑动平均所用的list
        self.evalloss_nowMin=np.Inf  #目前的最小值

    def check_update_r_state(self,cos_distance):
        """
        停止更新gamma的逻辑：当连续r_patience次与上一次的cos距离非常近时，就停止更新gamma！注意是连续的
        :param cos_distance: 这一次r与上一次r的cos距离
        """

        r_state=False
        if self.r_repeat_times <= self.r_patience:  # 如果连续r_patience个回合不动弹，那就停止更新gamma没有机会继续更新gamma
            r_state=True
            if np.abs(np.abs(cos_distance) - 1) < 1e-5:
                self.r_repeat_times += 1
            else:
                self.r_repeat_times =0
        return r_state

    def check_update_W_state2(self, eval_loss):
        """
        停止更新W的逻辑：比之前的最小值连续用eval_loss来判断W的停止。如果没有eval_loss就无法停止W的更新，只能更新到最大次数。
        当eval_loss连续W_patience次大于当前最小值时，则停止更新W。当然为了防止随机波动的影响，这个eval_loss是十次滑动平均的eval_loss。
        :param eval_loss:
        :param W_MAperiod:
        :return:
        """

        if eval_loss is None or self.W_patience == -1:
            # 没有eval_loss，那W没有停止标准，W只能继续更新
            # 如果W_patience==-1，意味着W没有停止标准
            return True
        if self.W_up_times > self.W_patience:  # 如果已经超过限制，就永远不得更新
            return False

        if eval_loss < self.evalloss_nowMin:
            self.evalloss_nowMin = eval_loss
            self.W_up_times = 0
        else:
            self.W_up_times += 1

        return True

    def check_update_W_state(self, eval_loss):
        """
        停止更新W的逻辑：用eval_loss来判断W的停止。如果没有eval_loss就无法停止W的更新，只能更新到最大次数。
        当eval_loss连续W_patience次大于当前最小值时，则停止更新W。当然为了防止随机波动的影响，这个eval_loss是十次滑动平均的eval_loss。
        :param eval_loss:
        :param W_MAperiod:
        :return:
        """

        if eval_loss is None or self.W_patience == -1:
            # 没有eval_loss，那W没有停止标准，W只能继续更新
            # 如果W_patience==-1，意味着W没有停止标准
            return True
        if self.W_up_times >= self.W_patience:  # 如果已经超过限制，就永远不得更新
            return False
        if len(self.evalloss_MovingAverage) < self.W_MAperiod:  # 滑动平均阶数还没到，那还是要继续更新
            self.evalloss_MovingAverage.append(eval_loss)
            return True

        self.evalloss_MovingAverage.append(eval_loss)
        self.evalloss_MovingAverage.pop(0)

        now_eval_loss = np.mean(self.evalloss_MovingAverage)
        if now_eval_loss < self.evalloss_nowMin:
            self.evalloss_nowMin = now_eval_loss
            self.W_up_times = 0
        else:
            self.W_up_times += 1

        return True

    def log_all_gamma_and_eye(self,r,eye,NNW_weight):

        self.all_gamma_and_eye[self.num_of_dimension_reduction] =       {'gamma':r,
                                                                         'eye':eye,
                                                                         'train_loss':self.all_stats[-1]['train_loss'],
                                                                         'eval_loss': self.all_stats[-1]['eval_loss'],
                                                                         'NNW_weight':NNW_weight.copy()
                                                                         }
        self.num_of_dimension_reduction+=1

    def log_gamma_only(self,r,true_p):
        gamma_only_stat = {}
        gamma_only_stat['epoch'] = self.update_r_times
        gamma_only_stat['num_of_dimension_reduction'] = self.num_of_dimension_reduction  # 第几次剥离
        for i in range(true_p):  # 储存r
            name = "r" + str(i + 1)
            gamma_only_stat[name] = r[i].item()
        for i in range(true_p):  # 储存r
            name = "r" + str(i + 1)+"平方"
            gamma_only_stat[name] = (r[i].item())**2

        self.gamma_only.append(gamma_only_stat)

    def log_all_stats(self,i, sdr_model, cos_distance,r_state,W_state, train_loss, eval_loss=None):

         # 将一个epoch的信息写进字典
        p = sdr_model.r.shape[0]
        stat = {}
        stat['epoch'] = i
        stat['num_of_dimension_reduction'] = self.num_of_dimension_reduction  #第几次剥离
        stat['update_r']=r_state
        stat['update_W']=W_state

        for name, parameter in sdr_model.NNWmodel.named_parameters():
            stat[name + '.mean'] = parameter.data.mean().item()
            stat[name + '.std'] = parameter.data.std().item()

        for i in range(p):  # 储存r
            name = "r" + str(i + 1)
            stat[name] = sdr_model.r[i].item()

        stat['cosdistance_versus_last_gamma']=cos_distance

        r_norm=0    #储存r的平方
        for i in range(p):  # 读取r
            name = "r" + str(i + 1)
            stat[name+"平方"] = (sdr_model.r[i].item())**2
            r_norm+=(sdr_model.r[i].item())**2
        stat['r_norm']=r_norm

        stat['train_loss'] = train_loss
        stat['eval_loss'] = eval_loss

        self.all_stats.append(stat)

class SDRmodel:

    def __init__(self,p, NNW_params, DEVICE,seed=4396):

        #self.r = None

        self.seed=seed
        self.true_p = p
        self.DEVICE=DEVICE
        self.NNWmodel = self.get_NNWmodel(NNW_params)

        self.loss_function = NNW_params['loss_function'].to(self.DEVICE)
        # self.optimizer=optimizer
        self.optimizer = torch.optim.Adam(self.NNWmodel.parameters(), lr=NNW_params['learning_rate'],weight_decay=NNW_params['weight_decay'])
        self.cos=nn.CosineSimilarity(dim=0, eps=1e-6)

        self.monitor=Monitor()

    def get_NNWmodel(self,NNW_params):

        torch.manual_seed(self.seed)  # 神经网络的权重初始化也是随机的，这个也挺坑的。
        hidden=NNW_params['hidden']
        if not isinstance(hidden,list):
            hidden=[hidden]
        NNWmodel = MLP([self.true_p]+hidden+[1],activation=NNW_params['activation'])
        NNWmodel.to(self.DEVICE)
        param_names = [name for name, _ in NNWmodel.named_parameters()]
        print(param_names)

        return NNWmodel

    @property
    def get_all_gamma_and_eye(self):
        tmp=self.monitor.all_gamma_and_eye.copy()
        for item in tmp.values():
            item.pop('NNW_weight')
        return tmp   #其实就是返回monitor.all_gamma_and_eye，但是把NNW_weight这个键去掉，这个太长了

    @property
    def get_all_stats(self):
        return self.monitor.all_stats.copy()
    @property
    def get_gamma_only(self):
        return self.monitor.gamma_only.copy()
    @property
    def get_best_DR_space(self):
        stats = self.monitor.all_gamma_and_eye.copy()
        if len(stats)==0:
            raise ValueError
        minLoss=np.Inf
        bestDimensionReduction=0
        #找出最优的降维次数
        for k,v in stats.items():
           if v['eval_loss']<minLoss:
                minLoss=v['eval_loss']
                bestDimensionReduction=k

        return {'method':'SDRNNW',
                'DimensionFound':self.true_p-bestDimensionReduction, #表示找到了几个低秩结构，所以应该是全部维度减去降维次数
                'EstimatedBasis': None,
                'EstimatedSpace':stats[bestDimensionReduction]['eye'].clone().numpy(),
                }

    def check_orthogonal(self):
        return self.monitor.check_orthogonal()

    def set_r(self,r):
        # 设置r，处理完一个维度之后要处理下一个维度，所以r需要被反复设置。
        self.r=r.clone()

    def update_eye(self,r=None):
        """
        剥离多次的情况，见算法。
        为了保证后面的降维维度和之前的降维维度正交。需将前面的方向减掉。等价于替换掉eye。
        如果不指定，就设置为单位eye（这是最刚开始的情况）
        :param r: 已经找好的方向。需要去掉
        :return:
        """
        if r is None:
            self.eye = torch.eye(self.true_p, self.true_p, dtype=torch.float, requires_grad=False).to(self.DEVICE)
        else:
            self.eye=self.eye-r.matmul(r.t())

    def test(self,x, y=None ,state="train"):
        """

        :param x:
        :param y:
        :param state:state="train", "finish"
        train意味着依然处于训练中，此时test就利用当前gamma值来算
        finish意味着已经结束训练，此时test应该根据最优降维次数下的参数计算，降维次数的判断通过self.monitor.all_gamma_and_eye中的eval_loss，该降维次数下的参数也存在这个dict里面
        remind that 调用state=='finish'会将
        :return:
        """
        if state=="train":
            _,loss=self.forward(x,y)
            return loss
        elif state=="finish":
            stats=self.monitor.all_gamma_and_eye.copy()

            minLoss=np.Inf
            bestDimensionReduction=0
            #找出最优的降维次数
            for k,v in stats.items():
                if v['eval_loss']<minLoss:
                    minLoss=v['eval_loss']
                    bestDimensionReduction=k

            tmpModel=copy.deepcopy(self.NNWmodel) #必须是深拷贝！
            tmpModel.load_state_dict(stats[bestDimensionReduction]['NNW_weight'])

            x_star = x.matmul(stats[bestDimensionReduction]['eye'])  # x_star
            yhat = tmpModel(x_star)

            result = (bestDimensionReduction,yhat,)
            if y is not None:
                if yhat.shape != y.shape:
                    raise ValueError("dimension not match !!")
                loss = self.loss_function(yhat, y)
                result += (loss,)
            return result

        else:
            raise ValueError("wrong input for state")

    def forward(self, x, y=None,r=None):
        if r is None:
            r=self.r.clone() #在少部分时候，使用的是自己的r，并不是类的r。当然如果不指明，就认为是用类的r

        x_star = x.matmul(self.eye - r.matmul(r.t()))  # x_star
        yhat = self.NNWmodel(x_star)
        result = (yhat,)
        if y is not None:
            if yhat.shape!=y.shape:
                raise ValueError("dimension not match !!")
            loss = self.loss_function(yhat, y)
            result += (loss,)
        return result

    def update_W(self, x, y, verbose=True):

        steps = 1
        while steps<=3:
            _, loss = self.forward(x, y)  # 第一个返回值是yhat，没用，只需要loss即可。
            if verbose:
                print("update W, loss:", loss.item())

            self.optimizer.zero_grad()
            loss.backward()
            self.optimizer.step()  # optimizer只会更新网络中的参数，不会更新r

            #new_param = self.NNWmodel.fc1._parameters['weight'].clone()
            #print("w1 norm:",(old_param-new_param).norm().item())
            # print("diff%f"%(new_norm-old_norm) )
            #             if np.abs((new_norm-old_norm))<threshold:
            #                 break
            # old_param = new_param.clone()
            steps += 1

    def cal_l_round_x(self,x,y,r=None):
        #对最原始的那个x的导数

        if r is None:
            r=self.r.clone()   #在少部分时候，使用的是自己的r，并不是类的r。当然如果不指明，就认为是用类的r
        n=x.shape[0]

        ##前向传播
        #eye = torch.eye(self.true_p, self.true_p, dtype=torch.float)
        x_star = x.matmul(self.eye - r.matmul(r.t()))
        x_star.requires_grad = True
        s2 = self.NNWmodel(x_star)

        if s2.shape!=y.shape:
            raise  ValueError ("dimension not match!!")

        loss = self.loss_function(s2, y)

        ##反向传播计算梯度，但是并不更新，目的只是去得到x_star的梯度。
        self.optimizer.zero_grad()
        loss.backward()

        l_round_x_star = x_star.grad  # n*p
        l_round_x=l_round_x_star.matmul( self.eye - r.matmul(r.t()) ) # n*p
        return l_round_x

    def cal_l_round_r(self, x, y,r=None):
        if r is None:
            r=self.r.clone()   #在少部分时候，使用的是自己的r，并不是类的r。当然如果不指明，就认为是用类的r
        n=x.shape[0]

        ##前向传播
        #eye = torch.eye(self.true_p, self.true_p, dtype=torch.float)
        x_star = x.matmul(self.eye - r.matmul(r.t()))
        x_star.requires_grad = True
        s2 = self.NNWmodel(x_star)

        if s2.shape!=y.shape:
            raise ValueError("dimension not match!!")

        loss = self.loss_function(s2, y)

        ##反向传播计算梯度，但是并不更新，目的只是去得到x_star的梯度。
        self.optimizer.zero_grad()
        loss.backward()

        l_round_x_star = x_star.grad

        ## 有了x_star的梯度，即去计算得出r的梯度
        l_round_r = torch.zeros(self.true_p, 1, dtype=torch.float).to(self.DEVICE)
        eye = torch.eye(self.true_p, self.true_p, dtype=torch.float).to(self.DEVICE)   #这里用的eye不是self.eye，看看梯度就知道了！
        for i in range(n):
            x_star_i_round_r = -( r.matmul(x[i, :].view(1, -1)) + r.t().matmul(x[i, :]) * eye  )  #负号为什么能忘
            l_i_round_r = x_star_i_round_r.matmul(l_round_x_star[i, :].reshape(self.true_p, 1))
            l_round_r += l_i_round_r
        return l_round_r

    def update_r(self, x, y, verbose=True, threshold=1e-5):
        # 更新参数r，此时W是不动的，相当于

        def find_best_t(h, r,method='random_search',search_points=1000,verbose=True):
            ## 用格点搜索的办法，找到一个最优的t
            ## 也可以梯度下降，但是格点法会简便一下。我认为效果不会差太多
            if method=='random_search':
                search_min = 0
                search_max = 2 * np.pi / h.norm()
                delta = (search_max - search_min) / search_points
                min_value = 1000000
                optimal_t = 0

                for i in range(search_points):
                    t = search_min + i * delta
                    this_r = cal_r_by_t(h, t, r)

                    yhat, this_loss = self.forward(x, y, this_r)
                    this_loss=this_loss.item()
                    if this_loss < min_value:
                        min_value = this_loss
                        optimal_t = t
                if verbose:
                    print("update r, loss:", min_value)
                return optimal_t
            elif method=="GD":
                t=0
                for _ in range(100):
                    f_t=cal_r_by_t(h,t,r)
                    yhat , loss=self.forward(x,y,f_t)
                    loss=loss.item()
                    print(f"in GD for t, round: {_},loss:{loss}")

                    l_round_r=self.cal_l_round_r(x, y, f_t)  # p*1
                    f_t_round_t=  - h.norm()* r * torch.sin(h.norm()*t) +  h*torch.cos(h.norm()*t )  #p*1
                    l_round_t = l_round_r.t().matmul(f_t_round_t)  #标量，t的梯度
                    t -= l_round_t*0.01
                if verbose:
                    print("update r, loss:", loss)
                return t

        def cal_r_by_t(h, t, r):
            result = r * torch.cos(h.norm() * t) + h / h.norm() * torch.sin(h.norm() * t)

            #本来计算出result就ok了，但是因为数值运算的误差，导致result模不严格等于1，与之前的gamma正交性也不能完全保证。
            #尽管这个误差非常小，但是在长期积累之后就相当客观，不可小觑，因此必须每次运算后都严格规范。
            #即规范本次gamma和之前gamma的正交性，规范其模长严格为1.
            tmp=self.eye.matmul(result)
            tmp=tmp/tmp.norm()
            return tmp

        r=self.r.clone()

        if verbose:
            print("start updating r in this round:", r)

        l_round_r = self.cal_l_round_r(x, y, r)
        g = (self.eye - r.matmul(r.t())).matmul(l_round_r)
        h = -g

        k = 0  # 计数项
        while k < 20:  # r还是不要弄太多次吧，时间太久了。虽然一般也不会超过20

            # 找到最优的t
            t = find_best_t(h,r,verbose=verbose)
            # 更新r，同时保留旧值
            old_r = r.clone()
            #print("check orthogonial of h and r",h.t().matmul(r).item())
            r = cal_r_by_t(h, t ,r)

            #记录只有gamma的统计情况
            self.monitor.log_gamma_only(r,self.true_p)

            # 判断终止条件
            # print("r_diff:",(self.r-old_r).norm())
            if (r - old_r).norm() <= threshold:
                break

            g_plus_1 = (self.eye - r.matmul(r.t())).matmul(self.cal_l_round_r(x, y, r))
            h_tao = -old_r * torch.sin(h.norm() * t)*h.norm()  + h  * torch.cos(h.norm() * t)
            g_tao = g - old_r * torch.sin(h.norm() * t) + h / h.norm() * (1 - torch.cos(h.norm() * t)) * h.t().matmul(
                g) / h.norm()
            delta = (g_plus_1 - g_tao).t().matmul(g_plus_1) / g.t().matmul(g)

            # 更新
            h = -g_plus_1 if (k + 1) % (self.true_p - 1) == 0 else (-g_plus_1 + delta * h_tao)
            g = g_plus_1
            k += 1

        if verbose:
            print("final updated r in this round:", r)
            print("cos distance between r and last r:",self.cos(self.r,r).item()  )
        self.r=r.clone()
        self.monitor.update_r_times+=1

    def train_one_dimension(self,x,y,max_exp_times,x_eval=None,y_eval=None, verbose=1):
        """

        :param x:
        :param y:
        :param exp_times: 最大训练次数，gamma的训练次数小于等于此
        :param x_eval:
        :param y_eval:
        :param verbose: 0:不打印任何，1:隔100个epoch打印, 2：全部打印
        :return:
        """
        print(f"start at {str(datetime.datetime.now())}")

        last_r=self.r.clone()  #用于计算cos距离的

        self.monitor.reset_monitor_variable()

        for i in range(max_exp_times ):

            this_epoch_print= verbose==2 or (verbose==1 and i%100==0)

            if this_epoch_print:
                print(f"*********{i}/{max_exp_times}*********")

            cos_distance=self.cos(last_r,self.r).item()  #和上一组gamma之间的cos距离
            last_r=self.r.clone()
            _, train_loss = self.forward(x, y)
            train_loss=train_loss.item()
            eval_loss=None
            if x_eval is not None and y_eval is not None:
                eval_loss = self.test(x_eval, y_eval)
                eval_loss=eval_loss.item()

            r_state = self.monitor.check_update_r_state(cos_distance)
            W_state = self.monitor.check_update_W_state(eval_loss)

            self.monitor.log_all_stats( i, self, cos_distance, r_state, W_state, train_loss, eval_loss )

            if not r_state and not W_state:
                break

            if W_state:
                self.update_W(x, y,verbose=this_epoch_print)
            if r_state:
                self.update_r(x, y,verbose=this_epoch_print)

    def generate_random_r(self,distribution,params,seed=1):
        if distribution == 'normal':
            np.random.seed(seed)
            r = np.random.normal(params['mean'], params['sigma'], (self.true_p,1))
        elif distribution == 'uniform':
            np.random.seed(seed)
            r = np.random.uniform(params['lbound'], params['ubound'], (self.true_p, 1))
        else:
            raise ValueError("distribution %s not avaiable" % distribution)

        r = torch.tensor(r, requires_grad=False, dtype=torch.float).to(self.DEVICE)
        return r

    def train(self,x,y,x_eval,y_eval,max_exp_times=3000,DR_times=None,verbose=1):
        # 如果没有指明降维次数，那就拉满，降维p-1次
        if DR_times is None:
            DR_times=self.true_p-1

        # 初始化第一次降维的r；之后几次降维中，r的初始化由SwitchToNextStrip完成

        r=self.generate_random_r('uniform',{'lbound':-1,'ubound':1},seed=self.seed)
        r = r / r.norm()
        self.set_r(r)
        self.update_eye()

        # 训练
        for DR_iter in range(1, DR_times + 1):  # 降几次维
            print(f"##现在是第{DR_iter}/{DR_times}次降维##")
            self.train_one_dimension(x, y, max_exp_times, x_eval=x_eval, y_eval=y_eval, verbose=verbose)
            self.SwitchToNextStrip(seed_for_r=self.seed + DR_iter)

    def SwitchToNextStrip(self,seed_for_r=None):
        #更新eye
        print("#########SwitchToNextStrip###########")
        r_completed = self.r.clone()
        r_completed = r_completed/r_completed.norm()
        print(r_completed)

        print(self.eye)
        self.update_eye(r_completed)
        print(self.eye)

        #储存gamma和eye的信息
        self.monitor.log_all_gamma_and_eye(r_completed.clone(),self.eye.clone(),self.NNWmodel.state_dict())

        #建立下一个r
        if seed_for_r is None:
            seed_for_r=np.random.randint(0,100000,1).item()
        r = self.generate_random_r('uniform', {'lbound': -1, 'ubound': 1}, seed=seed_for_r)
        ##为了保持正交
        r = self.eye.matmul(r)
        r = r / r.norm()

        for key,value in self.monitor.all_gamma_and_eye.items():
            print(f"compared with gamma {key}, the dot product is {value['gamma'].t().matmul(r).item()} "  )

        self.set_r(r)
        print("#########SwitchEnd###########")

def weights_init_uniform_rule(m):
    classname = m.__class__.__name__
    # for every Linear layer in a model..
    if classname.find('Linear') != -1:
        # get the number of the inputs
        n = m.in_features
        y = 1.0/np.sqrt(n)
        m.weight.data.uniform_(-y, y)
        m.bias.data.fill_(0)


#!/usr/bin/python
# -*- coding: utf-8 -*-

import re
import sys
import time
from char_normalization import char_normalizer
from BeautifulSoup import BeautifulStoneSoup
import htmllib

# For HTML processing
class unescaper:
    def __init__(self):
        self._p = htmllib.HTMLParser(None)
    def run(self,s):
        if len(s) > 0:
            aux = BeautifulStoneSoup(s,convertEntities=BeautifulStoneSoup.HTML_ENTITIES).contents
            if len(aux) > 0:
                return aux[0]
            else:
                return u''
        else:
            return u''
        # self._p.save_bgn()
        # self._p.feed(s)
        # return self._p.save_end()

class empty_line_remover:
    def __init__(self):
        self._empty_line_rgx = re.compile(u'\n[\n\r\s]+')
    def run(self,s):
        return self._empty_line_rgx.sub(u'\n',s)

def lookup(index,entry,default = 0):
    if entry in index:
        return index[entry]
    else:
        return default
def inc(index,entry):
    if entry in index:
        index[entry] += 1
    else:
        index[entry] = 1
def inc_info(index,entry,info):
    if entry in index:
        (score,old_info) = index[entry]
        index[entry] = (score+1,info)
    else:
        index[entry] = (1,info)
def float_inc_info(index,entry,info):
    if entry in index:
        (score,old_info) = index[entry]
        index[entry] = (score+1.0,info)
    else:
        index[entry] = (1.0,info)

def log_str(msg):
    return u'[%s] %s\n' % (time.ctime(),msg)
def log_stderr(msg):
    sys.stderr.write((u'[%s] %s\n' % (time.ctime(),msg)).encode('utf-8'))
def log_stdout(msg):
    sys.stdout.write((u'[%s] %s\n' % (time.ctime(),msg)).encode('utf-8'))

class normaliser:
    def __init__(self):
        self._white = re.compile(u'[\s\t\r\n,\.\\\/\(\)\[\]\!\?\{\}&\-:;\+\*\`\@\'\"]+')
    def run(self,s):
        return self._white.sub(u' ',s).lower().strip()
class advanced_normaliser:
    def __init__(self,configpath):
        self._norm  = char_normalizer()
        self._norm.load_latin_std(configpath)
        self._white = re.compile(u'[\s\t\r\n,\.\\\/\(\)\[\]\!\?\{\}&\-:;\+\*\`\@\'\"]+')
    def run(self,s):
        return self._white.sub(u' ',self._norm.run(s)).lower().strip()
class empty_normaliser:
    def run(self,s):
        return s

class stopwords:
    def __init__(self,path,include_numbers = False):
        self._index = {}
        f = open(path,'r')
        for line in f.readlines():
            self._index[line.strip().lower()] = True
        f.close()
        self._include_numbers = include_numbers
        self._numbers_rgx = re.compile(r'\d+')

    def is_stop(self,s):
        if s in self._index:
            return True
        elif self._include_numbers and self._numbers_rgx.match(s):
            return True
        else:
            return False

class clean_isbn10:
    def __init__(self):
        self._cleaner = re.compile(r'[\-; ]+')

    def run(self,isbn):
        return self._cleaner.sub('',isbn).strip()

# Add a column to a vector. Create empty columns if the vector
# is not long enough.
import sys
def add_col(vec,col,value):
    col = int(col)
    while len(vec) <= col:
        vec.append(u'')
    vec[col] = value

# Sorts a set of rows by the values of the specified column.
# When the specified column is empty, the default value will be
# assumed.
def veccmp(x,y):
    (vx,ex) = x
    (vy,ey) = y
    return cmp(vx,vy)
def sort_row_vec(vec,col,default = 0):
    auxvec = []
    for elem in vec:
        val = elem[col].strip()
        if val == '':
            val = default
        else:
            val = int(val)
        auxvec.append((val,elem))
    auxvec.sort(veccmp)
    return [elem for (val,elem) in auxvec]

# Makes a string ready for being used as a regex, by escaping
# characters that have special meanings in regex.
class regex_protector:
    def __init__(self):
        self._rgx = re.compile(u'([\(\)\[\]\!\?\.\,\-\*\+])')
    def run(self,s):
        return self._rgx.sub(r'\\\1',s)

def remove_overlaps(elements):
    """
    Expects _elements_ to be a list of (frompos,topos,info) triples
    and returns a reduced list such that there are no overlaps and
    only leftmost-longest elements are left in the resulting list.
    """
    results     = []
    prev_match  = None
    prev_endpos = 0
    for (frompos,endpos,infos) in elements:
        if frompos >= prev_endpos:
            if prev_match:
                results.append(prev_match)
            prev_match  = (frompos,endpos,infos)
            prev_endpos = endpos
        else:
            if prev_match is None or (endpos - frompos) > (prev_match[1] - prev_match[0]):
                prev_match = (frompos,endpos,infos)
                prev_endpos = endpos
    if prev_match:
        results.append(prev_match)
    return results

class UnicodeRange:
    def __init__(self):
        self.hiragana = u'\u3041-\u308f\u3092\u3093'
        self.rare_hiragana = u'\u3090\u3091\u3094-\u3096'
        self.special_hiragana = u'\u309b-\u309f'

unicode_range = UnicodeRange()